<!DOCTYPE html><html lang="en">
<head>
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<title>[2407.06483] Composable Interventions for Language Models</title><meta property="og:description" content="Test-time interventions for language models can enhance factual accuracy, mitigate harmful outputs, and improve model efficiency without costly retraining.
But despite a flood of new methods, different types of interve…">
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="twitter:card" content="summary">
<meta name="twitter:title" content="Composable Interventions for Language Models">
<meta name="twitter:image:src" content="https://ar5iv.labs.arxiv.org/assets/ar5iv_card.png">
<meta name="twitter:image:alt" content="ar5iv logo">
<meta property="og:title" content="Composable Interventions for Language Models">
<meta property="og:site_name" content="ar5iv">
<meta property="og:image" content="https://ar5iv.labs.arxiv.org/assets/ar5iv_card.png">
<meta property="og:type" content="article">
<meta property="og:url" content="https://ar5iv.labs.arxiv.org/html/2407.06483">

<!--Generated on Mon Aug  5 15:32:25 2024 by LaTeXML (version 0.8.8) http://dlmf.nist.gov/LaTeXML/.-->
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">

<script>
  function detectColorScheme(){
    var theme="light";
    var current_theme = localStorage.getItem("ar5iv_theme");
    if(current_theme){
      if(current_theme == "dark"){
        theme = "dark";
      } }
    else if(!window.matchMedia) { return false; }
    else if(window.matchMedia("(prefers-color-scheme: dark)").matches) {
      theme = "dark"; }
    if (theme=="dark") {
      document.documentElement.setAttribute("data-theme", "dark");
    } else {
      document.documentElement.setAttribute("data-theme", "light"); } }

  detectColorScheme();

  function toggleColorScheme(){
    var current_theme = localStorage.getItem("ar5iv_theme");
    if (current_theme) {
      if (current_theme == "light") {
        localStorage.setItem("ar5iv_theme", "dark"); }
      else {
        localStorage.setItem("ar5iv_theme", "light"); } }
    else {
        localStorage.setItem("ar5iv_theme", "dark"); }
    detectColorScheme(); }
</script>
<link media="all" rel="stylesheet" href="/assets/ar5iv-fonts.0.7.9.min.css"><link media="all" rel="stylesheet" href="/assets/ar5iv.0.7.9.min.css"><link media="all" rel="stylesheet" href="/assets/ar5iv-site.0.2.2.css">
</head>
<body>
<div class="ltx_page_main">
<div class="ltx_page_content">
<article class="ltx_document ltx_authors_1line">
<h1 class="ltx_title ltx_title_document">Composable Interventions for Language Models</h1>
<div class="ltx_authors">
<span class="ltx_creator ltx_role_author">
<span class="ltx_personname">Arinbjörn Kolbeinsson<math id="id1.1.m1.1" class="ltx_Math" alttext="{}^{*\hskip 2.84526pt1}" display="inline"><semantics id="id1.1.m1.1a"><msup id="id1.1.m1.1.1" xref="id1.1.m1.1.1.cmml"><mi id="id1.1.m1.1.1a" xref="id1.1.m1.1.1.cmml"></mi><mrow id="id1.1.m1.1.1.1" xref="id1.1.m1.1.1.1.cmml"><mi id="id1.1.m1.1.1.1.2" xref="id1.1.m1.1.1.1.2.cmml"></mi><mo lspace="0.222em" rspace="0.502em" id="id1.1.m1.1.1.1.1" xref="id1.1.m1.1.1.1.1.cmml">∗</mo><mn id="id1.1.m1.1.1.1.3" xref="id1.1.m1.1.1.1.3.cmml">1</mn></mrow></msup><annotation-xml encoding="MathML-Content" id="id1.1.m1.1b"><apply id="id1.1.m1.1.1.cmml" xref="id1.1.m1.1.1"><apply id="id1.1.m1.1.1.1.cmml" xref="id1.1.m1.1.1.1"><times id="id1.1.m1.1.1.1.1.cmml" xref="id1.1.m1.1.1.1.1"></times><csymbol cd="latexml" id="id1.1.m1.1.1.1.2.cmml" xref="id1.1.m1.1.1.1.2">absent</csymbol><cn type="integer" id="id1.1.m1.1.1.1.3.cmml" xref="id1.1.m1.1.1.1.3">1</cn></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="id1.1.m1.1c">{}^{*\hskip 2.84526pt1}</annotation></semantics></math>
    
Kyle O’Brien<math id="id2.2.m2.2" class="ltx_Math" alttext="{}^{*\hskip 2.84526pt2,3}" display="inline"><semantics id="id2.2.m2.2a"><msup id="id2.2.m2.2.2" xref="id2.2.m2.2.2.cmml"><mi id="id2.2.m2.2.2a" xref="id2.2.m2.2.2.cmml"></mi><mrow id="id2.2.m2.2.2.2.2" xref="id2.2.m2.2.2.2.3.cmml"><mrow id="id2.2.m2.2.2.2.2.1" xref="id2.2.m2.2.2.2.2.1.cmml"><mi id="id2.2.m2.2.2.2.2.1.2" xref="id2.2.m2.2.2.2.2.1.2.cmml"></mi><mo lspace="0.222em" rspace="0.502em" id="id2.2.m2.2.2.2.2.1.1" xref="id2.2.m2.2.2.2.2.1.1.cmml">∗</mo><mn id="id2.2.m2.2.2.2.2.1.3" xref="id2.2.m2.2.2.2.2.1.3.cmml">2</mn></mrow><mo id="id2.2.m2.2.2.2.2.2" xref="id2.2.m2.2.2.2.3.cmml">,</mo><mn id="id2.2.m2.1.1.1.1" xref="id2.2.m2.1.1.1.1.cmml">3</mn></mrow></msup><annotation-xml encoding="MathML-Content" id="id2.2.m2.2b"><apply id="id2.2.m2.2.2.cmml" xref="id2.2.m2.2.2"><list id="id2.2.m2.2.2.2.3.cmml" xref="id2.2.m2.2.2.2.2"><apply id="id2.2.m2.2.2.2.2.1.cmml" xref="id2.2.m2.2.2.2.2.1"><times id="id2.2.m2.2.2.2.2.1.1.cmml" xref="id2.2.m2.2.2.2.2.1.1"></times><csymbol cd="latexml" id="id2.2.m2.2.2.2.2.1.2.cmml" xref="id2.2.m2.2.2.2.2.1.2">absent</csymbol><cn type="integer" id="id2.2.m2.2.2.2.2.1.3.cmml" xref="id2.2.m2.2.2.2.2.1.3">2</cn></apply><cn type="integer" id="id2.2.m2.1.1.1.1.cmml" xref="id2.2.m2.1.1.1.1">3</cn></list></apply></annotation-xml><annotation encoding="application/x-tex" id="id2.2.m2.2c">{}^{*\hskip 2.84526pt2,3}</annotation></semantics></math>
    
Tianjin Huang<math id="id3.3.m3.2" class="ltx_Math" alttext="{}^{*\hskip 2.84526pt4,5}" display="inline"><semantics id="id3.3.m3.2a"><msup id="id3.3.m3.2.2" xref="id3.3.m3.2.2.cmml"><mi id="id3.3.m3.2.2a" xref="id3.3.m3.2.2.cmml"></mi><mrow id="id3.3.m3.2.2.2.2" xref="id3.3.m3.2.2.2.3.cmml"><mrow id="id3.3.m3.2.2.2.2.1" xref="id3.3.m3.2.2.2.2.1.cmml"><mi id="id3.3.m3.2.2.2.2.1.2" xref="id3.3.m3.2.2.2.2.1.2.cmml"></mi><mo lspace="0.222em" rspace="0.502em" id="id3.3.m3.2.2.2.2.1.1" xref="id3.3.m3.2.2.2.2.1.1.cmml">∗</mo><mn id="id3.3.m3.2.2.2.2.1.3" xref="id3.3.m3.2.2.2.2.1.3.cmml">4</mn></mrow><mo id="id3.3.m3.2.2.2.2.2" xref="id3.3.m3.2.2.2.3.cmml">,</mo><mn id="id3.3.m3.1.1.1.1" xref="id3.3.m3.1.1.1.1.cmml">5</mn></mrow></msup><annotation-xml encoding="MathML-Content" id="id3.3.m3.2b"><apply id="id3.3.m3.2.2.cmml" xref="id3.3.m3.2.2"><list id="id3.3.m3.2.2.2.3.cmml" xref="id3.3.m3.2.2.2.2"><apply id="id3.3.m3.2.2.2.2.1.cmml" xref="id3.3.m3.2.2.2.2.1"><times id="id3.3.m3.2.2.2.2.1.1.cmml" xref="id3.3.m3.2.2.2.2.1.1"></times><csymbol cd="latexml" id="id3.3.m3.2.2.2.2.1.2.cmml" xref="id3.3.m3.2.2.2.2.1.2">absent</csymbol><cn type="integer" id="id3.3.m3.2.2.2.2.1.3.cmml" xref="id3.3.m3.2.2.2.2.1.3">4</cn></apply><cn type="integer" id="id3.3.m3.1.1.1.1.cmml" xref="id3.3.m3.1.1.1.1">5</cn></list></apply></annotation-xml><annotation encoding="application/x-tex" id="id3.3.m3.2c">{}^{*\hskip 2.84526pt4,5}</annotation></semantics></math>
    
Shanghua Gao<sup id="id22.22.id1" class="ltx_sup">6</sup>
<br class="ltx_break"> <span id="id5.5.1" class="ltx_text ltx_font_bold">Shiwei Liu<sup id="id5.5.1.1" class="ltx_sup"><span id="id5.5.1.1.1" class="ltx_text ltx_font_medium">7</span></sup></span>
    
<span id="id6.6.2" class="ltx_text ltx_font_bold">Jonathan Richard Schwarz<sup id="id6.6.2.1" class="ltx_sup"><span id="id6.6.2.1.1" class="ltx_text ltx_font_medium">6</span></sup></span>
    
<span id="id7.7.3" class="ltx_text ltx_font_bold">Anurag Vaidya<sup id="id7.7.3.1" class="ltx_sup"><span id="id7.7.3.1.1" class="ltx_text ltx_font_medium ltx_font_italic">6,8</span></sup>
<br class="ltx_break"></span> <span id="id8.8.4" class="ltx_text ltx_font_bold">Faisal Mahmood<sup id="id8.8.4.1" class="ltx_sup"><span id="id8.8.4.1.1" class="ltx_text ltx_font_medium ltx_font_italic">6,8</span></sup></span>
    
<span id="id9.9.5" class="ltx_text ltx_font_bold">Marinka Zitnik<sup id="id9.9.5.1" class="ltx_sup"><span id="id9.9.5.1.1" class="ltx_text ltx_font_medium">6</span></sup></span>
    
<span id="id10.10.6" class="ltx_text ltx_font_bold">Tianlong Chen<sup id="id10.10.6.1" class="ltx_sup"><span id="id10.10.6.1.1" class="ltx_text ltx_font_medium">9</span></sup></span>
    
<span id="id12.12.8" class="ltx_text ltx_font_bold">Thomas Hartvigsen<sup id="id12.12.8.1" class="ltx_sup"><span id="id12.12.8.1.1" class="ltx_text ltx_font_medium">1</span></sup>
<br class="ltx_break">
<br class="ltx_break"><sup id="id12.12.8.2" class="ltx_sup"><span id="id12.12.8.2.1" class="ltx_text ltx_font_medium">1</span></sup></span>University of Virginia
<sup id="id23.23.id2" class="ltx_sup">2</sup>EleutherAI
<sup id="id24.24.id3" class="ltx_sup">3</sup>Microsoft
<sup id="id25.25.id4" class="ltx_sup">4</sup>University of Exeter
<br class="ltx_break"><sup id="id26.26.id5" class="ltx_sup">5</sup>Eindhoven University of Technology
<sup id="id27.27.id6" class="ltx_sup">6</sup>Harvard Medical School
<sup id="id28.28.id7" class="ltx_sup">7</sup>University of Oxford
<br class="ltx_break"><sup id="id29.29.id8" class="ltx_sup">8</sup>Mass General Brigham
<sup id="id30.30.id9" class="ltx_sup">9</sup>UNC Chapel Hill
<br class="ltx_break"><sup id="id31.31.id10" class="ltx_sup">∗</sup>Equal Contribution
</span></span>
</div>

<div class="ltx_abstract">
<h6 class="ltx_title ltx_title_abstract">Abstract</h6>
<p id="id32.id1" class="ltx_p">Test-time interventions for language models can enhance factual accuracy, mitigate harmful outputs, and improve model efficiency without costly retraining.
But despite a flood of new methods, different types of interventions are largely developing independently.
In practice, multiple interventions must be applied sequentially to the same model, yet we lack standardized ways to study how interventions interact.
We fill this gap by introducing <span id="id32.id1.1" class="ltx_text ltx_font_italic">composable interventions</span>, a framework to study the effects of using multiple interventions on the same language models, featuring new metrics and a unified codebase.
Using our framework, we conduct extensive experiments and compose popular methods from three emerging intervention categories—<span id="id32.id1.2" class="ltx_text ltx_font_italic">knowledge editing</span>, <span id="id32.id1.3" class="ltx_text ltx_font_italic">model compression</span>, and <span id="id32.id1.4" class="ltx_text ltx_font_italic">machine unlearning</span>.
Our results from 310 different compositions uncover meaningful interactions: compression hinders editing and unlearning, composing interventions hinges on their order of application, and popular general-purpose metrics are inadequate for assessing composability.
Taken together, our findings showcase clear gaps in composability, suggesting a need for new multi-objective interventions.<span id="footnote1" class="ltx_note ltx_role_footnote"><sup class="ltx_note_mark">1</sup><span class="ltx_note_outer"><span class="ltx_note_content"><sup class="ltx_note_mark">1</sup><span class="ltx_tag ltx_tag_note">1</span>All of our code is public: <a target="_blank" href="https://github.com/hartvigsen-group/composable-interventions" title="" class="ltx_ref ltx_href"><span class="ltx_ref ltx_url ltx_font_typewriter">github.com/hartvigsen-group/composable-interventions</span></a></span></span></span><span id="footnote2" class="ltx_note ltx_role_footnote"><sup class="ltx_note_mark">†</sup><span class="ltx_note_outer"><span class="ltx_note_content"><sup class="ltx_note_mark">†</sup>Correspondence: <span id="footnote2.1" class="ltx_text ltx_font_typewriter">{arinbjorn,hartvigsen}@virginia.edu</span>, <span id="footnote2.2" class="ltx_text ltx_font_typewriter">kyobrien@microsoft.com</span></span></span></span></p>
</div>
<section id="S1" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">1 </span>Introduction</h2>

<div id="S1.p1" class="ltx_para">
<p id="S1.p1.1" class="ltx_p">Language models (LMs) exhibit striking capabilities on important tasks in myriad domains including medicine <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib1" title="" class="ltx_ref">1</a>]</cite>, finance <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib2" title="" class="ltx_ref">2</a>]</cite>, science <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib3" title="" class="ltx_ref">3</a>]</cite>, and entertainment <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib4" title="" class="ltx_ref">4</a>]</cite>. But despite high performance, deployed LMs can still misbehave unpredictably and require updates.
For example, LMs generate content that is <span id="S1.p1.1.1" class="ltx_text ltx_font_italic">hallucinatory</span> <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib5" title="" class="ltx_ref">5</a>]</cite>, <span id="S1.p1.1.2" class="ltx_text ltx_font_italic">factually incorrect</span> <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib6" title="" class="ltx_ref">6</a>]</cite>, and <span id="S1.p1.1.3" class="ltx_text ltx_font_italic">harmful</span> <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib7" title="" class="ltx_ref">7</a>, <a href="#bib.bib8" title="" class="ltx_ref">8</a>, <a href="#bib.bib9" title="" class="ltx_ref">9</a>]</cite>.
Beyond unwanted behaviors, user requirements also change over time.
For example, regulations arise <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib10" title="" class="ltx_ref">10</a>]</cite>, computational resources constrict, knowledge gets outdated <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib11" title="" class="ltx_ref">11</a>]</cite>, and copyrighted training materials are identified <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib12" title="" class="ltx_ref">12</a>]</cite>.
Without ways to quickly address these issues, models can be left miscalibrated, outdated, and biased, limiting their widespread responsible use <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib13" title="" class="ltx_ref">13</a>]</cite>.</p>
</div>
<div id="S1.p2" class="ltx_para">
<p id="S1.p2.1" class="ltx_p">Many recent works study efficient, in-place updates for LMs.
We broadly refer to these as <span id="S1.p2.1.1" class="ltx_text ltx_font_italic">interventions</span>—updates to targeted properties of LMs applied after pretraining (and optional fine-tuning).
For example, we can view model compression <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib14" title="" class="ltx_ref">14</a>, <a href="#bib.bib15" title="" class="ltx_ref">15</a>, <a href="#bib.bib16" title="" class="ltx_ref">16</a>]</cite> as an intervention to make language models more inference- or memory-efficient.
Other rapidly-advancing examples include knowledge editing <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib17" title="" class="ltx_ref">17</a>, <a href="#bib.bib18" title="" class="ltx_ref">18</a>, <a href="#bib.bib19" title="" class="ltx_ref">19</a>, <a href="#bib.bib20" title="" class="ltx_ref">20</a>, <a href="#bib.bib21" title="" class="ltx_ref">21</a>, <a href="#bib.bib22" title="" class="ltx_ref">22</a>]</cite>, detoxification <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib23" title="" class="ltx_ref">23</a>, <a href="#bib.bib24" title="" class="ltx_ref">24</a>, <a href="#bib.bib25" title="" class="ltx_ref">25</a>, <a href="#bib.bib26" title="" class="ltx_ref">26</a>]</cite>, and unlearning <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib27" title="" class="ltx_ref">27</a>, <a href="#bib.bib28" title="" class="ltx_ref">28</a>, <a href="#bib.bib29" title="" class="ltx_ref">29</a>]</cite>.
However, these interventions are largely advancing independently.
In practice, we usually have <span id="S1.p2.1.2" class="ltx_text ltx_font_italic">multiple</span> requirements for properties of our models (<span id="S1.p2.1.3" class="ltx_text ltx_font_italic">e.g.,</span> factuality <span id="S1.p2.1.4" class="ltx_text ltx_font_italic">and</span> efficiency).
And as requirements change over time, new interventions must be applied.
While some works have started studying interactions between training objectives <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib30" title="" class="ltx_ref">30</a>, <a href="#bib.bib31" title="" class="ltx_ref">31</a>, <a href="#bib.bib32" title="" class="ltx_ref">32</a>]</cite>, practical widespread use is limited without unified evaluations for how interventions interact.</p>
</div>
<div id="S1.p3" class="ltx_para">
<p id="S1.p3.1" class="ltx_p">As depicted in Figure <a href="#S1.F1" title="Figure 1 ‣ 1 Introduction ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a>, we propose that practical interventions should be <span id="S1.p3.1.1" class="ltx_text ltx_font_italic">composable</span>: When an intervention is applied to a model, it should not interfere with prior or future interventions.
For example, we might repair a factual error in an LM using a knowledge editing method.
Later on, we might quantize that model.
A <span id="S1.p3.1.2" class="ltx_text ltx_font_italic">composable</span> quantization method should specifically preserve the edited fact and a composable editor should withstand quantization.
However, it remains unknown how well existing interventions compose, and we lack formal notions of intervention composition.
We therefore propose two metrics for composability: 1) <span id="S1.p3.1.3" class="ltx_text ltx_font_italic">Order-free Error</span>, where an intervention is composable if its application leaves others’ success unimpacted, and 2) <span id="S1.p3.1.4" class="ltx_text ltx_font_italic">Order Sensitivity</span>, where the combined success of multiple interventions should not depend on the order in which they are applied.
We instantiate these metrics in a codebase that includes popular interventions, tackling a major lack of standardized code that has inhibited cross-intervention evaluations until now.</p>
</div>
<figure id="S1.F1" class="ltx_figure"><img src="/html/2407.06483/assets/x1.png" id="S1.F1.g1" class="ltx_graphics ltx_centering ltx_img_landscape" width="438" height="182" alt="Refer to caption">
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure"><span id="S1.F1.3.1.1" class="ltx_text" style="font-size:90%;">Figure 1</span>: </span><span id="S1.F1.4.2" class="ltx_text" style="font-size:90%;">Interventions aim to update targeted properties of language models without impacting unrelated behaviors or adding excessive compute. We introduce and extensively experiment with <span id="S1.F1.4.2.1" class="ltx_text ltx_font_italic">composability</span> of different interventions used on the same model.</span></figcaption>
</figure>
<div id="S1.p4" class="ltx_para">
<p id="S1.p4.3" class="ltx_p">We use our framework to extensively study composability of state-of-the-art <span id="S1.p4.3.1" class="ltx_text ltx_font_italic">knowledge editing</span>, <span id="S1.p4.3.2" class="ltx_text ltx_font_italic">model compression</span>, and <span id="S1.p4.3.3" class="ltx_text ltx_font_italic">machine unlearning</span> interventions on Llama3-8B <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib33" title="" class="ltx_ref">33</a>]</cite>.
Our experiments with 310 different compositions unearth novel insights and offer guidance for composing recent methods.
Three key results are as follows:
 <svg id="S1.p4.1.pic1" class="ltx_picture" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S1.p4.1.pic1.1.1.1.1.1" class="ltx_text">1</span></foreignObject></g></g></svg> <span id="S1.p4.3.4" class="ltx_text ltx_font_bold">Model compression often limits the success of other interventions</span>.
This suggests a significant drawback of existing compression methods, which are nearly universal in LM deployments.
 <svg id="S1.p4.2.pic2" class="ltx_picture" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S1.p4.2.pic2.1.1.1.1.1" class="ltx_text">2</span></foreignObject></g></g></svg> <span id="S1.p4.3.5" class="ltx_text ltx_font_bold">The order in which interventions are applied dramatically alters their success</span>.
Therefore, we need new interventions explicitly designed for composability.
 <svg id="S1.p4.3.pic3" class="ltx_picture" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S1.p4.3.pic3.1.1.1.1.1" class="ltx_text">3</span></foreignObject></g></g></svg> <span id="S1.p4.3.6" class="ltx_text ltx_font_bold">General-purpose post-intervention model performance is a poor proxy for composability</span>.
This indicates that targeting composability as a metric has the potential to drive the development of new, practically-grounded interventions.
Taken alongside our other findings (Section <a href="#S4" title="4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4</span></a>), our experiments suggest we sorely need to broaden intervention method evaluations and to design new methods explicitly for composability.</p>
</div>
<div id="S1.p5" class="ltx_para">
<p id="S1.p5.1" class="ltx_p">Our key contributions are as follows:</p>
<ul id="S1.I1" class="ltx_itemize">
<li id="S1.I1.i1" class="ltx_item" style="list-style-type:none;">
<span class="ltx_tag ltx_tag_item">•</span> 
<div id="S1.I1.i1.p1" class="ltx_para">
<p id="S1.I1.i1.p1.1" class="ltx_p">We introduce the notion of <span id="S1.I1.i1.p1.1.1" class="ltx_text ltx_font_italic">composability</span> to language model interventions, formalizing the sequential applications of multiple interventions to LMs. Our work opens doors to address crucial, practical challenges in online LM updates and broadens evaluation criteria for interventions.</p>
</div>
</li>
<li id="S1.I1.i2" class="ltx_item" style="list-style-type:none;">
<span class="ltx_tag ltx_tag_item">•</span> 
<div id="S1.I1.i2.p1" class="ltx_para">
<p id="S1.I1.i2.p1.1" class="ltx_p">Our main contribution is extensive experimentation, identifying unknown interactions between knowledge editing, model compression, and machine unlearning.</p>
</div>
</li>
<li id="S1.I1.i3" class="ltx_item" style="list-style-type:none;">
<span class="ltx_tag ltx_tag_item">•</span> 
<div id="S1.I1.i3.p1" class="ltx_para">
<p id="S1.I1.i3.p1.1" class="ltx_p">Our findings suggest a clear need to develop novel interventions that target composability, a crucial property of practical interventions.</p>
</div>
</li>
<li id="S1.I1.i4" class="ltx_item" style="list-style-type:none;">
<span class="ltx_tag ltx_tag_item">•</span> 
<div id="S1.I1.i4.p1" class="ltx_para">
<p id="S1.I1.i4.p1.1" class="ltx_p">We release an extendable codebase that unifies many state-of-the-art implementations to enable others to develop new multi-objective interventions.</p>
</div>
</li>
</ul>
</div>
</section>
<section id="S2" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">2 </span>An Evaluation Framework for Composable Interventions</h2>

<section id="S2.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">2.1 </span>Preliminaries: Interventions for Language Models</h3>

<div id="S2.SS1.p1" class="ltx_para">
<p id="S2.SS1.p1.15" class="ltx_p">Assume we are given a language model <math id="S2.SS1.p1.1.m1.1" class="ltx_Math" alttext="f_{\theta}" display="inline"><semantics id="S2.SS1.p1.1.m1.1a"><msub id="S2.SS1.p1.1.m1.1.1" xref="S2.SS1.p1.1.m1.1.1.cmml"><mi id="S2.SS1.p1.1.m1.1.1.2" xref="S2.SS1.p1.1.m1.1.1.2.cmml">f</mi><mi id="S2.SS1.p1.1.m1.1.1.3" xref="S2.SS1.p1.1.m1.1.1.3.cmml">θ</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.1.m1.1b"><apply id="S2.SS1.p1.1.m1.1.1.cmml" xref="S2.SS1.p1.1.m1.1.1"><csymbol cd="ambiguous" id="S2.SS1.p1.1.m1.1.1.1.cmml" xref="S2.SS1.p1.1.m1.1.1">subscript</csymbol><ci id="S2.SS1.p1.1.m1.1.1.2.cmml" xref="S2.SS1.p1.1.m1.1.1.2">𝑓</ci><ci id="S2.SS1.p1.1.m1.1.1.3.cmml" xref="S2.SS1.p1.1.m1.1.1.3">𝜃</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.1.m1.1c">f_{\theta}</annotation></semantics></math>, where <math id="S2.SS1.p1.2.m2.1" class="ltx_Math" alttext="\theta\in\Theta" display="inline"><semantics id="S2.SS1.p1.2.m2.1a"><mrow id="S2.SS1.p1.2.m2.1.1" xref="S2.SS1.p1.2.m2.1.1.cmml"><mi id="S2.SS1.p1.2.m2.1.1.2" xref="S2.SS1.p1.2.m2.1.1.2.cmml">θ</mi><mo id="S2.SS1.p1.2.m2.1.1.1" xref="S2.SS1.p1.2.m2.1.1.1.cmml">∈</mo><mi mathvariant="normal" id="S2.SS1.p1.2.m2.1.1.3" xref="S2.SS1.p1.2.m2.1.1.3.cmml">Θ</mi></mrow><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.2.m2.1b"><apply id="S2.SS1.p1.2.m2.1.1.cmml" xref="S2.SS1.p1.2.m2.1.1"><in id="S2.SS1.p1.2.m2.1.1.1.cmml" xref="S2.SS1.p1.2.m2.1.1.1"></in><ci id="S2.SS1.p1.2.m2.1.1.2.cmml" xref="S2.SS1.p1.2.m2.1.1.2">𝜃</ci><ci id="S2.SS1.p1.2.m2.1.1.3.cmml" xref="S2.SS1.p1.2.m2.1.1.3">Θ</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.2.m2.1c">\theta\in\Theta</annotation></semantics></math> contains its pre-trained parameters and <math id="S2.SS1.p1.3.m3.1" class="ltx_Math" alttext="\Theta" display="inline"><semantics id="S2.SS1.p1.3.m3.1a"><mi mathvariant="normal" id="S2.SS1.p1.3.m3.1.1" xref="S2.SS1.p1.3.m3.1.1.cmml">Θ</mi><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.3.m3.1b"><ci id="S2.SS1.p1.3.m3.1.1.cmml" xref="S2.SS1.p1.3.m3.1.1">Θ</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.3.m3.1c">\Theta</annotation></semantics></math> is the parameter space.
We define an <span id="S2.SS1.p1.15.1" class="ltx_text ltx_font_italic">intervention</span> <math id="S2.SS1.p1.4.m4.1" class="ltx_Math" alttext="\phi" display="inline"><semantics id="S2.SS1.p1.4.m4.1a"><mi id="S2.SS1.p1.4.m4.1.1" xref="S2.SS1.p1.4.m4.1.1.cmml">ϕ</mi><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.4.m4.1b"><ci id="S2.SS1.p1.4.m4.1.1.cmml" xref="S2.SS1.p1.4.m4.1.1">italic-ϕ</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.4.m4.1c">\phi</annotation></semantics></math> to be the combination of four things:
1) an operator <math id="S2.SS1.p1.5.m5.2" class="ltx_Math" alttext="\omega(f_{\theta},\gamma)" display="inline"><semantics id="S2.SS1.p1.5.m5.2a"><mrow id="S2.SS1.p1.5.m5.2.2" xref="S2.SS1.p1.5.m5.2.2.cmml"><mi id="S2.SS1.p1.5.m5.2.2.3" xref="S2.SS1.p1.5.m5.2.2.3.cmml">ω</mi><mo lspace="0em" rspace="0em" id="S2.SS1.p1.5.m5.2.2.2" xref="S2.SS1.p1.5.m5.2.2.2.cmml">​</mo><mrow id="S2.SS1.p1.5.m5.2.2.1.1" xref="S2.SS1.p1.5.m5.2.2.1.2.cmml"><mo stretchy="false" id="S2.SS1.p1.5.m5.2.2.1.1.2" xref="S2.SS1.p1.5.m5.2.2.1.2.cmml">(</mo><msub id="S2.SS1.p1.5.m5.2.2.1.1.1" xref="S2.SS1.p1.5.m5.2.2.1.1.1.cmml"><mi id="S2.SS1.p1.5.m5.2.2.1.1.1.2" xref="S2.SS1.p1.5.m5.2.2.1.1.1.2.cmml">f</mi><mi id="S2.SS1.p1.5.m5.2.2.1.1.1.3" xref="S2.SS1.p1.5.m5.2.2.1.1.1.3.cmml">θ</mi></msub><mo id="S2.SS1.p1.5.m5.2.2.1.1.3" xref="S2.SS1.p1.5.m5.2.2.1.2.cmml">,</mo><mi id="S2.SS1.p1.5.m5.1.1" xref="S2.SS1.p1.5.m5.1.1.cmml">γ</mi><mo stretchy="false" id="S2.SS1.p1.5.m5.2.2.1.1.4" xref="S2.SS1.p1.5.m5.2.2.1.2.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.5.m5.2b"><apply id="S2.SS1.p1.5.m5.2.2.cmml" xref="S2.SS1.p1.5.m5.2.2"><times id="S2.SS1.p1.5.m5.2.2.2.cmml" xref="S2.SS1.p1.5.m5.2.2.2"></times><ci id="S2.SS1.p1.5.m5.2.2.3.cmml" xref="S2.SS1.p1.5.m5.2.2.3">𝜔</ci><interval closure="open" id="S2.SS1.p1.5.m5.2.2.1.2.cmml" xref="S2.SS1.p1.5.m5.2.2.1.1"><apply id="S2.SS1.p1.5.m5.2.2.1.1.1.cmml" xref="S2.SS1.p1.5.m5.2.2.1.1.1"><csymbol cd="ambiguous" id="S2.SS1.p1.5.m5.2.2.1.1.1.1.cmml" xref="S2.SS1.p1.5.m5.2.2.1.1.1">subscript</csymbol><ci id="S2.SS1.p1.5.m5.2.2.1.1.1.2.cmml" xref="S2.SS1.p1.5.m5.2.2.1.1.1.2">𝑓</ci><ci id="S2.SS1.p1.5.m5.2.2.1.1.1.3.cmml" xref="S2.SS1.p1.5.m5.2.2.1.1.1.3">𝜃</ci></apply><ci id="S2.SS1.p1.5.m5.1.1.cmml" xref="S2.SS1.p1.5.m5.1.1">𝛾</ci></interval></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.5.m5.2c">\omega(f_{\theta},\gamma)</annotation></semantics></math> that produces a new model <math id="S2.SS1.p1.6.m6.1" class="ltx_Math" alttext="f_{\theta}" display="inline"><semantics id="S2.SS1.p1.6.m6.1a"><msub id="S2.SS1.p1.6.m6.1.1" xref="S2.SS1.p1.6.m6.1.1.cmml"><mi id="S2.SS1.p1.6.m6.1.1.2" xref="S2.SS1.p1.6.m6.1.1.2.cmml">f</mi><mi id="S2.SS1.p1.6.m6.1.1.3" xref="S2.SS1.p1.6.m6.1.1.3.cmml">θ</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.6.m6.1b"><apply id="S2.SS1.p1.6.m6.1.1.cmml" xref="S2.SS1.p1.6.m6.1.1"><csymbol cd="ambiguous" id="S2.SS1.p1.6.m6.1.1.1.cmml" xref="S2.SS1.p1.6.m6.1.1">subscript</csymbol><ci id="S2.SS1.p1.6.m6.1.1.2.cmml" xref="S2.SS1.p1.6.m6.1.1.2">𝑓</ci><ci id="S2.SS1.p1.6.m6.1.1.3.cmml" xref="S2.SS1.p1.6.m6.1.1.3">𝜃</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.6.m6.1c">f_{\theta}</annotation></semantics></math>,
2) hyperparameters <math id="S2.SS1.p1.7.m7.1" class="ltx_Math" alttext="\gamma" display="inline"><semantics id="S2.SS1.p1.7.m7.1a"><mi id="S2.SS1.p1.7.m7.1.1" xref="S2.SS1.p1.7.m7.1.1.cmml">γ</mi><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.7.m7.1b"><ci id="S2.SS1.p1.7.m7.1.1.cmml" xref="S2.SS1.p1.7.m7.1.1">𝛾</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.7.m7.1c">\gamma</annotation></semantics></math>, which typically control the strength of intervention,
3) a loss <math id="S2.SS1.p1.8.m8.2" class="ltx_Math" alttext="\ell(\omega(f_{\theta},\gamma))" display="inline"><semantics id="S2.SS1.p1.8.m8.2a"><mrow id="S2.SS1.p1.8.m8.2.2" xref="S2.SS1.p1.8.m8.2.2.cmml"><mi mathvariant="normal" id="S2.SS1.p1.8.m8.2.2.3" xref="S2.SS1.p1.8.m8.2.2.3.cmml">ℓ</mi><mo lspace="0em" rspace="0em" id="S2.SS1.p1.8.m8.2.2.2" xref="S2.SS1.p1.8.m8.2.2.2.cmml">​</mo><mrow id="S2.SS1.p1.8.m8.2.2.1.1" xref="S2.SS1.p1.8.m8.2.2.1.1.1.cmml"><mo stretchy="false" id="S2.SS1.p1.8.m8.2.2.1.1.2" xref="S2.SS1.p1.8.m8.2.2.1.1.1.cmml">(</mo><mrow id="S2.SS1.p1.8.m8.2.2.1.1.1" xref="S2.SS1.p1.8.m8.2.2.1.1.1.cmml"><mi id="S2.SS1.p1.8.m8.2.2.1.1.1.3" xref="S2.SS1.p1.8.m8.2.2.1.1.1.3.cmml">ω</mi><mo lspace="0em" rspace="0em" id="S2.SS1.p1.8.m8.2.2.1.1.1.2" xref="S2.SS1.p1.8.m8.2.2.1.1.1.2.cmml">​</mo><mrow id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.2.cmml"><mo stretchy="false" id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.2" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.2.cmml">(</mo><msub id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.cmml"><mi id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.2" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.3" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.3.cmml">θ</mi></msub><mo id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.3" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.2.cmml">,</mo><mi id="S2.SS1.p1.8.m8.1.1" xref="S2.SS1.p1.8.m8.1.1.cmml">γ</mi><mo stretchy="false" id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.4" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.2.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.SS1.p1.8.m8.2.2.1.1.3" xref="S2.SS1.p1.8.m8.2.2.1.1.1.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.8.m8.2b"><apply id="S2.SS1.p1.8.m8.2.2.cmml" xref="S2.SS1.p1.8.m8.2.2"><times id="S2.SS1.p1.8.m8.2.2.2.cmml" xref="S2.SS1.p1.8.m8.2.2.2"></times><ci id="S2.SS1.p1.8.m8.2.2.3.cmml" xref="S2.SS1.p1.8.m8.2.2.3">ℓ</ci><apply id="S2.SS1.p1.8.m8.2.2.1.1.1.cmml" xref="S2.SS1.p1.8.m8.2.2.1.1"><times id="S2.SS1.p1.8.m8.2.2.1.1.1.2.cmml" xref="S2.SS1.p1.8.m8.2.2.1.1.1.2"></times><ci id="S2.SS1.p1.8.m8.2.2.1.1.1.3.cmml" xref="S2.SS1.p1.8.m8.2.2.1.1.1.3">𝜔</ci><interval closure="open" id="S2.SS1.p1.8.m8.2.2.1.1.1.1.2.cmml" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.1"><apply id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.cmml" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.1.cmml" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1">subscript</csymbol><ci id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.2.cmml" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.2">𝑓</ci><ci id="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.3.cmml" xref="S2.SS1.p1.8.m8.2.2.1.1.1.1.1.1.3">𝜃</ci></apply><ci id="S2.SS1.p1.8.m8.1.1.cmml" xref="S2.SS1.p1.8.m8.1.1">𝛾</ci></interval></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.8.m8.2c">\ell(\omega(f_{\theta},\gamma))</annotation></semantics></math> that specifies the targeted update,<span id="footnote2a" class="ltx_note ltx_role_footnote"><sup class="ltx_note_mark">2</sup><span class="ltx_note_outer"><span class="ltx_note_content"><sup class="ltx_note_mark">2</sup><span class="ltx_tag ltx_tag_note">2</span>Some interventions require data, some do not. We remove direct data dependencies for simplicity.</span></span></span>
and 4) evaluation criteria <math id="S2.SS1.p1.9.m9.4" class="ltx_Math" alttext="\kappa(f_{\theta},\mathcal{D})\in[0,1]" display="inline"><semantics id="S2.SS1.p1.9.m9.4a"><mrow id="S2.SS1.p1.9.m9.4.4" xref="S2.SS1.p1.9.m9.4.4.cmml"><mrow id="S2.SS1.p1.9.m9.4.4.1" xref="S2.SS1.p1.9.m9.4.4.1.cmml"><mi id="S2.SS1.p1.9.m9.4.4.1.3" xref="S2.SS1.p1.9.m9.4.4.1.3.cmml">κ</mi><mo lspace="0em" rspace="0em" id="S2.SS1.p1.9.m9.4.4.1.2" xref="S2.SS1.p1.9.m9.4.4.1.2.cmml">​</mo><mrow id="S2.SS1.p1.9.m9.4.4.1.1.1" xref="S2.SS1.p1.9.m9.4.4.1.1.2.cmml"><mo stretchy="false" id="S2.SS1.p1.9.m9.4.4.1.1.1.2" xref="S2.SS1.p1.9.m9.4.4.1.1.2.cmml">(</mo><msub id="S2.SS1.p1.9.m9.4.4.1.1.1.1" xref="S2.SS1.p1.9.m9.4.4.1.1.1.1.cmml"><mi id="S2.SS1.p1.9.m9.4.4.1.1.1.1.2" xref="S2.SS1.p1.9.m9.4.4.1.1.1.1.2.cmml">f</mi><mi id="S2.SS1.p1.9.m9.4.4.1.1.1.1.3" xref="S2.SS1.p1.9.m9.4.4.1.1.1.1.3.cmml">θ</mi></msub><mo id="S2.SS1.p1.9.m9.4.4.1.1.1.3" xref="S2.SS1.p1.9.m9.4.4.1.1.2.cmml">,</mo><mi class="ltx_font_mathcaligraphic" id="S2.SS1.p1.9.m9.1.1" xref="S2.SS1.p1.9.m9.1.1.cmml">𝒟</mi><mo stretchy="false" id="S2.SS1.p1.9.m9.4.4.1.1.1.4" xref="S2.SS1.p1.9.m9.4.4.1.1.2.cmml">)</mo></mrow></mrow><mo id="S2.SS1.p1.9.m9.4.4.2" xref="S2.SS1.p1.9.m9.4.4.2.cmml">∈</mo><mrow id="S2.SS1.p1.9.m9.4.4.3.2" xref="S2.SS1.p1.9.m9.4.4.3.1.cmml"><mo stretchy="false" id="S2.SS1.p1.9.m9.4.4.3.2.1" xref="S2.SS1.p1.9.m9.4.4.3.1.cmml">[</mo><mn id="S2.SS1.p1.9.m9.2.2" xref="S2.SS1.p1.9.m9.2.2.cmml">0</mn><mo id="S2.SS1.p1.9.m9.4.4.3.2.2" xref="S2.SS1.p1.9.m9.4.4.3.1.cmml">,</mo><mn id="S2.SS1.p1.9.m9.3.3" xref="S2.SS1.p1.9.m9.3.3.cmml">1</mn><mo stretchy="false" id="S2.SS1.p1.9.m9.4.4.3.2.3" xref="S2.SS1.p1.9.m9.4.4.3.1.cmml">]</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.9.m9.4b"><apply id="S2.SS1.p1.9.m9.4.4.cmml" xref="S2.SS1.p1.9.m9.4.4"><in id="S2.SS1.p1.9.m9.4.4.2.cmml" xref="S2.SS1.p1.9.m9.4.4.2"></in><apply id="S2.SS1.p1.9.m9.4.4.1.cmml" xref="S2.SS1.p1.9.m9.4.4.1"><times id="S2.SS1.p1.9.m9.4.4.1.2.cmml" xref="S2.SS1.p1.9.m9.4.4.1.2"></times><ci id="S2.SS1.p1.9.m9.4.4.1.3.cmml" xref="S2.SS1.p1.9.m9.4.4.1.3">𝜅</ci><interval closure="open" id="S2.SS1.p1.9.m9.4.4.1.1.2.cmml" xref="S2.SS1.p1.9.m9.4.4.1.1.1"><apply id="S2.SS1.p1.9.m9.4.4.1.1.1.1.cmml" xref="S2.SS1.p1.9.m9.4.4.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS1.p1.9.m9.4.4.1.1.1.1.1.cmml" xref="S2.SS1.p1.9.m9.4.4.1.1.1.1">subscript</csymbol><ci id="S2.SS1.p1.9.m9.4.4.1.1.1.1.2.cmml" xref="S2.SS1.p1.9.m9.4.4.1.1.1.1.2">𝑓</ci><ci id="S2.SS1.p1.9.m9.4.4.1.1.1.1.3.cmml" xref="S2.SS1.p1.9.m9.4.4.1.1.1.1.3">𝜃</ci></apply><ci id="S2.SS1.p1.9.m9.1.1.cmml" xref="S2.SS1.p1.9.m9.1.1">𝒟</ci></interval></apply><interval closure="closed" id="S2.SS1.p1.9.m9.4.4.3.1.cmml" xref="S2.SS1.p1.9.m9.4.4.3.2"><cn type="integer" id="S2.SS1.p1.9.m9.2.2.cmml" xref="S2.SS1.p1.9.m9.2.2">0</cn><cn type="integer" id="S2.SS1.p1.9.m9.3.3.cmml" xref="S2.SS1.p1.9.m9.3.3">1</cn></interval></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.9.m9.4c">\kappa(f_{\theta},\mathcal{D})\in[0,1]</annotation></semantics></math>, where higher scores indicate better performance.
Each intervention’s operator <math id="S2.SS1.p1.10.m10.1" class="ltx_Math" alttext="\omega" display="inline"><semantics id="S2.SS1.p1.10.m10.1a"><mi id="S2.SS1.p1.10.m10.1.1" xref="S2.SS1.p1.10.m10.1.1.cmml">ω</mi><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.10.m10.1b"><ci id="S2.SS1.p1.10.m10.1.1.cmml" xref="S2.SS1.p1.10.m10.1.1">𝜔</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.10.m10.1c">\omega</annotation></semantics></math> aims to minimize loss <math id="S2.SS1.p1.11.m11.1" class="ltx_Math" alttext="\ell" display="inline"><semantics id="S2.SS1.p1.11.m11.1a"><mi mathvariant="normal" id="S2.SS1.p1.11.m11.1.1" xref="S2.SS1.p1.11.m11.1.1.cmml">ℓ</mi><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.11.m11.1b"><ci id="S2.SS1.p1.11.m11.1.1.cmml" xref="S2.SS1.p1.11.m11.1.1">ℓ</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.11.m11.1c">\ell</annotation></semantics></math> and ultimately achieve high performance according to <math id="S2.SS1.p1.12.m12.1" class="ltx_Math" alttext="\kappa" display="inline"><semantics id="S2.SS1.p1.12.m12.1a"><mi id="S2.SS1.p1.12.m12.1.1" xref="S2.SS1.p1.12.m12.1.1.cmml">κ</mi><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.12.m12.1b"><ci id="S2.SS1.p1.12.m12.1.1.cmml" xref="S2.SS1.p1.12.m12.1.1">𝜅</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.12.m12.1c">\kappa</annotation></semantics></math>.
For readability, we denote the application of intervention <math id="S2.SS1.p1.13.m13.1" class="ltx_Math" alttext="\phi" display="inline"><semantics id="S2.SS1.p1.13.m13.1a"><mi id="S2.SS1.p1.13.m13.1.1" xref="S2.SS1.p1.13.m13.1.1.cmml">ϕ</mi><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.13.m13.1b"><ci id="S2.SS1.p1.13.m13.1.1.cmml" xref="S2.SS1.p1.13.m13.1.1">italic-ϕ</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.13.m13.1c">\phi</annotation></semantics></math> to model <math id="S2.SS1.p1.14.m14.1" class="ltx_Math" alttext="f_{\theta}" display="inline"><semantics id="S2.SS1.p1.14.m14.1a"><msub id="S2.SS1.p1.14.m14.1.1" xref="S2.SS1.p1.14.m14.1.1.cmml"><mi id="S2.SS1.p1.14.m14.1.1.2" xref="S2.SS1.p1.14.m14.1.1.2.cmml">f</mi><mi id="S2.SS1.p1.14.m14.1.1.3" xref="S2.SS1.p1.14.m14.1.1.3.cmml">θ</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.14.m14.1b"><apply id="S2.SS1.p1.14.m14.1.1.cmml" xref="S2.SS1.p1.14.m14.1.1"><csymbol cd="ambiguous" id="S2.SS1.p1.14.m14.1.1.1.cmml" xref="S2.SS1.p1.14.m14.1.1">subscript</csymbol><ci id="S2.SS1.p1.14.m14.1.1.2.cmml" xref="S2.SS1.p1.14.m14.1.1.2">𝑓</ci><ci id="S2.SS1.p1.14.m14.1.1.3.cmml" xref="S2.SS1.p1.14.m14.1.1.3">𝜃</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.14.m14.1c">f_{\theta}</annotation></semantics></math> as <math id="S2.SS1.p1.15.m15.1" class="ltx_Math" alttext="\phi(f)" display="inline"><semantics id="S2.SS1.p1.15.m15.1a"><mrow id="S2.SS1.p1.15.m15.1.2" xref="S2.SS1.p1.15.m15.1.2.cmml"><mi id="S2.SS1.p1.15.m15.1.2.2" xref="S2.SS1.p1.15.m15.1.2.2.cmml">ϕ</mi><mo lspace="0em" rspace="0em" id="S2.SS1.p1.15.m15.1.2.1" xref="S2.SS1.p1.15.m15.1.2.1.cmml">​</mo><mrow id="S2.SS1.p1.15.m15.1.2.3.2" xref="S2.SS1.p1.15.m15.1.2.cmml"><mo stretchy="false" id="S2.SS1.p1.15.m15.1.2.3.2.1" xref="S2.SS1.p1.15.m15.1.2.cmml">(</mo><mi id="S2.SS1.p1.15.m15.1.1" xref="S2.SS1.p1.15.m15.1.1.cmml">f</mi><mo stretchy="false" id="S2.SS1.p1.15.m15.1.2.3.2.2" xref="S2.SS1.p1.15.m15.1.2.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS1.p1.15.m15.1b"><apply id="S2.SS1.p1.15.m15.1.2.cmml" xref="S2.SS1.p1.15.m15.1.2"><times id="S2.SS1.p1.15.m15.1.2.1.cmml" xref="S2.SS1.p1.15.m15.1.2.1"></times><ci id="S2.SS1.p1.15.m15.1.2.2.cmml" xref="S2.SS1.p1.15.m15.1.2.2">italic-ϕ</ci><ci id="S2.SS1.p1.15.m15.1.1.cmml" xref="S2.SS1.p1.15.m15.1.1">𝑓</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS1.p1.15.m15.1c">\phi(f)</annotation></semantics></math>.</p>
</div>
</section>
<section id="S2.SS2" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">2.2 </span>Composable Interventions for Language Models</h3>

<div id="S2.SS2.p1" class="ltx_para">
<p id="S2.SS2.p1.8" class="ltx_p">Now consider <math id="S2.SS2.p1.1.m1.1" class="ltx_Math" alttext="N" display="inline"><semantics id="S2.SS2.p1.1.m1.1a"><mi id="S2.SS2.p1.1.m1.1.1" xref="S2.SS2.p1.1.m1.1.1.cmml">N</mi><annotation-xml encoding="MathML-Content" id="S2.SS2.p1.1.m1.1b"><ci id="S2.SS2.p1.1.m1.1.1.cmml" xref="S2.SS2.p1.1.m1.1.1">𝑁</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p1.1.m1.1c">N</annotation></semantics></math> possible interventions <math id="S2.SS2.p1.2.m2.1" class="ltx_Math" alttext="\{\phi_{i}\}_{i=0}^{N-1}" display="inline"><semantics id="S2.SS2.p1.2.m2.1a"><msubsup id="S2.SS2.p1.2.m2.1.1" xref="S2.SS2.p1.2.m2.1.1.cmml"><mrow id="S2.SS2.p1.2.m2.1.1.1.1.1" xref="S2.SS2.p1.2.m2.1.1.1.1.2.cmml"><mo stretchy="false" id="S2.SS2.p1.2.m2.1.1.1.1.1.2" xref="S2.SS2.p1.2.m2.1.1.1.1.2.cmml">{</mo><msub id="S2.SS2.p1.2.m2.1.1.1.1.1.1" xref="S2.SS2.p1.2.m2.1.1.1.1.1.1.cmml"><mi id="S2.SS2.p1.2.m2.1.1.1.1.1.1.2" xref="S2.SS2.p1.2.m2.1.1.1.1.1.1.2.cmml">ϕ</mi><mi id="S2.SS2.p1.2.m2.1.1.1.1.1.1.3" xref="S2.SS2.p1.2.m2.1.1.1.1.1.1.3.cmml">i</mi></msub><mo stretchy="false" id="S2.SS2.p1.2.m2.1.1.1.1.1.3" xref="S2.SS2.p1.2.m2.1.1.1.1.2.cmml">}</mo></mrow><mrow id="S2.SS2.p1.2.m2.1.1.1.3" xref="S2.SS2.p1.2.m2.1.1.1.3.cmml"><mi id="S2.SS2.p1.2.m2.1.1.1.3.2" xref="S2.SS2.p1.2.m2.1.1.1.3.2.cmml">i</mi><mo id="S2.SS2.p1.2.m2.1.1.1.3.1" xref="S2.SS2.p1.2.m2.1.1.1.3.1.cmml">=</mo><mn id="S2.SS2.p1.2.m2.1.1.1.3.3" xref="S2.SS2.p1.2.m2.1.1.1.3.3.cmml">0</mn></mrow><mrow id="S2.SS2.p1.2.m2.1.1.3" xref="S2.SS2.p1.2.m2.1.1.3.cmml"><mi id="S2.SS2.p1.2.m2.1.1.3.2" xref="S2.SS2.p1.2.m2.1.1.3.2.cmml">N</mi><mo id="S2.SS2.p1.2.m2.1.1.3.1" xref="S2.SS2.p1.2.m2.1.1.3.1.cmml">−</mo><mn id="S2.SS2.p1.2.m2.1.1.3.3" xref="S2.SS2.p1.2.m2.1.1.3.3.cmml">1</mn></mrow></msubsup><annotation-xml encoding="MathML-Content" id="S2.SS2.p1.2.m2.1b"><apply id="S2.SS2.p1.2.m2.1.1.cmml" xref="S2.SS2.p1.2.m2.1.1"><csymbol cd="ambiguous" id="S2.SS2.p1.2.m2.1.1.2.cmml" xref="S2.SS2.p1.2.m2.1.1">superscript</csymbol><apply id="S2.SS2.p1.2.m2.1.1.1.cmml" xref="S2.SS2.p1.2.m2.1.1"><csymbol cd="ambiguous" id="S2.SS2.p1.2.m2.1.1.1.2.cmml" xref="S2.SS2.p1.2.m2.1.1">subscript</csymbol><set id="S2.SS2.p1.2.m2.1.1.1.1.2.cmml" xref="S2.SS2.p1.2.m2.1.1.1.1.1"><apply id="S2.SS2.p1.2.m2.1.1.1.1.1.1.cmml" xref="S2.SS2.p1.2.m2.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p1.2.m2.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p1.2.m2.1.1.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p1.2.m2.1.1.1.1.1.1.2.cmml" xref="S2.SS2.p1.2.m2.1.1.1.1.1.1.2">italic-ϕ</ci><ci id="S2.SS2.p1.2.m2.1.1.1.1.1.1.3.cmml" xref="S2.SS2.p1.2.m2.1.1.1.1.1.1.3">𝑖</ci></apply></set><apply id="S2.SS2.p1.2.m2.1.1.1.3.cmml" xref="S2.SS2.p1.2.m2.1.1.1.3"><eq id="S2.SS2.p1.2.m2.1.1.1.3.1.cmml" xref="S2.SS2.p1.2.m2.1.1.1.3.1"></eq><ci id="S2.SS2.p1.2.m2.1.1.1.3.2.cmml" xref="S2.SS2.p1.2.m2.1.1.1.3.2">𝑖</ci><cn type="integer" id="S2.SS2.p1.2.m2.1.1.1.3.3.cmml" xref="S2.SS2.p1.2.m2.1.1.1.3.3">0</cn></apply></apply><apply id="S2.SS2.p1.2.m2.1.1.3.cmml" xref="S2.SS2.p1.2.m2.1.1.3"><minus id="S2.SS2.p1.2.m2.1.1.3.1.cmml" xref="S2.SS2.p1.2.m2.1.1.3.1"></minus><ci id="S2.SS2.p1.2.m2.1.1.3.2.cmml" xref="S2.SS2.p1.2.m2.1.1.3.2">𝑁</ci><cn type="integer" id="S2.SS2.p1.2.m2.1.1.3.3.cmml" xref="S2.SS2.p1.2.m2.1.1.3.3">1</cn></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p1.2.m2.1c">\{\phi_{i}\}_{i=0}^{N-1}</annotation></semantics></math>.
We define intervention <span id="S2.SS2.p1.8.1" class="ltx_text ltx_font_italic">composition</span> as the application of multiple interventions to a model <math id="S2.SS2.p1.3.m3.1" class="ltx_Math" alttext="f" display="inline"><semantics id="S2.SS2.p1.3.m3.1a"><mi id="S2.SS2.p1.3.m3.1.1" xref="S2.SS2.p1.3.m3.1.1.cmml">f</mi><annotation-xml encoding="MathML-Content" id="S2.SS2.p1.3.m3.1b"><ci id="S2.SS2.p1.3.m3.1.1.cmml" xref="S2.SS2.p1.3.m3.1.1">𝑓</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p1.3.m3.1c">f</annotation></semantics></math> in sequence.<span id="footnote3" class="ltx_note ltx_role_footnote"><sup class="ltx_note_mark">3</sup><span class="ltx_note_outer"><span class="ltx_note_content"><sup class="ltx_note_mark">3</sup><span class="ltx_tag ltx_tag_note">3</span>For readability, we describe our framework in terms of two interventions.</span></span></span>
For example, we might apply intervention <math id="S2.SS2.p1.4.m4.1" class="ltx_Math" alttext="\phi_{0}" display="inline"><semantics id="S2.SS2.p1.4.m4.1a"><msub id="S2.SS2.p1.4.m4.1.1" xref="S2.SS2.p1.4.m4.1.1.cmml"><mi id="S2.SS2.p1.4.m4.1.1.2" xref="S2.SS2.p1.4.m4.1.1.2.cmml">ϕ</mi><mn id="S2.SS2.p1.4.m4.1.1.3" xref="S2.SS2.p1.4.m4.1.1.3.cmml">0</mn></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p1.4.m4.1b"><apply id="S2.SS2.p1.4.m4.1.1.cmml" xref="S2.SS2.p1.4.m4.1.1"><csymbol cd="ambiguous" id="S2.SS2.p1.4.m4.1.1.1.cmml" xref="S2.SS2.p1.4.m4.1.1">subscript</csymbol><ci id="S2.SS2.p1.4.m4.1.1.2.cmml" xref="S2.SS2.p1.4.m4.1.1.2">italic-ϕ</ci><cn type="integer" id="S2.SS2.p1.4.m4.1.1.3.cmml" xref="S2.SS2.p1.4.m4.1.1.3">0</cn></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p1.4.m4.1c">\phi_{0}</annotation></semantics></math> to model <math id="S2.SS2.p1.5.m5.1" class="ltx_Math" alttext="f_{\theta}" display="inline"><semantics id="S2.SS2.p1.5.m5.1a"><msub id="S2.SS2.p1.5.m5.1.1" xref="S2.SS2.p1.5.m5.1.1.cmml"><mi id="S2.SS2.p1.5.m5.1.1.2" xref="S2.SS2.p1.5.m5.1.1.2.cmml">f</mi><mi id="S2.SS2.p1.5.m5.1.1.3" xref="S2.SS2.p1.5.m5.1.1.3.cmml">θ</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p1.5.m5.1b"><apply id="S2.SS2.p1.5.m5.1.1.cmml" xref="S2.SS2.p1.5.m5.1.1"><csymbol cd="ambiguous" id="S2.SS2.p1.5.m5.1.1.1.cmml" xref="S2.SS2.p1.5.m5.1.1">subscript</csymbol><ci id="S2.SS2.p1.5.m5.1.1.2.cmml" xref="S2.SS2.p1.5.m5.1.1.2">𝑓</ci><ci id="S2.SS2.p1.5.m5.1.1.3.cmml" xref="S2.SS2.p1.5.m5.1.1.3">𝜃</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p1.5.m5.1c">f_{\theta}</annotation></semantics></math>, then <math id="S2.SS2.p1.6.m6.1" class="ltx_Math" alttext="\phi_{1}" display="inline"><semantics id="S2.SS2.p1.6.m6.1a"><msub id="S2.SS2.p1.6.m6.1.1" xref="S2.SS2.p1.6.m6.1.1.cmml"><mi id="S2.SS2.p1.6.m6.1.1.2" xref="S2.SS2.p1.6.m6.1.1.2.cmml">ϕ</mi><mn id="S2.SS2.p1.6.m6.1.1.3" xref="S2.SS2.p1.6.m6.1.1.3.cmml">1</mn></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p1.6.m6.1b"><apply id="S2.SS2.p1.6.m6.1.1.cmml" xref="S2.SS2.p1.6.m6.1.1"><csymbol cd="ambiguous" id="S2.SS2.p1.6.m6.1.1.1.cmml" xref="S2.SS2.p1.6.m6.1.1">subscript</csymbol><ci id="S2.SS2.p1.6.m6.1.1.2.cmml" xref="S2.SS2.p1.6.m6.1.1.2">italic-ϕ</ci><cn type="integer" id="S2.SS2.p1.6.m6.1.1.3.cmml" xref="S2.SS2.p1.6.m6.1.1.3">1</cn></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p1.6.m6.1c">\phi_{1}</annotation></semantics></math> to the intervened-upon model.
We can write this composition as either <math id="S2.SS2.p1.7.m7.1" class="ltx_Math" alttext="\phi_{1}(\phi_{0}(f_{\theta}))" display="inline"><semantics id="S2.SS2.p1.7.m7.1a"><mrow id="S2.SS2.p1.7.m7.1.1" xref="S2.SS2.p1.7.m7.1.1.cmml"><msub id="S2.SS2.p1.7.m7.1.1.3" xref="S2.SS2.p1.7.m7.1.1.3.cmml"><mi id="S2.SS2.p1.7.m7.1.1.3.2" xref="S2.SS2.p1.7.m7.1.1.3.2.cmml">ϕ</mi><mn id="S2.SS2.p1.7.m7.1.1.3.3" xref="S2.SS2.p1.7.m7.1.1.3.3.cmml">1</mn></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p1.7.m7.1.1.2" xref="S2.SS2.p1.7.m7.1.1.2.cmml">​</mo><mrow id="S2.SS2.p1.7.m7.1.1.1.1" xref="S2.SS2.p1.7.m7.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p1.7.m7.1.1.1.1.2" xref="S2.SS2.p1.7.m7.1.1.1.1.1.cmml">(</mo><mrow id="S2.SS2.p1.7.m7.1.1.1.1.1" xref="S2.SS2.p1.7.m7.1.1.1.1.1.cmml"><msub id="S2.SS2.p1.7.m7.1.1.1.1.1.3" xref="S2.SS2.p1.7.m7.1.1.1.1.1.3.cmml"><mi id="S2.SS2.p1.7.m7.1.1.1.1.1.3.2" xref="S2.SS2.p1.7.m7.1.1.1.1.1.3.2.cmml">ϕ</mi><mn id="S2.SS2.p1.7.m7.1.1.1.1.1.3.3" xref="S2.SS2.p1.7.m7.1.1.1.1.1.3.3.cmml">0</mn></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p1.7.m7.1.1.1.1.1.2" xref="S2.SS2.p1.7.m7.1.1.1.1.1.2.cmml">​</mo><mrow id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.2" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.cmml">(</mo><msub id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.cmml"><mi id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.2" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.3" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.3" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.SS2.p1.7.m7.1.1.1.1.3" xref="S2.SS2.p1.7.m7.1.1.1.1.1.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS2.p1.7.m7.1b"><apply id="S2.SS2.p1.7.m7.1.1.cmml" xref="S2.SS2.p1.7.m7.1.1"><times id="S2.SS2.p1.7.m7.1.1.2.cmml" xref="S2.SS2.p1.7.m7.1.1.2"></times><apply id="S2.SS2.p1.7.m7.1.1.3.cmml" xref="S2.SS2.p1.7.m7.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p1.7.m7.1.1.3.1.cmml" xref="S2.SS2.p1.7.m7.1.1.3">subscript</csymbol><ci id="S2.SS2.p1.7.m7.1.1.3.2.cmml" xref="S2.SS2.p1.7.m7.1.1.3.2">italic-ϕ</ci><cn type="integer" id="S2.SS2.p1.7.m7.1.1.3.3.cmml" xref="S2.SS2.p1.7.m7.1.1.3.3">1</cn></apply><apply id="S2.SS2.p1.7.m7.1.1.1.1.1.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1"><times id="S2.SS2.p1.7.m7.1.1.1.1.1.2.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.2"></times><apply id="S2.SS2.p1.7.m7.1.1.1.1.1.3.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p1.7.m7.1.1.1.1.1.3.1.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.3">subscript</csymbol><ci id="S2.SS2.p1.7.m7.1.1.1.1.1.3.2.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.3.2">italic-ϕ</ci><cn type="integer" id="S2.SS2.p1.7.m7.1.1.1.1.1.3.3.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.3.3">0</cn></apply><apply id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.2.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.2">𝑓</ci><ci id="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.3.cmml" xref="S2.SS2.p1.7.m7.1.1.1.1.1.1.1.1.3">𝜃</ci></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p1.7.m7.1c">\phi_{1}(\phi_{0}(f_{\theta}))</annotation></semantics></math> or <math id="S2.SS2.p1.8.m8.1" class="ltx_Math" alttext="\phi_{1}\circ\phi_{0}(f_{\theta})" display="inline"><semantics id="S2.SS2.p1.8.m8.1a"><mrow id="S2.SS2.p1.8.m8.1.1" xref="S2.SS2.p1.8.m8.1.1.cmml"><mrow id="S2.SS2.p1.8.m8.1.1.3" xref="S2.SS2.p1.8.m8.1.1.3.cmml"><msub id="S2.SS2.p1.8.m8.1.1.3.2" xref="S2.SS2.p1.8.m8.1.1.3.2.cmml"><mi id="S2.SS2.p1.8.m8.1.1.3.2.2" xref="S2.SS2.p1.8.m8.1.1.3.2.2.cmml">ϕ</mi><mn id="S2.SS2.p1.8.m8.1.1.3.2.3" xref="S2.SS2.p1.8.m8.1.1.3.2.3.cmml">1</mn></msub><mo lspace="0.222em" rspace="0.222em" id="S2.SS2.p1.8.m8.1.1.3.1" xref="S2.SS2.p1.8.m8.1.1.3.1.cmml">∘</mo><msub id="S2.SS2.p1.8.m8.1.1.3.3" xref="S2.SS2.p1.8.m8.1.1.3.3.cmml"><mi id="S2.SS2.p1.8.m8.1.1.3.3.2" xref="S2.SS2.p1.8.m8.1.1.3.3.2.cmml">ϕ</mi><mn id="S2.SS2.p1.8.m8.1.1.3.3.3" xref="S2.SS2.p1.8.m8.1.1.3.3.3.cmml">0</mn></msub></mrow><mo lspace="0em" rspace="0em" id="S2.SS2.p1.8.m8.1.1.2" xref="S2.SS2.p1.8.m8.1.1.2.cmml">​</mo><mrow id="S2.SS2.p1.8.m8.1.1.1.1" xref="S2.SS2.p1.8.m8.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p1.8.m8.1.1.1.1.2" xref="S2.SS2.p1.8.m8.1.1.1.1.1.cmml">(</mo><msub id="S2.SS2.p1.8.m8.1.1.1.1.1" xref="S2.SS2.p1.8.m8.1.1.1.1.1.cmml"><mi id="S2.SS2.p1.8.m8.1.1.1.1.1.2" xref="S2.SS2.p1.8.m8.1.1.1.1.1.2.cmml">f</mi><mi id="S2.SS2.p1.8.m8.1.1.1.1.1.3" xref="S2.SS2.p1.8.m8.1.1.1.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.SS2.p1.8.m8.1.1.1.1.3" xref="S2.SS2.p1.8.m8.1.1.1.1.1.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS2.p1.8.m8.1b"><apply id="S2.SS2.p1.8.m8.1.1.cmml" xref="S2.SS2.p1.8.m8.1.1"><times id="S2.SS2.p1.8.m8.1.1.2.cmml" xref="S2.SS2.p1.8.m8.1.1.2"></times><apply id="S2.SS2.p1.8.m8.1.1.3.cmml" xref="S2.SS2.p1.8.m8.1.1.3"><compose id="S2.SS2.p1.8.m8.1.1.3.1.cmml" xref="S2.SS2.p1.8.m8.1.1.3.1"></compose><apply id="S2.SS2.p1.8.m8.1.1.3.2.cmml" xref="S2.SS2.p1.8.m8.1.1.3.2"><csymbol cd="ambiguous" id="S2.SS2.p1.8.m8.1.1.3.2.1.cmml" xref="S2.SS2.p1.8.m8.1.1.3.2">subscript</csymbol><ci id="S2.SS2.p1.8.m8.1.1.3.2.2.cmml" xref="S2.SS2.p1.8.m8.1.1.3.2.2">italic-ϕ</ci><cn type="integer" id="S2.SS2.p1.8.m8.1.1.3.2.3.cmml" xref="S2.SS2.p1.8.m8.1.1.3.2.3">1</cn></apply><apply id="S2.SS2.p1.8.m8.1.1.3.3.cmml" xref="S2.SS2.p1.8.m8.1.1.3.3"><csymbol cd="ambiguous" id="S2.SS2.p1.8.m8.1.1.3.3.1.cmml" xref="S2.SS2.p1.8.m8.1.1.3.3">subscript</csymbol><ci id="S2.SS2.p1.8.m8.1.1.3.3.2.cmml" xref="S2.SS2.p1.8.m8.1.1.3.3.2">italic-ϕ</ci><cn type="integer" id="S2.SS2.p1.8.m8.1.1.3.3.3.cmml" xref="S2.SS2.p1.8.m8.1.1.3.3.3">0</cn></apply></apply><apply id="S2.SS2.p1.8.m8.1.1.1.1.1.cmml" xref="S2.SS2.p1.8.m8.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p1.8.m8.1.1.1.1.1.1.cmml" xref="S2.SS2.p1.8.m8.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p1.8.m8.1.1.1.1.1.2.cmml" xref="S2.SS2.p1.8.m8.1.1.1.1.1.2">𝑓</ci><ci id="S2.SS2.p1.8.m8.1.1.1.1.1.3.cmml" xref="S2.SS2.p1.8.m8.1.1.1.1.1.3">𝜃</ci></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p1.8.m8.1c">\phi_{1}\circ\phi_{0}(f_{\theta})</annotation></semantics></math>, and here we use the latter format.</p>
</div>
<div id="S2.SS2.p2" class="ltx_para">
<p id="S2.SS2.p2.3" class="ltx_p">Most interventions <math id="S2.SS2.p2.1.m1.1" class="ltx_Math" alttext="\phi_{i}" display="inline"><semantics id="S2.SS2.p2.1.m1.1a"><msub id="S2.SS2.p2.1.m1.1.1" xref="S2.SS2.p2.1.m1.1.1.cmml"><mi id="S2.SS2.p2.1.m1.1.1.2" xref="S2.SS2.p2.1.m1.1.1.2.cmml">ϕ</mi><mi id="S2.SS2.p2.1.m1.1.1.3" xref="S2.SS2.p2.1.m1.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p2.1.m1.1b"><apply id="S2.SS2.p2.1.m1.1.1.cmml" xref="S2.SS2.p2.1.m1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p2.1.m1.1.1.1.cmml" xref="S2.SS2.p2.1.m1.1.1">subscript</csymbol><ci id="S2.SS2.p2.1.m1.1.1.2.cmml" xref="S2.SS2.p2.1.m1.1.1.2">italic-ϕ</ci><ci id="S2.SS2.p2.1.m1.1.1.3.cmml" xref="S2.SS2.p2.1.m1.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p2.1.m1.1c">\phi_{i}</annotation></semantics></math> are developed specifically for their own criterion <math id="S2.SS2.p2.2.m2.1" class="ltx_Math" alttext="\kappa_{i}" display="inline"><semantics id="S2.SS2.p2.2.m2.1a"><msub id="S2.SS2.p2.2.m2.1.1" xref="S2.SS2.p2.2.m2.1.1.cmml"><mi id="S2.SS2.p2.2.m2.1.1.2" xref="S2.SS2.p2.2.m2.1.1.2.cmml">κ</mi><mi id="S2.SS2.p2.2.m2.1.1.3" xref="S2.SS2.p2.2.m2.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p2.2.m2.1b"><apply id="S2.SS2.p2.2.m2.1.1.cmml" xref="S2.SS2.p2.2.m2.1.1"><csymbol cd="ambiguous" id="S2.SS2.p2.2.m2.1.1.1.cmml" xref="S2.SS2.p2.2.m2.1.1">subscript</csymbol><ci id="S2.SS2.p2.2.m2.1.1.2.cmml" xref="S2.SS2.p2.2.m2.1.1.2">𝜅</ci><ci id="S2.SS2.p2.2.m2.1.1.3.cmml" xref="S2.SS2.p2.2.m2.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p2.2.m2.1c">\kappa_{i}</annotation></semantics></math>.
For simplicity, we let each intervention have one criterion, though in practice they often have multiple.
We propose that the quality of many interventions in practice should also relate to <span id="S2.SS2.p2.3.1" class="ltx_text ltx_font_italic">how an intervention impacts our ability to apply other interventions</span>.
To meet this need, we propose two metrics that measure such <span id="S2.SS2.p2.3.2" class="ltx_text ltx_font_italic">composability</span>.
Each is computed for a given criterion <math id="S2.SS2.p2.3.m3.1" class="ltx_Math" alttext="\kappa_{i}" display="inline"><semantics id="S2.SS2.p2.3.m3.1a"><msub id="S2.SS2.p2.3.m3.1.1" xref="S2.SS2.p2.3.m3.1.1.cmml"><mi id="S2.SS2.p2.3.m3.1.1.2" xref="S2.SS2.p2.3.m3.1.1.2.cmml">κ</mi><mi id="S2.SS2.p2.3.m3.1.1.3" xref="S2.SS2.p2.3.m3.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p2.3.m3.1b"><apply id="S2.SS2.p2.3.m3.1.1.cmml" xref="S2.SS2.p2.3.m3.1.1"><csymbol cd="ambiguous" id="S2.SS2.p2.3.m3.1.1.1.cmml" xref="S2.SS2.p2.3.m3.1.1">subscript</csymbol><ci id="S2.SS2.p2.3.m3.1.1.2.cmml" xref="S2.SS2.p2.3.m3.1.1.2">𝜅</ci><ci id="S2.SS2.p2.3.m3.1.1.3.cmml" xref="S2.SS2.p2.3.m3.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p2.3.m3.1c">\kappa_{i}</annotation></semantics></math> and can be computed for a single hyperparameter setting or ranged over multiple settings to attain a richer description of intervention interactions.</p>
</div>
<div id="S2.SS2.p3" class="ltx_para">
<p id="S2.SS2.p3.9" class="ltx_p">Our evaluation strategy is straightforward: After applying an intervention <math id="S2.SS2.p3.1.m1.1" class="ltx_Math" alttext="\phi_{i}(f_{\theta})" display="inline"><semantics id="S2.SS2.p3.1.m1.1a"><mrow id="S2.SS2.p3.1.m1.1.1" xref="S2.SS2.p3.1.m1.1.1.cmml"><msub id="S2.SS2.p3.1.m1.1.1.3" xref="S2.SS2.p3.1.m1.1.1.3.cmml"><mi id="S2.SS2.p3.1.m1.1.1.3.2" xref="S2.SS2.p3.1.m1.1.1.3.2.cmml">ϕ</mi><mi id="S2.SS2.p3.1.m1.1.1.3.3" xref="S2.SS2.p3.1.m1.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p3.1.m1.1.1.2" xref="S2.SS2.p3.1.m1.1.1.2.cmml">​</mo><mrow id="S2.SS2.p3.1.m1.1.1.1.1" xref="S2.SS2.p3.1.m1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p3.1.m1.1.1.1.1.2" xref="S2.SS2.p3.1.m1.1.1.1.1.1.cmml">(</mo><msub id="S2.SS2.p3.1.m1.1.1.1.1.1" xref="S2.SS2.p3.1.m1.1.1.1.1.1.cmml"><mi id="S2.SS2.p3.1.m1.1.1.1.1.1.2" xref="S2.SS2.p3.1.m1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.SS2.p3.1.m1.1.1.1.1.1.3" xref="S2.SS2.p3.1.m1.1.1.1.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.SS2.p3.1.m1.1.1.1.1.3" xref="S2.SS2.p3.1.m1.1.1.1.1.1.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.1.m1.1b"><apply id="S2.SS2.p3.1.m1.1.1.cmml" xref="S2.SS2.p3.1.m1.1.1"><times id="S2.SS2.p3.1.m1.1.1.2.cmml" xref="S2.SS2.p3.1.m1.1.1.2"></times><apply id="S2.SS2.p3.1.m1.1.1.3.cmml" xref="S2.SS2.p3.1.m1.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p3.1.m1.1.1.3.1.cmml" xref="S2.SS2.p3.1.m1.1.1.3">subscript</csymbol><ci id="S2.SS2.p3.1.m1.1.1.3.2.cmml" xref="S2.SS2.p3.1.m1.1.1.3.2">italic-ϕ</ci><ci id="S2.SS2.p3.1.m1.1.1.3.3.cmml" xref="S2.SS2.p3.1.m1.1.1.3.3">𝑖</ci></apply><apply id="S2.SS2.p3.1.m1.1.1.1.1.1.cmml" xref="S2.SS2.p3.1.m1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.1.m1.1.1.1.1.1.1.cmml" xref="S2.SS2.p3.1.m1.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p3.1.m1.1.1.1.1.1.2.cmml" xref="S2.SS2.p3.1.m1.1.1.1.1.1.2">𝑓</ci><ci id="S2.SS2.p3.1.m1.1.1.1.1.1.3.cmml" xref="S2.SS2.p3.1.m1.1.1.1.1.1.3">𝜃</ci></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.1.m1.1c">\phi_{i}(f_{\theta})</annotation></semantics></math>, we measure its success as <math id="S2.SS2.p3.2.m2.1" class="ltx_Math" alttext="\kappa_{i}(\phi_{i}(f_{\theta}))" display="inline"><semantics id="S2.SS2.p3.2.m2.1a"><mrow id="S2.SS2.p3.2.m2.1.1" xref="S2.SS2.p3.2.m2.1.1.cmml"><msub id="S2.SS2.p3.2.m2.1.1.3" xref="S2.SS2.p3.2.m2.1.1.3.cmml"><mi id="S2.SS2.p3.2.m2.1.1.3.2" xref="S2.SS2.p3.2.m2.1.1.3.2.cmml">κ</mi><mi id="S2.SS2.p3.2.m2.1.1.3.3" xref="S2.SS2.p3.2.m2.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p3.2.m2.1.1.2" xref="S2.SS2.p3.2.m2.1.1.2.cmml">​</mo><mrow id="S2.SS2.p3.2.m2.1.1.1.1" xref="S2.SS2.p3.2.m2.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p3.2.m2.1.1.1.1.2" xref="S2.SS2.p3.2.m2.1.1.1.1.1.cmml">(</mo><mrow id="S2.SS2.p3.2.m2.1.1.1.1.1" xref="S2.SS2.p3.2.m2.1.1.1.1.1.cmml"><msub id="S2.SS2.p3.2.m2.1.1.1.1.1.3" xref="S2.SS2.p3.2.m2.1.1.1.1.1.3.cmml"><mi id="S2.SS2.p3.2.m2.1.1.1.1.1.3.2" xref="S2.SS2.p3.2.m2.1.1.1.1.1.3.2.cmml">ϕ</mi><mi id="S2.SS2.p3.2.m2.1.1.1.1.1.3.3" xref="S2.SS2.p3.2.m2.1.1.1.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p3.2.m2.1.1.1.1.1.2" xref="S2.SS2.p3.2.m2.1.1.1.1.1.2.cmml">​</mo><mrow id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.2" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.cmml">(</mo><msub id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.cmml"><mi id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.2" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.3" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.3" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.SS2.p3.2.m2.1.1.1.1.3" xref="S2.SS2.p3.2.m2.1.1.1.1.1.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.2.m2.1b"><apply id="S2.SS2.p3.2.m2.1.1.cmml" xref="S2.SS2.p3.2.m2.1.1"><times id="S2.SS2.p3.2.m2.1.1.2.cmml" xref="S2.SS2.p3.2.m2.1.1.2"></times><apply id="S2.SS2.p3.2.m2.1.1.3.cmml" xref="S2.SS2.p3.2.m2.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p3.2.m2.1.1.3.1.cmml" xref="S2.SS2.p3.2.m2.1.1.3">subscript</csymbol><ci id="S2.SS2.p3.2.m2.1.1.3.2.cmml" xref="S2.SS2.p3.2.m2.1.1.3.2">𝜅</ci><ci id="S2.SS2.p3.2.m2.1.1.3.3.cmml" xref="S2.SS2.p3.2.m2.1.1.3.3">𝑖</ci></apply><apply id="S2.SS2.p3.2.m2.1.1.1.1.1.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1"><times id="S2.SS2.p3.2.m2.1.1.1.1.1.2.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.2"></times><apply id="S2.SS2.p3.2.m2.1.1.1.1.1.3.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p3.2.m2.1.1.1.1.1.3.1.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.3">subscript</csymbol><ci id="S2.SS2.p3.2.m2.1.1.1.1.1.3.2.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.3.2">italic-ϕ</ci><ci id="S2.SS2.p3.2.m2.1.1.1.1.1.3.3.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.3.3">𝑖</ci></apply><apply id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.2.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.2">𝑓</ci><ci id="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.3.cmml" xref="S2.SS2.p3.2.m2.1.1.1.1.1.1.1.1.3">𝜃</ci></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.2.m2.1c">\kappa_{i}(\phi_{i}(f_{\theta}))</annotation></semantics></math>.
Then, we can add another intervention <math id="S2.SS2.p3.3.m3.1" class="ltx_Math" alttext="\phi_{j}" display="inline"><semantics id="S2.SS2.p3.3.m3.1a"><msub id="S2.SS2.p3.3.m3.1.1" xref="S2.SS2.p3.3.m3.1.1.cmml"><mi id="S2.SS2.p3.3.m3.1.1.2" xref="S2.SS2.p3.3.m3.1.1.2.cmml">ϕ</mi><mi id="S2.SS2.p3.3.m3.1.1.3" xref="S2.SS2.p3.3.m3.1.1.3.cmml">j</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.3.m3.1b"><apply id="S2.SS2.p3.3.m3.1.1.cmml" xref="S2.SS2.p3.3.m3.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.3.m3.1.1.1.cmml" xref="S2.SS2.p3.3.m3.1.1">subscript</csymbol><ci id="S2.SS2.p3.3.m3.1.1.2.cmml" xref="S2.SS2.p3.3.m3.1.1.2">italic-ϕ</ci><ci id="S2.SS2.p3.3.m3.1.1.3.cmml" xref="S2.SS2.p3.3.m3.1.1.3">𝑗</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.3.m3.1c">\phi_{j}</annotation></semantics></math> and measure <math id="S2.SS2.p3.4.m4.1" class="ltx_Math" alttext="\kappa_{i}(\phi_{j}\circ\phi_{i}(f_{\theta}))" display="inline"><semantics id="S2.SS2.p3.4.m4.1a"><mrow id="S2.SS2.p3.4.m4.1.1" xref="S2.SS2.p3.4.m4.1.1.cmml"><msub id="S2.SS2.p3.4.m4.1.1.3" xref="S2.SS2.p3.4.m4.1.1.3.cmml"><mi id="S2.SS2.p3.4.m4.1.1.3.2" xref="S2.SS2.p3.4.m4.1.1.3.2.cmml">κ</mi><mi id="S2.SS2.p3.4.m4.1.1.3.3" xref="S2.SS2.p3.4.m4.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p3.4.m4.1.1.2" xref="S2.SS2.p3.4.m4.1.1.2.cmml">​</mo><mrow id="S2.SS2.p3.4.m4.1.1.1.1" xref="S2.SS2.p3.4.m4.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p3.4.m4.1.1.1.1.2" xref="S2.SS2.p3.4.m4.1.1.1.1.1.cmml">(</mo><mrow id="S2.SS2.p3.4.m4.1.1.1.1.1" xref="S2.SS2.p3.4.m4.1.1.1.1.1.cmml"><mrow id="S2.SS2.p3.4.m4.1.1.1.1.1.3" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.cmml"><msub id="S2.SS2.p3.4.m4.1.1.1.1.1.3.2" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.cmml"><mi id="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.2" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.2.cmml">ϕ</mi><mi id="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.3" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.3.cmml">j</mi></msub><mo lspace="0.222em" rspace="0.222em" id="S2.SS2.p3.4.m4.1.1.1.1.1.3.1" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.1.cmml">∘</mo><msub id="S2.SS2.p3.4.m4.1.1.1.1.1.3.3" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.cmml"><mi id="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.2" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.2.cmml">ϕ</mi><mi id="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.3" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.3.cmml">i</mi></msub></mrow><mo lspace="0em" rspace="0em" id="S2.SS2.p3.4.m4.1.1.1.1.1.2" xref="S2.SS2.p3.4.m4.1.1.1.1.1.2.cmml">​</mo><mrow id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.2" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.cmml">(</mo><msub id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.cmml"><mi id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.2" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.3" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.3" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.SS2.p3.4.m4.1.1.1.1.3" xref="S2.SS2.p3.4.m4.1.1.1.1.1.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.4.m4.1b"><apply id="S2.SS2.p3.4.m4.1.1.cmml" xref="S2.SS2.p3.4.m4.1.1"><times id="S2.SS2.p3.4.m4.1.1.2.cmml" xref="S2.SS2.p3.4.m4.1.1.2"></times><apply id="S2.SS2.p3.4.m4.1.1.3.cmml" xref="S2.SS2.p3.4.m4.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p3.4.m4.1.1.3.1.cmml" xref="S2.SS2.p3.4.m4.1.1.3">subscript</csymbol><ci id="S2.SS2.p3.4.m4.1.1.3.2.cmml" xref="S2.SS2.p3.4.m4.1.1.3.2">𝜅</ci><ci id="S2.SS2.p3.4.m4.1.1.3.3.cmml" xref="S2.SS2.p3.4.m4.1.1.3.3">𝑖</ci></apply><apply id="S2.SS2.p3.4.m4.1.1.1.1.1.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1"><times id="S2.SS2.p3.4.m4.1.1.1.1.1.2.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.2"></times><apply id="S2.SS2.p3.4.m4.1.1.1.1.1.3.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3"><compose id="S2.SS2.p3.4.m4.1.1.1.1.1.3.1.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.1"></compose><apply id="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.2"><csymbol cd="ambiguous" id="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.1.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.2">subscript</csymbol><ci id="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.2.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.2">italic-ϕ</ci><ci id="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.3.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.2.3">𝑗</ci></apply><apply id="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.3"><csymbol cd="ambiguous" id="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.1.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.3">subscript</csymbol><ci id="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.2.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.2">italic-ϕ</ci><ci id="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.3.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.3.3.3">𝑖</ci></apply></apply><apply id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.2.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.2">𝑓</ci><ci id="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.3.cmml" xref="S2.SS2.p3.4.m4.1.1.1.1.1.1.1.1.3">𝜃</ci></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.4.m4.1c">\kappa_{i}(\phi_{j}\circ\phi_{i}(f_{\theta}))</annotation></semantics></math>.
The difference between these measures then elicits how <math id="S2.SS2.p3.5.m5.1" class="ltx_Math" alttext="\phi_{j}" display="inline"><semantics id="S2.SS2.p3.5.m5.1a"><msub id="S2.SS2.p3.5.m5.1.1" xref="S2.SS2.p3.5.m5.1.1.cmml"><mi id="S2.SS2.p3.5.m5.1.1.2" xref="S2.SS2.p3.5.m5.1.1.2.cmml">ϕ</mi><mi id="S2.SS2.p3.5.m5.1.1.3" xref="S2.SS2.p3.5.m5.1.1.3.cmml">j</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.5.m5.1b"><apply id="S2.SS2.p3.5.m5.1.1.cmml" xref="S2.SS2.p3.5.m5.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.5.m5.1.1.1.cmml" xref="S2.SS2.p3.5.m5.1.1">subscript</csymbol><ci id="S2.SS2.p3.5.m5.1.1.2.cmml" xref="S2.SS2.p3.5.m5.1.1.2">italic-ϕ</ci><ci id="S2.SS2.p3.5.m5.1.1.3.cmml" xref="S2.SS2.p3.5.m5.1.1.3">𝑗</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.5.m5.1c">\phi_{j}</annotation></semantics></math> impacts the success of <math id="S2.SS2.p3.6.m6.1" class="ltx_Math" alttext="\phi_{i}" display="inline"><semantics id="S2.SS2.p3.6.m6.1a"><msub id="S2.SS2.p3.6.m6.1.1" xref="S2.SS2.p3.6.m6.1.1.cmml"><mi id="S2.SS2.p3.6.m6.1.1.2" xref="S2.SS2.p3.6.m6.1.1.2.cmml">ϕ</mi><mi id="S2.SS2.p3.6.m6.1.1.3" xref="S2.SS2.p3.6.m6.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.6.m6.1b"><apply id="S2.SS2.p3.6.m6.1.1.cmml" xref="S2.SS2.p3.6.m6.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.6.m6.1.1.1.cmml" xref="S2.SS2.p3.6.m6.1.1">subscript</csymbol><ci id="S2.SS2.p3.6.m6.1.1.2.cmml" xref="S2.SS2.p3.6.m6.1.1.2">italic-ϕ</ci><ci id="S2.SS2.p3.6.m6.1.1.3.cmml" xref="S2.SS2.p3.6.m6.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.6.m6.1c">\phi_{i}</annotation></semantics></math> according to <math id="S2.SS2.p3.7.m7.1" class="ltx_Math" alttext="\kappa_{i}" display="inline"><semantics id="S2.SS2.p3.7.m7.1a"><msub id="S2.SS2.p3.7.m7.1.1" xref="S2.SS2.p3.7.m7.1.1.cmml"><mi id="S2.SS2.p3.7.m7.1.1.2" xref="S2.SS2.p3.7.m7.1.1.2.cmml">κ</mi><mi id="S2.SS2.p3.7.m7.1.1.3" xref="S2.SS2.p3.7.m7.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.7.m7.1b"><apply id="S2.SS2.p3.7.m7.1.1.cmml" xref="S2.SS2.p3.7.m7.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.7.m7.1.1.1.cmml" xref="S2.SS2.p3.7.m7.1.1">subscript</csymbol><ci id="S2.SS2.p3.7.m7.1.1.2.cmml" xref="S2.SS2.p3.7.m7.1.1.2">𝜅</ci><ci id="S2.SS2.p3.7.m7.1.1.3.cmml" xref="S2.SS2.p3.7.m7.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.7.m7.1c">\kappa_{i}</annotation></semantics></math>.
To disentangle absolute performance on <math id="S2.SS2.p3.8.m8.1" class="ltx_Math" alttext="\kappa_{i}" display="inline"><semantics id="S2.SS2.p3.8.m8.1a"><msub id="S2.SS2.p3.8.m8.1.1" xref="S2.SS2.p3.8.m8.1.1.cmml"><mi id="S2.SS2.p3.8.m8.1.1.2" xref="S2.SS2.p3.8.m8.1.1.2.cmml">κ</mi><mi id="S2.SS2.p3.8.m8.1.1.3" xref="S2.SS2.p3.8.m8.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.8.m8.1b"><apply id="S2.SS2.p3.8.m8.1.1.cmml" xref="S2.SS2.p3.8.m8.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.8.m8.1.1.1.cmml" xref="S2.SS2.p3.8.m8.1.1">subscript</csymbol><ci id="S2.SS2.p3.8.m8.1.1.2.cmml" xref="S2.SS2.p3.8.m8.1.1.2">𝜅</ci><ci id="S2.SS2.p3.8.m8.1.1.3.cmml" xref="S2.SS2.p3.8.m8.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.8.m8.1c">\kappa_{i}</annotation></semantics></math> from the interventions, we explore the impact of <span id="S2.SS2.p3.9.1" class="ltx_text ltx_font_italic">ordering</span> by reversing the interventions and computing <math id="S2.SS2.p3.9.m9.1" class="ltx_Math" alttext="\kappa_{i}(\phi_{i}\circ\phi_{j}(f_{\theta}))" display="inline"><semantics id="S2.SS2.p3.9.m9.1a"><mrow id="S2.SS2.p3.9.m9.1.1" xref="S2.SS2.p3.9.m9.1.1.cmml"><msub id="S2.SS2.p3.9.m9.1.1.3" xref="S2.SS2.p3.9.m9.1.1.3.cmml"><mi id="S2.SS2.p3.9.m9.1.1.3.2" xref="S2.SS2.p3.9.m9.1.1.3.2.cmml">κ</mi><mi id="S2.SS2.p3.9.m9.1.1.3.3" xref="S2.SS2.p3.9.m9.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p3.9.m9.1.1.2" xref="S2.SS2.p3.9.m9.1.1.2.cmml">​</mo><mrow id="S2.SS2.p3.9.m9.1.1.1.1" xref="S2.SS2.p3.9.m9.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p3.9.m9.1.1.1.1.2" xref="S2.SS2.p3.9.m9.1.1.1.1.1.cmml">(</mo><mrow id="S2.SS2.p3.9.m9.1.1.1.1.1" xref="S2.SS2.p3.9.m9.1.1.1.1.1.cmml"><mrow id="S2.SS2.p3.9.m9.1.1.1.1.1.3" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.cmml"><msub id="S2.SS2.p3.9.m9.1.1.1.1.1.3.2" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.cmml"><mi id="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.2" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.2.cmml">ϕ</mi><mi id="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.3" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.3.cmml">i</mi></msub><mo lspace="0.222em" rspace="0.222em" id="S2.SS2.p3.9.m9.1.1.1.1.1.3.1" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.1.cmml">∘</mo><msub id="S2.SS2.p3.9.m9.1.1.1.1.1.3.3" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.cmml"><mi id="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.2" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.2.cmml">ϕ</mi><mi id="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.3" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.3.cmml">j</mi></msub></mrow><mo lspace="0em" rspace="0em" id="S2.SS2.p3.9.m9.1.1.1.1.1.2" xref="S2.SS2.p3.9.m9.1.1.1.1.1.2.cmml">​</mo><mrow id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.2" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.cmml">(</mo><msub id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.cmml"><mi id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.2" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.3" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.3" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.SS2.p3.9.m9.1.1.1.1.3" xref="S2.SS2.p3.9.m9.1.1.1.1.1.cmml">)</mo></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS2.p3.9.m9.1b"><apply id="S2.SS2.p3.9.m9.1.1.cmml" xref="S2.SS2.p3.9.m9.1.1"><times id="S2.SS2.p3.9.m9.1.1.2.cmml" xref="S2.SS2.p3.9.m9.1.1.2"></times><apply id="S2.SS2.p3.9.m9.1.1.3.cmml" xref="S2.SS2.p3.9.m9.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p3.9.m9.1.1.3.1.cmml" xref="S2.SS2.p3.9.m9.1.1.3">subscript</csymbol><ci id="S2.SS2.p3.9.m9.1.1.3.2.cmml" xref="S2.SS2.p3.9.m9.1.1.3.2">𝜅</ci><ci id="S2.SS2.p3.9.m9.1.1.3.3.cmml" xref="S2.SS2.p3.9.m9.1.1.3.3">𝑖</ci></apply><apply id="S2.SS2.p3.9.m9.1.1.1.1.1.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1"><times id="S2.SS2.p3.9.m9.1.1.1.1.1.2.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.2"></times><apply id="S2.SS2.p3.9.m9.1.1.1.1.1.3.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3"><compose id="S2.SS2.p3.9.m9.1.1.1.1.1.3.1.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.1"></compose><apply id="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.2"><csymbol cd="ambiguous" id="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.1.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.2">subscript</csymbol><ci id="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.2.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.2">italic-ϕ</ci><ci id="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.3.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.2.3">𝑖</ci></apply><apply id="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.3"><csymbol cd="ambiguous" id="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.1.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.3">subscript</csymbol><ci id="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.2.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.2">italic-ϕ</ci><ci id="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.3.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.3.3.3">𝑗</ci></apply></apply><apply id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.2.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.2">𝑓</ci><ci id="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.3.cmml" xref="S2.SS2.p3.9.m9.1.1.1.1.1.1.1.1.3">𝜃</ci></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p3.9.m9.1c">\kappa_{i}(\phi_{i}\circ\phi_{j}(f_{\theta}))</annotation></semantics></math>.
By pairing absolute changes with results from different orderings, we isolate direct interactions between interventions.
Overall, this approach provides guidance when choosing interventions to compose and illuminates failure modes of existing methods.
We next describe two concrete composability metrics.
The first describes best-case composability regardless of order, and the second isolates the impact of ordering.</p>
</div>
<div id="S2.SS2.p4" class="ltx_para ltx_noindent">
<p id="S2.SS2.p4.2" class="ltx_p"><span id="S2.SS2.p4.2.1" class="ltx_text ltx_font_bold">Composability Error Metric 1: Order-free Error.</span>
Interventions are more composable if they achieve high performance when applied in either order.
We therefore measure the impacts of adding a second intervention to a model regardless of order for a chosen criterion <math id="S2.SS2.p4.1.m1.1" class="ltx_Math" alttext="\kappa_{i}" display="inline"><semantics id="S2.SS2.p4.1.m1.1a"><msub id="S2.SS2.p4.1.m1.1.1" xref="S2.SS2.p4.1.m1.1.1.cmml"><mi id="S2.SS2.p4.1.m1.1.1.2" xref="S2.SS2.p4.1.m1.1.1.2.cmml">κ</mi><mi id="S2.SS2.p4.1.m1.1.1.3" xref="S2.SS2.p4.1.m1.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p4.1.m1.1b"><apply id="S2.SS2.p4.1.m1.1.1.cmml" xref="S2.SS2.p4.1.m1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p4.1.m1.1.1.1.cmml" xref="S2.SS2.p4.1.m1.1.1">subscript</csymbol><ci id="S2.SS2.p4.1.m1.1.1.2.cmml" xref="S2.SS2.p4.1.m1.1.1.2">𝜅</ci><ci id="S2.SS2.p4.1.m1.1.1.3.cmml" xref="S2.SS2.p4.1.m1.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p4.1.m1.1c">\kappa_{i}</annotation></semantics></math> using <span id="S2.SS2.p4.2.2" class="ltx_text ltx_font_italic">order-free error</span>, computed as the area <span id="S2.SS2.p4.2.3" class="ltx_text ltx_font_italic">above</span> the maximum of two curves.
A small area means higher overall performance on the metric.
We use the maximum criterion value as an upper bound for this area, and assume it is 1, though this is easily changed in practice.
Each curve corresponds to an order in which each intervention is applied and sweeps over hyperparameter choices for <math id="S2.SS2.p4.2.m2.1" class="ltx_Math" alttext="\gamma_{i}" display="inline"><semantics id="S2.SS2.p4.2.m2.1a"><msub id="S2.SS2.p4.2.m2.1.1" xref="S2.SS2.p4.2.m2.1.1.cmml"><mi id="S2.SS2.p4.2.m2.1.1.2" xref="S2.SS2.p4.2.m2.1.1.2.cmml">γ</mi><mi id="S2.SS2.p4.2.m2.1.1.3" xref="S2.SS2.p4.2.m2.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p4.2.m2.1b"><apply id="S2.SS2.p4.2.m2.1.1.cmml" xref="S2.SS2.p4.2.m2.1.1"><csymbol cd="ambiguous" id="S2.SS2.p4.2.m2.1.1.1.cmml" xref="S2.SS2.p4.2.m2.1.1">subscript</csymbol><ci id="S2.SS2.p4.2.m2.1.1.2.cmml" xref="S2.SS2.p4.2.m2.1.1.2">𝛾</ci><ci id="S2.SS2.p4.2.m2.1.1.3.cmml" xref="S2.SS2.p4.2.m2.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p4.2.m2.1c">\gamma_{i}</annotation></semantics></math>:</p>
<table id="S2.E1" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S2.E1.m1.6" class="ltx_Math" alttext="1-\int_{\gamma_{i}}\min(\underbrace{\kappa_{i}(\phi_{j}\circ\phi_{i}(f_{\theta},\gamma_{i}))}_{\text{intervention order $i\rightarrow j$}},\underbrace{\kappa_{i}(\phi_{i}(\gamma_{i})\circ\phi_{j}(f_{\theta}))}_{\text{intervention order $j\rightarrow i$}})." display="block"><semantics id="S2.E1.m1.6a"><mrow id="S2.E1.m1.6.6.1" xref="S2.E1.m1.6.6.1.1.cmml"><mrow id="S2.E1.m1.6.6.1.1" xref="S2.E1.m1.6.6.1.1.cmml"><mn id="S2.E1.m1.6.6.1.1.4" xref="S2.E1.m1.6.6.1.1.4.cmml">1</mn><mo rspace="0.055em" id="S2.E1.m1.6.6.1.1.3" xref="S2.E1.m1.6.6.1.1.3.cmml">−</mo><mrow id="S2.E1.m1.6.6.1.1.2" xref="S2.E1.m1.6.6.1.1.2.cmml"><msub id="S2.E1.m1.6.6.1.1.2.3" xref="S2.E1.m1.6.6.1.1.2.3.cmml"><mo id="S2.E1.m1.6.6.1.1.2.3.2" xref="S2.E1.m1.6.6.1.1.2.3.2.cmml">∫</mo><msub id="S2.E1.m1.6.6.1.1.2.3.3" xref="S2.E1.m1.6.6.1.1.2.3.3.cmml"><mi id="S2.E1.m1.6.6.1.1.2.3.3.2" xref="S2.E1.m1.6.6.1.1.2.3.3.2.cmml">γ</mi><mi id="S2.E1.m1.6.6.1.1.2.3.3.3" xref="S2.E1.m1.6.6.1.1.2.3.3.3.cmml">i</mi></msub></msub><mrow id="S2.E1.m1.6.6.1.1.2.2.2" xref="S2.E1.m1.6.6.1.1.2.2.3.cmml"><mi id="S2.E1.m1.5.5" xref="S2.E1.m1.5.5.cmml">min</mi><mo id="S2.E1.m1.6.6.1.1.2.2.2a" xref="S2.E1.m1.6.6.1.1.2.2.3.cmml">⁡</mo><mrow id="S2.E1.m1.6.6.1.1.2.2.2.2" xref="S2.E1.m1.6.6.1.1.2.2.3.cmml"><mo stretchy="false" id="S2.E1.m1.6.6.1.1.2.2.2.2.3" xref="S2.E1.m1.6.6.1.1.2.2.3.cmml">(</mo><munder id="S2.E1.m1.6.6.1.1.1.1.1.1.1" xref="S2.E1.m1.6.6.1.1.1.1.1.1.1.cmml"><munder accentunder="true" id="S2.E1.m1.3.3" xref="S2.E1.m1.3.3.cmml"><mrow id="S2.E1.m1.3.3.1" xref="S2.E1.m1.3.3.1.cmml"><msub id="S2.E1.m1.3.3.1.3" xref="S2.E1.m1.3.3.1.3.cmml"><mi id="S2.E1.m1.3.3.1.3.2" xref="S2.E1.m1.3.3.1.3.2.cmml">κ</mi><mi id="S2.E1.m1.3.3.1.3.3" xref="S2.E1.m1.3.3.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.E1.m1.3.3.1.2" xref="S2.E1.m1.3.3.1.2.cmml">​</mo><mrow id="S2.E1.m1.3.3.1.1.1" xref="S2.E1.m1.3.3.1.1.1.1.cmml"><mo stretchy="false" id="S2.E1.m1.3.3.1.1.1.2" xref="S2.E1.m1.3.3.1.1.1.1.cmml">(</mo><mrow id="S2.E1.m1.3.3.1.1.1.1" xref="S2.E1.m1.3.3.1.1.1.1.cmml"><mrow id="S2.E1.m1.3.3.1.1.1.1.4" xref="S2.E1.m1.3.3.1.1.1.1.4.cmml"><msub id="S2.E1.m1.3.3.1.1.1.1.4.2" xref="S2.E1.m1.3.3.1.1.1.1.4.2.cmml"><mi id="S2.E1.m1.3.3.1.1.1.1.4.2.2" xref="S2.E1.m1.3.3.1.1.1.1.4.2.2.cmml">ϕ</mi><mi id="S2.E1.m1.3.3.1.1.1.1.4.2.3" xref="S2.E1.m1.3.3.1.1.1.1.4.2.3.cmml">j</mi></msub><mo lspace="0.222em" rspace="0.222em" id="S2.E1.m1.3.3.1.1.1.1.4.1" xref="S2.E1.m1.3.3.1.1.1.1.4.1.cmml">∘</mo><msub id="S2.E1.m1.3.3.1.1.1.1.4.3" xref="S2.E1.m1.3.3.1.1.1.1.4.3.cmml"><mi id="S2.E1.m1.3.3.1.1.1.1.4.3.2" xref="S2.E1.m1.3.3.1.1.1.1.4.3.2.cmml">ϕ</mi><mi id="S2.E1.m1.3.3.1.1.1.1.4.3.3" xref="S2.E1.m1.3.3.1.1.1.1.4.3.3.cmml">i</mi></msub></mrow><mo lspace="0em" rspace="0em" id="S2.E1.m1.3.3.1.1.1.1.3" xref="S2.E1.m1.3.3.1.1.1.1.3.cmml">​</mo><mrow id="S2.E1.m1.3.3.1.1.1.1.2.2" xref="S2.E1.m1.3.3.1.1.1.1.2.3.cmml"><mo stretchy="false" id="S2.E1.m1.3.3.1.1.1.1.2.2.3" xref="S2.E1.m1.3.3.1.1.1.1.2.3.cmml">(</mo><msub id="S2.E1.m1.3.3.1.1.1.1.1.1.1" xref="S2.E1.m1.3.3.1.1.1.1.1.1.1.cmml"><mi id="S2.E1.m1.3.3.1.1.1.1.1.1.1.2" xref="S2.E1.m1.3.3.1.1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.E1.m1.3.3.1.1.1.1.1.1.1.3" xref="S2.E1.m1.3.3.1.1.1.1.1.1.1.3.cmml">θ</mi></msub><mo id="S2.E1.m1.3.3.1.1.1.1.2.2.4" xref="S2.E1.m1.3.3.1.1.1.1.2.3.cmml">,</mo><msub id="S2.E1.m1.3.3.1.1.1.1.2.2.2" xref="S2.E1.m1.3.3.1.1.1.1.2.2.2.cmml"><mi id="S2.E1.m1.3.3.1.1.1.1.2.2.2.2" xref="S2.E1.m1.3.3.1.1.1.1.2.2.2.2.cmml">γ</mi><mi id="S2.E1.m1.3.3.1.1.1.1.2.2.2.3" xref="S2.E1.m1.3.3.1.1.1.1.2.2.2.3.cmml">i</mi></msub><mo stretchy="false" id="S2.E1.m1.3.3.1.1.1.1.2.2.5" xref="S2.E1.m1.3.3.1.1.1.1.2.3.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.E1.m1.3.3.1.1.1.3" xref="S2.E1.m1.3.3.1.1.1.1.cmml">)</mo></mrow></mrow><mo id="S2.E1.m1.3.3.2" xref="S2.E1.m1.3.3.2.cmml">⏟</mo></munder><mrow id="S2.E1.m1.1.1.1" xref="S2.E1.m1.1.1.1b.cmml"><mtext id="S2.E1.m1.1.1.1a" xref="S2.E1.m1.1.1.1b.cmml">intervention order </mtext><mrow id="S2.E1.m1.1.1.1.1.1.m1.1.1" xref="S2.E1.m1.1.1.1.1.1.m1.1.1.cmml"><mi id="S2.E1.m1.1.1.1.1.1.m1.1.1.2" xref="S2.E1.m1.1.1.1.1.1.m1.1.1.2.cmml">i</mi><mo stretchy="false" id="S2.E1.m1.1.1.1.1.1.m1.1.1.1" xref="S2.E1.m1.1.1.1.1.1.m1.1.1.1.cmml">→</mo><mi id="S2.E1.m1.1.1.1.1.1.m1.1.1.3" xref="S2.E1.m1.1.1.1.1.1.m1.1.1.3.cmml">j</mi></mrow></mrow></munder><mo id="S2.E1.m1.6.6.1.1.2.2.2.2.4" xref="S2.E1.m1.6.6.1.1.2.2.3.cmml">,</mo><munder id="S2.E1.m1.6.6.1.1.2.2.2.2.2" xref="S2.E1.m1.6.6.1.1.2.2.2.2.2.cmml"><munder accentunder="true" id="S2.E1.m1.4.4" xref="S2.E1.m1.4.4.cmml"><mrow id="S2.E1.m1.4.4.1" xref="S2.E1.m1.4.4.1.cmml"><msub id="S2.E1.m1.4.4.1.3" xref="S2.E1.m1.4.4.1.3.cmml"><mi id="S2.E1.m1.4.4.1.3.2" xref="S2.E1.m1.4.4.1.3.2.cmml">κ</mi><mi id="S2.E1.m1.4.4.1.3.3" xref="S2.E1.m1.4.4.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.E1.m1.4.4.1.2" xref="S2.E1.m1.4.4.1.2.cmml">​</mo><mrow id="S2.E1.m1.4.4.1.1.1" xref="S2.E1.m1.4.4.1.1.1.1.cmml"><mo stretchy="false" id="S2.E1.m1.4.4.1.1.1.2" xref="S2.E1.m1.4.4.1.1.1.1.cmml">(</mo><mrow id="S2.E1.m1.4.4.1.1.1.1" xref="S2.E1.m1.4.4.1.1.1.1.cmml"><mrow id="S2.E1.m1.4.4.1.1.1.1.1" xref="S2.E1.m1.4.4.1.1.1.1.1.cmml"><mrow id="S2.E1.m1.4.4.1.1.1.1.1.1" xref="S2.E1.m1.4.4.1.1.1.1.1.1.cmml"><msub id="S2.E1.m1.4.4.1.1.1.1.1.1.3" xref="S2.E1.m1.4.4.1.1.1.1.1.1.3.cmml"><mi id="S2.E1.m1.4.4.1.1.1.1.1.1.3.2" xref="S2.E1.m1.4.4.1.1.1.1.1.1.3.2.cmml">ϕ</mi><mi id="S2.E1.m1.4.4.1.1.1.1.1.1.3.3" xref="S2.E1.m1.4.4.1.1.1.1.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.E1.m1.4.4.1.1.1.1.1.1.2" xref="S2.E1.m1.4.4.1.1.1.1.1.1.2.cmml">​</mo><mrow id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.2" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.cmml">(</mo><msub id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.cmml"><mi id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.2" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.2.cmml">γ</mi><mi id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.3" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.3.cmml">i</mi></msub><mo rspace="0.055em" stretchy="false" id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.3" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo rspace="0.222em" id="S2.E1.m1.4.4.1.1.1.1.1.2" xref="S2.E1.m1.4.4.1.1.1.1.1.2.cmml">∘</mo><msub id="S2.E1.m1.4.4.1.1.1.1.1.3" xref="S2.E1.m1.4.4.1.1.1.1.1.3.cmml"><mi id="S2.E1.m1.4.4.1.1.1.1.1.3.2" xref="S2.E1.m1.4.4.1.1.1.1.1.3.2.cmml">ϕ</mi><mi id="S2.E1.m1.4.4.1.1.1.1.1.3.3" xref="S2.E1.m1.4.4.1.1.1.1.1.3.3.cmml">j</mi></msub></mrow><mo lspace="0em" rspace="0em" id="S2.E1.m1.4.4.1.1.1.1.3" xref="S2.E1.m1.4.4.1.1.1.1.3.cmml">​</mo><mrow id="S2.E1.m1.4.4.1.1.1.1.2.1" xref="S2.E1.m1.4.4.1.1.1.1.2.1.1.cmml"><mo stretchy="false" id="S2.E1.m1.4.4.1.1.1.1.2.1.2" xref="S2.E1.m1.4.4.1.1.1.1.2.1.1.cmml">(</mo><msub id="S2.E1.m1.4.4.1.1.1.1.2.1.1" xref="S2.E1.m1.4.4.1.1.1.1.2.1.1.cmml"><mi id="S2.E1.m1.4.4.1.1.1.1.2.1.1.2" xref="S2.E1.m1.4.4.1.1.1.1.2.1.1.2.cmml">f</mi><mi id="S2.E1.m1.4.4.1.1.1.1.2.1.1.3" xref="S2.E1.m1.4.4.1.1.1.1.2.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.E1.m1.4.4.1.1.1.1.2.1.3" xref="S2.E1.m1.4.4.1.1.1.1.2.1.1.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.E1.m1.4.4.1.1.1.3" xref="S2.E1.m1.4.4.1.1.1.1.cmml">)</mo></mrow></mrow><mo id="S2.E1.m1.4.4.2" xref="S2.E1.m1.4.4.2.cmml">⏟</mo></munder><mrow id="S2.E1.m1.2.2.1" xref="S2.E1.m1.2.2.1b.cmml"><mtext id="S2.E1.m1.2.2.1a" xref="S2.E1.m1.2.2.1b.cmml">intervention order </mtext><mrow id="S2.E1.m1.2.2.1.1.1.m1.1.1" xref="S2.E1.m1.2.2.1.1.1.m1.1.1.cmml"><mi id="S2.E1.m1.2.2.1.1.1.m1.1.1.2" xref="S2.E1.m1.2.2.1.1.1.m1.1.1.2.cmml">j</mi><mo stretchy="false" id="S2.E1.m1.2.2.1.1.1.m1.1.1.1" xref="S2.E1.m1.2.2.1.1.1.m1.1.1.1.cmml">→</mo><mi id="S2.E1.m1.2.2.1.1.1.m1.1.1.3" xref="S2.E1.m1.2.2.1.1.1.m1.1.1.3.cmml">i</mi></mrow></mrow></munder><mo stretchy="false" id="S2.E1.m1.6.6.1.1.2.2.2.2.5" xref="S2.E1.m1.6.6.1.1.2.2.3.cmml">)</mo></mrow></mrow></mrow></mrow><mo lspace="0em" id="S2.E1.m1.6.6.1.2" xref="S2.E1.m1.6.6.1.1.cmml">.</mo></mrow><annotation-xml encoding="MathML-Content" id="S2.E1.m1.6b"><apply id="S2.E1.m1.6.6.1.1.cmml" xref="S2.E1.m1.6.6.1"><minus id="S2.E1.m1.6.6.1.1.3.cmml" xref="S2.E1.m1.6.6.1.1.3"></minus><cn type="integer" id="S2.E1.m1.6.6.1.1.4.cmml" xref="S2.E1.m1.6.6.1.1.4">1</cn><apply id="S2.E1.m1.6.6.1.1.2.cmml" xref="S2.E1.m1.6.6.1.1.2"><apply id="S2.E1.m1.6.6.1.1.2.3.cmml" xref="S2.E1.m1.6.6.1.1.2.3"><csymbol cd="ambiguous" id="S2.E1.m1.6.6.1.1.2.3.1.cmml" xref="S2.E1.m1.6.6.1.1.2.3">subscript</csymbol><int id="S2.E1.m1.6.6.1.1.2.3.2.cmml" xref="S2.E1.m1.6.6.1.1.2.3.2"></int><apply id="S2.E1.m1.6.6.1.1.2.3.3.cmml" xref="S2.E1.m1.6.6.1.1.2.3.3"><csymbol cd="ambiguous" id="S2.E1.m1.6.6.1.1.2.3.3.1.cmml" xref="S2.E1.m1.6.6.1.1.2.3.3">subscript</csymbol><ci id="S2.E1.m1.6.6.1.1.2.3.3.2.cmml" xref="S2.E1.m1.6.6.1.1.2.3.3.2">𝛾</ci><ci id="S2.E1.m1.6.6.1.1.2.3.3.3.cmml" xref="S2.E1.m1.6.6.1.1.2.3.3.3">𝑖</ci></apply></apply><apply id="S2.E1.m1.6.6.1.1.2.2.3.cmml" xref="S2.E1.m1.6.6.1.1.2.2.2"><min id="S2.E1.m1.5.5.cmml" xref="S2.E1.m1.5.5"></min><apply id="S2.E1.m1.6.6.1.1.1.1.1.1.1.cmml" xref="S2.E1.m1.6.6.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.E1.m1.6.6.1.1.1.1.1.1.1.1.cmml" xref="S2.E1.m1.6.6.1.1.1.1.1.1.1">subscript</csymbol><apply id="S2.E1.m1.3.3.cmml" xref="S2.E1.m1.3.3"><ci id="S2.E1.m1.3.3.2.cmml" xref="S2.E1.m1.3.3.2">⏟</ci><apply id="S2.E1.m1.3.3.1.cmml" xref="S2.E1.m1.3.3.1"><times id="S2.E1.m1.3.3.1.2.cmml" xref="S2.E1.m1.3.3.1.2"></times><apply id="S2.E1.m1.3.3.1.3.cmml" xref="S2.E1.m1.3.3.1.3"><csymbol cd="ambiguous" id="S2.E1.m1.3.3.1.3.1.cmml" xref="S2.E1.m1.3.3.1.3">subscript</csymbol><ci id="S2.E1.m1.3.3.1.3.2.cmml" xref="S2.E1.m1.3.3.1.3.2">𝜅</ci><ci id="S2.E1.m1.3.3.1.3.3.cmml" xref="S2.E1.m1.3.3.1.3.3">𝑖</ci></apply><apply id="S2.E1.m1.3.3.1.1.1.1.cmml" xref="S2.E1.m1.3.3.1.1.1"><times id="S2.E1.m1.3.3.1.1.1.1.3.cmml" xref="S2.E1.m1.3.3.1.1.1.1.3"></times><apply id="S2.E1.m1.3.3.1.1.1.1.4.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4"><compose id="S2.E1.m1.3.3.1.1.1.1.4.1.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.1"></compose><apply id="S2.E1.m1.3.3.1.1.1.1.4.2.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.2"><csymbol cd="ambiguous" id="S2.E1.m1.3.3.1.1.1.1.4.2.1.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.2">subscript</csymbol><ci id="S2.E1.m1.3.3.1.1.1.1.4.2.2.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.2.2">italic-ϕ</ci><ci id="S2.E1.m1.3.3.1.1.1.1.4.2.3.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.2.3">𝑗</ci></apply><apply id="S2.E1.m1.3.3.1.1.1.1.4.3.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.3"><csymbol cd="ambiguous" id="S2.E1.m1.3.3.1.1.1.1.4.3.1.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.3">subscript</csymbol><ci id="S2.E1.m1.3.3.1.1.1.1.4.3.2.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.3.2">italic-ϕ</ci><ci id="S2.E1.m1.3.3.1.1.1.1.4.3.3.cmml" xref="S2.E1.m1.3.3.1.1.1.1.4.3.3">𝑖</ci></apply></apply><interval closure="open" id="S2.E1.m1.3.3.1.1.1.1.2.3.cmml" xref="S2.E1.m1.3.3.1.1.1.1.2.2"><apply id="S2.E1.m1.3.3.1.1.1.1.1.1.1.cmml" xref="S2.E1.m1.3.3.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.E1.m1.3.3.1.1.1.1.1.1.1.1.cmml" xref="S2.E1.m1.3.3.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.E1.m1.3.3.1.1.1.1.1.1.1.2.cmml" xref="S2.E1.m1.3.3.1.1.1.1.1.1.1.2">𝑓</ci><ci id="S2.E1.m1.3.3.1.1.1.1.1.1.1.3.cmml" xref="S2.E1.m1.3.3.1.1.1.1.1.1.1.3">𝜃</ci></apply><apply id="S2.E1.m1.3.3.1.1.1.1.2.2.2.cmml" xref="S2.E1.m1.3.3.1.1.1.1.2.2.2"><csymbol cd="ambiguous" id="S2.E1.m1.3.3.1.1.1.1.2.2.2.1.cmml" xref="S2.E1.m1.3.3.1.1.1.1.2.2.2">subscript</csymbol><ci id="S2.E1.m1.3.3.1.1.1.1.2.2.2.2.cmml" xref="S2.E1.m1.3.3.1.1.1.1.2.2.2.2">𝛾</ci><ci id="S2.E1.m1.3.3.1.1.1.1.2.2.2.3.cmml" xref="S2.E1.m1.3.3.1.1.1.1.2.2.2.3">𝑖</ci></apply></interval></apply></apply></apply><ci id="S2.E1.m1.1.1.1b.cmml" xref="S2.E1.m1.1.1.1"><mrow id="S2.E1.m1.1.1.1.cmml" xref="S2.E1.m1.1.1.1"><mtext mathsize="70%" id="S2.E1.m1.1.1.1a.cmml" xref="S2.E1.m1.1.1.1">intervention order </mtext><mrow id="S2.E1.m1.1.1.1.1.1.m1.1.1.cmml" xref="S2.E1.m1.1.1.1.1.1.m1.1.1"><mi mathsize="70%" id="S2.E1.m1.1.1.1.1.1.m1.1.1.2.cmml" xref="S2.E1.m1.1.1.1.1.1.m1.1.1.2">i</mi><mo mathsize="70%" stretchy="false" id="S2.E1.m1.1.1.1.1.1.m1.1.1.1.cmml" xref="S2.E1.m1.1.1.1.1.1.m1.1.1.1">→</mo><mi mathsize="70%" id="S2.E1.m1.1.1.1.1.1.m1.1.1.3.cmml" xref="S2.E1.m1.1.1.1.1.1.m1.1.1.3">j</mi></mrow></mrow></ci></apply><apply id="S2.E1.m1.6.6.1.1.2.2.2.2.2.cmml" xref="S2.E1.m1.6.6.1.1.2.2.2.2.2"><csymbol cd="ambiguous" id="S2.E1.m1.6.6.1.1.2.2.2.2.2.1.cmml" xref="S2.E1.m1.6.6.1.1.2.2.2.2.2">subscript</csymbol><apply id="S2.E1.m1.4.4.cmml" xref="S2.E1.m1.4.4"><ci id="S2.E1.m1.4.4.2.cmml" xref="S2.E1.m1.4.4.2">⏟</ci><apply id="S2.E1.m1.4.4.1.cmml" xref="S2.E1.m1.4.4.1"><times id="S2.E1.m1.4.4.1.2.cmml" xref="S2.E1.m1.4.4.1.2"></times><apply id="S2.E1.m1.4.4.1.3.cmml" xref="S2.E1.m1.4.4.1.3"><csymbol cd="ambiguous" id="S2.E1.m1.4.4.1.3.1.cmml" xref="S2.E1.m1.4.4.1.3">subscript</csymbol><ci id="S2.E1.m1.4.4.1.3.2.cmml" xref="S2.E1.m1.4.4.1.3.2">𝜅</ci><ci id="S2.E1.m1.4.4.1.3.3.cmml" xref="S2.E1.m1.4.4.1.3.3">𝑖</ci></apply><apply id="S2.E1.m1.4.4.1.1.1.1.cmml" xref="S2.E1.m1.4.4.1.1.1"><times id="S2.E1.m1.4.4.1.1.1.1.3.cmml" xref="S2.E1.m1.4.4.1.1.1.1.3"></times><apply id="S2.E1.m1.4.4.1.1.1.1.1.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1"><compose id="S2.E1.m1.4.4.1.1.1.1.1.2.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.2"></compose><apply id="S2.E1.m1.4.4.1.1.1.1.1.1.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1"><times id="S2.E1.m1.4.4.1.1.1.1.1.1.2.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.2"></times><apply id="S2.E1.m1.4.4.1.1.1.1.1.1.3.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.E1.m1.4.4.1.1.1.1.1.1.3.1.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.3">subscript</csymbol><ci id="S2.E1.m1.4.4.1.1.1.1.1.1.3.2.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.3.2">italic-ϕ</ci><ci id="S2.E1.m1.4.4.1.1.1.1.1.1.3.3.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.3.3">𝑖</ci></apply><apply id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.1.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.2.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.2">𝛾</ci><ci id="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.3.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.1.1.1.1.3">𝑖</ci></apply></apply><apply id="S2.E1.m1.4.4.1.1.1.1.1.3.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.E1.m1.4.4.1.1.1.1.1.3.1.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.3">subscript</csymbol><ci id="S2.E1.m1.4.4.1.1.1.1.1.3.2.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.3.2">italic-ϕ</ci><ci id="S2.E1.m1.4.4.1.1.1.1.1.3.3.cmml" xref="S2.E1.m1.4.4.1.1.1.1.1.3.3">𝑗</ci></apply></apply><apply id="S2.E1.m1.4.4.1.1.1.1.2.1.1.cmml" xref="S2.E1.m1.4.4.1.1.1.1.2.1"><csymbol cd="ambiguous" id="S2.E1.m1.4.4.1.1.1.1.2.1.1.1.cmml" xref="S2.E1.m1.4.4.1.1.1.1.2.1">subscript</csymbol><ci id="S2.E1.m1.4.4.1.1.1.1.2.1.1.2.cmml" xref="S2.E1.m1.4.4.1.1.1.1.2.1.1.2">𝑓</ci><ci id="S2.E1.m1.4.4.1.1.1.1.2.1.1.3.cmml" xref="S2.E1.m1.4.4.1.1.1.1.2.1.1.3">𝜃</ci></apply></apply></apply></apply><ci id="S2.E1.m1.2.2.1b.cmml" xref="S2.E1.m1.2.2.1"><mrow id="S2.E1.m1.2.2.1.cmml" xref="S2.E1.m1.2.2.1"><mtext mathsize="70%" id="S2.E1.m1.2.2.1a.cmml" xref="S2.E1.m1.2.2.1">intervention order </mtext><mrow id="S2.E1.m1.2.2.1.1.1.m1.1.1.cmml" xref="S2.E1.m1.2.2.1.1.1.m1.1.1"><mi mathsize="70%" id="S2.E1.m1.2.2.1.1.1.m1.1.1.2.cmml" xref="S2.E1.m1.2.2.1.1.1.m1.1.1.2">j</mi><mo mathsize="70%" stretchy="false" id="S2.E1.m1.2.2.1.1.1.m1.1.1.1.cmml" xref="S2.E1.m1.2.2.1.1.1.m1.1.1.1">→</mo><mi mathsize="70%" id="S2.E1.m1.2.2.1.1.1.m1.1.1.3.cmml" xref="S2.E1.m1.2.2.1.1.1.m1.1.1.3">i</mi></mrow></mrow></ci></apply></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.E1.m1.6c">1-\int_{\gamma_{i}}\min(\underbrace{\kappa_{i}(\phi_{j}\circ\phi_{i}(f_{\theta},\gamma_{i}))}_{\text{intervention order $i\rightarrow j$}},\underbrace{\kappa_{i}(\phi_{i}(\gamma_{i})\circ\phi_{j}(f_{\theta}))}_{\text{intervention order $j\rightarrow i$}}).</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
<td rowspan="1" class="ltx_eqn_cell ltx_eqn_eqno ltx_align_middle ltx_align_right"><span class="ltx_tag ltx_tag_equation ltx_align_right">(1)</span></td>
</tr></tbody>
</table>
<p id="S2.SS2.p4.6" class="ltx_p">A smaller area indicates better composability.
By sweeping over hyperparameters, this measure provides a general sense of composability without choosing hyperparameters ahead of time.
The area can be computed for any criteria <math id="S2.SS2.p4.3.m1.1" class="ltx_Math" alttext="\kappa" display="inline"><semantics id="S2.SS2.p4.3.m1.1a"><mi id="S2.SS2.p4.3.m1.1.1" xref="S2.SS2.p4.3.m1.1.1.cmml">κ</mi><annotation-xml encoding="MathML-Content" id="S2.SS2.p4.3.m1.1b"><ci id="S2.SS2.p4.3.m1.1.1.cmml" xref="S2.SS2.p4.3.m1.1.1">𝜅</ci></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p4.3.m1.1c">\kappa</annotation></semantics></math>—in our experiments, we evaluate all criteria for each composition.
In practice, not all hyperparameters are practical.
For example, harsh model compression can have catastrophic impacts on model utility and so comparing interventions at extreme levels may be unrealistic.
Therefore, we can also set a bound on acceptable performance decay with respect to <math id="S2.SS2.p4.4.m2.1" class="ltx_Math" alttext="\kappa_{i}" display="inline"><semantics id="S2.SS2.p4.4.m2.1a"><msub id="S2.SS2.p4.4.m2.1.1" xref="S2.SS2.p4.4.m2.1.1.cmml"><mi id="S2.SS2.p4.4.m2.1.1.2" xref="S2.SS2.p4.4.m2.1.1.2.cmml">κ</mi><mi id="S2.SS2.p4.4.m2.1.1.3" xref="S2.SS2.p4.4.m2.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p4.4.m2.1b"><apply id="S2.SS2.p4.4.m2.1.1.cmml" xref="S2.SS2.p4.4.m2.1.1"><csymbol cd="ambiguous" id="S2.SS2.p4.4.m2.1.1.1.cmml" xref="S2.SS2.p4.4.m2.1.1">subscript</csymbol><ci id="S2.SS2.p4.4.m2.1.1.2.cmml" xref="S2.SS2.p4.4.m2.1.1.2">𝜅</ci><ci id="S2.SS2.p4.4.m2.1.1.3.cmml" xref="S2.SS2.p4.4.m2.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p4.4.m2.1c">\kappa_{i}</annotation></semantics></math> or any other criterion when using Equation <a href="#S2.E1" title="In 2.2 Composable Interventions for Language Models ‣ 2 An Evaluation Framework for Composable Interventions ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a> and can also easily be measured for hand-chosen hyperparameters, and for a single setting <math id="S2.SS2.p4.5.m3.1" class="ltx_Math" alttext="\gamma_{i}" display="inline"><semantics id="S2.SS2.p4.5.m3.1a"><msub id="S2.SS2.p4.5.m3.1.1" xref="S2.SS2.p4.5.m3.1.1.cmml"><mi id="S2.SS2.p4.5.m3.1.1.2" xref="S2.SS2.p4.5.m3.1.1.2.cmml">γ</mi><mi id="S2.SS2.p4.5.m3.1.1.3" xref="S2.SS2.p4.5.m3.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p4.5.m3.1b"><apply id="S2.SS2.p4.5.m3.1.1.cmml" xref="S2.SS2.p4.5.m3.1.1"><csymbol cd="ambiguous" id="S2.SS2.p4.5.m3.1.1.1.cmml" xref="S2.SS2.p4.5.m3.1.1">subscript</csymbol><ci id="S2.SS2.p4.5.m3.1.1.2.cmml" xref="S2.SS2.p4.5.m3.1.1.2">𝛾</ci><ci id="S2.SS2.p4.5.m3.1.1.3.cmml" xref="S2.SS2.p4.5.m3.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p4.5.m3.1c">\gamma_{i}</annotation></semantics></math>, Order-free Error reduces to the simple error score:
<math id="S2.SS2.p4.6.m4.3" class="ltx_Math" alttext="1-\min(\kappa_{i}(\phi_{j}\circ\phi_{i}(f_{\theta},\gamma_{i})),\kappa_{i}(\phi_{i}(\gamma_{i})\circ\phi_{j}(f_{\theta})))" display="inline"><semantics id="S2.SS2.p4.6.m4.3a"><mrow id="S2.SS2.p4.6.m4.3.3" xref="S2.SS2.p4.6.m4.3.3.cmml"><mn id="S2.SS2.p4.6.m4.3.3.4" xref="S2.SS2.p4.6.m4.3.3.4.cmml">1</mn><mo id="S2.SS2.p4.6.m4.3.3.3" xref="S2.SS2.p4.6.m4.3.3.3.cmml">−</mo><mrow id="S2.SS2.p4.6.m4.3.3.2.2" xref="S2.SS2.p4.6.m4.3.3.2.3.cmml"><mi id="S2.SS2.p4.6.m4.1.1" xref="S2.SS2.p4.6.m4.1.1.cmml">min</mi><mo id="S2.SS2.p4.6.m4.3.3.2.2a" xref="S2.SS2.p4.6.m4.3.3.2.3.cmml">⁡</mo><mrow id="S2.SS2.p4.6.m4.3.3.2.2.2" xref="S2.SS2.p4.6.m4.3.3.2.3.cmml"><mo stretchy="false" id="S2.SS2.p4.6.m4.3.3.2.2.2.3" xref="S2.SS2.p4.6.m4.3.3.2.3.cmml">(</mo><mrow id="S2.SS2.p4.6.m4.2.2.1.1.1.1" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.cmml"><msub id="S2.SS2.p4.6.m4.2.2.1.1.1.1.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.cmml"><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.2.cmml">κ</mi><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.2.cmml">​</mo><mrow id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.cmml">(</mo><mrow id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.cmml"><mrow id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.cmml"><msub id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.cmml"><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.2.cmml">ϕ</mi><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.3.cmml">j</mi></msub><mo lspace="0.222em" rspace="0.222em" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.1" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.1.cmml">∘</mo><msub id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.cmml"><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.2.cmml">ϕ</mi><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.3.cmml">i</mi></msub></mrow><mo lspace="0em" rspace="0em" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.3.cmml">​</mo><mrow id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.3.cmml"><mo stretchy="false" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.3.cmml">(</mo><msub id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.cmml"><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.3.cmml">θ</mi></msub><mo id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.4" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.3.cmml">,</mo><msub id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.cmml"><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.2" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.2.cmml">γ</mi><mi id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.3.cmml">i</mi></msub><mo stretchy="false" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.5" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.3.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.3" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo id="S2.SS2.p4.6.m4.3.3.2.2.2.4" xref="S2.SS2.p4.6.m4.3.3.2.3.cmml">,</mo><mrow id="S2.SS2.p4.6.m4.3.3.2.2.2.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.cmml"><msub id="S2.SS2.p4.6.m4.3.3.2.2.2.2.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.cmml"><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.2.cmml">κ</mi><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.2.cmml">​</mo><mrow id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.cmml">(</mo><mrow id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.cmml"><mrow id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.cmml"><mrow id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.cmml"><msub id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.cmml"><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.2.cmml">ϕ</mi><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.2.cmml">​</mo><mrow id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.cmml">(</mo><msub id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.cmml"><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.2.cmml">γ</mi><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.3.cmml">i</mi></msub><mo rspace="0.055em" stretchy="false" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo rspace="0.222em" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.2.cmml">∘</mo><msub id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.cmml"><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.2.cmml">ϕ</mi><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.3.cmml">j</mi></msub></mrow><mo lspace="0em" rspace="0em" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.3.cmml">​</mo><mrow id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.cmml"><mo stretchy="false" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.cmml">(</mo><msub id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.cmml"><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.2" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.2.cmml">f</mi><mi id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.3" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.SS2.p4.6.m4.3.3.2.2.2.5" xref="S2.SS2.p4.6.m4.3.3.2.3.cmml">)</mo></mrow></mrow></mrow><annotation-xml encoding="MathML-Content" id="S2.SS2.p4.6.m4.3b"><apply id="S2.SS2.p4.6.m4.3.3.cmml" xref="S2.SS2.p4.6.m4.3.3"><minus id="S2.SS2.p4.6.m4.3.3.3.cmml" xref="S2.SS2.p4.6.m4.3.3.3"></minus><cn type="integer" id="S2.SS2.p4.6.m4.3.3.4.cmml" xref="S2.SS2.p4.6.m4.3.3.4">1</cn><apply id="S2.SS2.p4.6.m4.3.3.2.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2"><min id="S2.SS2.p4.6.m4.1.1.cmml" xref="S2.SS2.p4.6.m4.1.1"></min><apply id="S2.SS2.p4.6.m4.2.2.1.1.1.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1"><times id="S2.SS2.p4.6.m4.2.2.1.1.1.1.2.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.2"></times><apply id="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.3">subscript</csymbol><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.2.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.2">𝜅</ci><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.3.3">𝑖</ci></apply><apply id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1"><times id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.3"></times><apply id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4"><compose id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.1"></compose><apply id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2">subscript</csymbol><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.2.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.2">italic-ϕ</ci><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.2.3">𝑗</ci></apply><apply id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3">subscript</csymbol><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.2.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.2">italic-ϕ</ci><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.4.3.3">𝑖</ci></apply></apply><interval closure="open" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2"><apply id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.2.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.2">𝑓</ci><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.1.1.1.3">𝜃</ci></apply><apply id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.1.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2">subscript</csymbol><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.2.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.2">𝛾</ci><ci id="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.3.cmml" xref="S2.SS2.p4.6.m4.2.2.1.1.1.1.1.1.1.2.2.2.3">𝑖</ci></apply></interval></apply></apply><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2"><times id="S2.SS2.p4.6.m4.3.3.2.2.2.2.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.2"></times><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.3"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.3">subscript</csymbol><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.2">𝜅</ci><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.3.3">𝑖</ci></apply><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1"><times id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.3"></times><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1"><compose id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.2"></compose><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1"><times id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.2"></times><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3">subscript</csymbol><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.2">italic-ϕ</ci><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.3.3">𝑖</ci></apply><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.2">𝛾</ci><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.1.1.1.1.3">𝑖</ci></apply></apply><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3">subscript</csymbol><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.2">italic-ϕ</ci><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.1.3.3">𝑗</ci></apply></apply><apply id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1"><csymbol cd="ambiguous" id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.1.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1">subscript</csymbol><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.2.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.2">𝑓</ci><ci id="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.3.cmml" xref="S2.SS2.p4.6.m4.3.3.2.2.2.2.1.1.1.2.1.1.3">𝜃</ci></apply></apply></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p4.6.m4.3c">1-\min(\kappa_{i}(\phi_{j}\circ\phi_{i}(f_{\theta},\gamma_{i})),\kappa_{i}(\phi_{i}(\gamma_{i})\circ\phi_{j}(f_{\theta})))</annotation></semantics></math>.</p>
</div>
<div id="S2.SS2.p5" class="ltx_para ltx_noindent">
<p id="S2.SS2.p5.2" class="ltx_p"><span id="S2.SS2.p5.2.1" class="ltx_text ltx_font_bold">Composability Error Metric 2: Order Sensitivity.</span>
Interventions are more composable if their performance is invariant to their order of application.
We measure <span id="S2.SS2.p5.2.2" class="ltx_text ltx_font_italic">order invariance</span> for criterion <math id="S2.SS2.p5.1.m1.1" class="ltx_Math" alttext="\kappa_{i}" display="inline"><semantics id="S2.SS2.p5.1.m1.1a"><msub id="S2.SS2.p5.1.m1.1.1" xref="S2.SS2.p5.1.m1.1.1.cmml"><mi id="S2.SS2.p5.1.m1.1.1.2" xref="S2.SS2.p5.1.m1.1.1.2.cmml">κ</mi><mi id="S2.SS2.p5.1.m1.1.1.3" xref="S2.SS2.p5.1.m1.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p5.1.m1.1b"><apply id="S2.SS2.p5.1.m1.1.1.cmml" xref="S2.SS2.p5.1.m1.1.1"><csymbol cd="ambiguous" id="S2.SS2.p5.1.m1.1.1.1.cmml" xref="S2.SS2.p5.1.m1.1.1">subscript</csymbol><ci id="S2.SS2.p5.1.m1.1.1.2.cmml" xref="S2.SS2.p5.1.m1.1.1.2">𝜅</ci><ci id="S2.SS2.p5.1.m1.1.1.3.cmml" xref="S2.SS2.p5.1.m1.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p5.1.m1.1c">\kappa_{i}</annotation></semantics></math> as the area <span id="S2.SS2.p5.2.3" class="ltx_text ltx_font_italic">between</span> the two curves defined by each direction of intervention application.
Again, we compute this across hyperparameter choices for <math id="S2.SS2.p5.2.m2.1" class="ltx_Math" alttext="\gamma_{i}" display="inline"><semantics id="S2.SS2.p5.2.m2.1a"><msub id="S2.SS2.p5.2.m2.1.1" xref="S2.SS2.p5.2.m2.1.1.cmml"><mi id="S2.SS2.p5.2.m2.1.1.2" xref="S2.SS2.p5.2.m2.1.1.2.cmml">γ</mi><mi id="S2.SS2.p5.2.m2.1.1.3" xref="S2.SS2.p5.2.m2.1.1.3.cmml">i</mi></msub><annotation-xml encoding="MathML-Content" id="S2.SS2.p5.2.m2.1b"><apply id="S2.SS2.p5.2.m2.1.1.cmml" xref="S2.SS2.p5.2.m2.1.1"><csymbol cd="ambiguous" id="S2.SS2.p5.2.m2.1.1.1.cmml" xref="S2.SS2.p5.2.m2.1.1">subscript</csymbol><ci id="S2.SS2.p5.2.m2.1.1.2.cmml" xref="S2.SS2.p5.2.m2.1.1.2">𝛾</ci><ci id="S2.SS2.p5.2.m2.1.1.3.cmml" xref="S2.SS2.p5.2.m2.1.1.3">𝑖</ci></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.SS2.p5.2.m2.1c">\gamma_{i}</annotation></semantics></math>:</p>
<table id="S2.E2" class="ltx_equation ltx_eqn_table">

<tbody><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_eqn_cell ltx_align_center"><math id="S2.E2.m1.1" class="ltx_Math" alttext="\int_{\gamma_{i}}|\kappa_{i}(\phi_{j}\circ\phi_{i}(f_{\theta},\gamma_{i}))-\kappa_{i}(\phi_{i}(\gamma_{i})\circ\phi_{j}(f_{\theta}))|." display="block"><semantics id="S2.E2.m1.1a"><mrow id="S2.E2.m1.1.1.1" xref="S2.E2.m1.1.1.1.1.cmml"><mrow id="S2.E2.m1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.cmml"><msub id="S2.E2.m1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.2.cmml"><mo id="S2.E2.m1.1.1.1.1.2.2" xref="S2.E2.m1.1.1.1.1.2.2.cmml">∫</mo><msub id="S2.E2.m1.1.1.1.1.2.3" xref="S2.E2.m1.1.1.1.1.2.3.cmml"><mi id="S2.E2.m1.1.1.1.1.2.3.2" xref="S2.E2.m1.1.1.1.1.2.3.2.cmml">γ</mi><mi id="S2.E2.m1.1.1.1.1.2.3.3" xref="S2.E2.m1.1.1.1.1.2.3.3.cmml">i</mi></msub></msub><mrow id="S2.E2.m1.1.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.2.cmml"><mo lspace="0em" stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.2.1.cmml">|</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.cmml"><mrow id="S2.E2.m1.1.1.1.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.1.cmml"><msub id="S2.E2.m1.1.1.1.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.3.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.3.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.3.2.cmml">κ</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.3.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.E2.m1.1.1.1.1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.2.cmml">​</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.cmml">(</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.cmml"><mrow id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.cmml"><msub id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.2.cmml">ϕ</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.3.cmml">j</mi></msub><mo lspace="0.222em" rspace="0.222em" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.1" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.1.cmml">∘</mo><msub id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.2.cmml">ϕ</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.3.cmml">i</mi></msub></mrow><mo lspace="0em" rspace="0em" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.3.cmml">​</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.3.cmml"><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.3.cmml">(</mo><msub id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.2.cmml">f</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.3.cmml">θ</mi></msub><mo id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.4" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.3.cmml">,</mo><msub id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.2" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.2.cmml">γ</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.3.cmml">i</mi></msub><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.5" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.3.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo id="S2.E2.m1.1.1.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.3.cmml">−</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.cmml"><msub id="S2.E2.m1.1.1.1.1.1.1.1.2.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.3.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.3.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.3.2.cmml">κ</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.3.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.E2.m1.1.1.1.1.1.1.1.2.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.2.cmml">​</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.cmml"><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.cmml">(</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.cmml"><mrow id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.cmml"><mrow id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.cmml"><msub id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.2.cmml">ϕ</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.3.cmml">i</mi></msub><mo lspace="0em" rspace="0em" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.2.cmml">​</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.cmml"><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.cmml">(</mo><msub id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.2.cmml">γ</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.3.cmml">i</mi></msub><mo rspace="0.055em" stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.cmml">)</mo></mrow></mrow><mo rspace="0.222em" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.2.cmml">∘</mo><msub id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.2.cmml">ϕ</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.3.cmml">j</mi></msub></mrow><mo lspace="0em" rspace="0em" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.3.cmml">​</mo><mrow id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.cmml"><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.cmml">(</mo><msub id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.cmml"><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.2" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.2.cmml">f</mi><mi id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.3.cmml">θ</mi></msub><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.cmml">)</mo></mrow></mrow><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.3" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.cmml">)</mo></mrow></mrow></mrow><mo stretchy="false" id="S2.E2.m1.1.1.1.1.1.1.3" xref="S2.E2.m1.1.1.1.1.1.2.1.cmml">|</mo></mrow></mrow><mo lspace="0em" id="S2.E2.m1.1.1.1.2" xref="S2.E2.m1.1.1.1.1.cmml">.</mo></mrow><annotation-xml encoding="MathML-Content" id="S2.E2.m1.1b"><apply id="S2.E2.m1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1"><apply id="S2.E2.m1.1.1.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.2"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.2.1.cmml" xref="S2.E2.m1.1.1.1.1.2">subscript</csymbol><int id="S2.E2.m1.1.1.1.1.2.2.cmml" xref="S2.E2.m1.1.1.1.1.2.2"></int><apply id="S2.E2.m1.1.1.1.1.2.3.cmml" xref="S2.E2.m1.1.1.1.1.2.3"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.2.3.1.cmml" xref="S2.E2.m1.1.1.1.1.2.3">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.2.3.2.cmml" xref="S2.E2.m1.1.1.1.1.2.3.2">𝛾</ci><ci id="S2.E2.m1.1.1.1.1.2.3.3.cmml" xref="S2.E2.m1.1.1.1.1.2.3.3">𝑖</ci></apply></apply><apply id="S2.E2.m1.1.1.1.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1"><abs id="S2.E2.m1.1.1.1.1.1.2.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.2"></abs><apply id="S2.E2.m1.1.1.1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1"><minus id="S2.E2.m1.1.1.1.1.1.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.3"></minus><apply id="S2.E2.m1.1.1.1.1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1"><times id="S2.E2.m1.1.1.1.1.1.1.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.2"></times><apply id="S2.E2.m1.1.1.1.1.1.1.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.1.3.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.3">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.3.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.3.2">𝜅</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.3.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.3.3">𝑖</ci></apply><apply id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1"><times id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.3"></times><apply id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4"><compose id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.1"></compose><apply id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.2">italic-ϕ</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.2.3">𝑗</ci></apply><apply id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.2">italic-ϕ</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.4.3.3">𝑖</ci></apply></apply><interval closure="open" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2"><apply id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.2">𝑓</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.1.1.1.3">𝜃</ci></apply><apply id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.2">𝛾</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.1.1.1.1.2.2.2.3">𝑖</ci></apply></interval></apply></apply><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2"><times id="S2.E2.m1.1.1.1.1.1.1.1.2.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.2"></times><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.3"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.2.3.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.3">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.3.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.3.2">𝜅</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.3.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.3.3">𝑖</ci></apply><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1"><times id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.3"></times><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1"><compose id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.2"></compose><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1"><times id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.2"></times><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.2">italic-ϕ</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.3.3">𝑖</ci></apply><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.2">𝛾</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.1.1.1.1.3">𝑖</ci></apply></apply><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.2">italic-ϕ</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.1.3.3">𝑗</ci></apply></apply><apply id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1"><csymbol cd="ambiguous" id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.1.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1">subscript</csymbol><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.2.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.2">𝑓</ci><ci id="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.3.cmml" xref="S2.E2.m1.1.1.1.1.1.1.1.2.1.1.1.2.1.1.3">𝜃</ci></apply></apply></apply></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="S2.E2.m1.1c">\int_{\gamma_{i}}|\kappa_{i}(\phi_{j}\circ\phi_{i}(f_{\theta},\gamma_{i}))-\kappa_{i}(\phi_{i}(\gamma_{i})\circ\phi_{j}(f_{\theta}))|.</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
<td rowspan="1" class="ltx_eqn_cell ltx_eqn_eqno ltx_align_middle ltx_align_right"><span class="ltx_tag ltx_tag_equation ltx_align_right">(2)</span></td>
</tr></tbody>
</table>
<p id="S2.SS2.p5.3" class="ltx_p">A smaller area again indicates better composability.
And again, if hyperparameters are already chosen, this simplifies to a simple error: <math id="S2.SS2.p5.3.m1.1" class="ltx_math_unparsed" alttext="|\kappa_{i}(\phi_{j}(\phi_{i}(f_{\theta})))-\kappa_{i}(\phi_{i}(\gamma_{i})\circ\phi_{j}(f_{\theta})))|" display="inline"><semantics id="S2.SS2.p5.3.m1.1a"><mrow id="S2.SS2.p5.3.m1.1b"><mo fence="false" rspace="0.167em" stretchy="false" id="S2.SS2.p5.3.m1.1.1">|</mo><msub id="S2.SS2.p5.3.m1.1.2"><mi id="S2.SS2.p5.3.m1.1.2.2">κ</mi><mi id="S2.SS2.p5.3.m1.1.2.3">i</mi></msub><mrow id="S2.SS2.p5.3.m1.1.3"><mo stretchy="false" id="S2.SS2.p5.3.m1.1.3.1">(</mo><msub id="S2.SS2.p5.3.m1.1.3.2"><mi id="S2.SS2.p5.3.m1.1.3.2.2">ϕ</mi><mi id="S2.SS2.p5.3.m1.1.3.2.3">j</mi></msub><mrow id="S2.SS2.p5.3.m1.1.3.3"><mo stretchy="false" id="S2.SS2.p5.3.m1.1.3.3.1">(</mo><msub id="S2.SS2.p5.3.m1.1.3.3.2"><mi id="S2.SS2.p5.3.m1.1.3.3.2.2">ϕ</mi><mi id="S2.SS2.p5.3.m1.1.3.3.2.3">i</mi></msub><mrow id="S2.SS2.p5.3.m1.1.3.3.3"><mo stretchy="false" id="S2.SS2.p5.3.m1.1.3.3.3.1">(</mo><msub id="S2.SS2.p5.3.m1.1.3.3.3.2"><mi id="S2.SS2.p5.3.m1.1.3.3.3.2.2">f</mi><mi id="S2.SS2.p5.3.m1.1.3.3.3.2.3">θ</mi></msub><mo stretchy="false" id="S2.SS2.p5.3.m1.1.3.3.3.3">)</mo></mrow><mo stretchy="false" id="S2.SS2.p5.3.m1.1.3.3.4">)</mo></mrow><mo stretchy="false" id="S2.SS2.p5.3.m1.1.3.4">)</mo></mrow><mo id="S2.SS2.p5.3.m1.1.4">−</mo><msub id="S2.SS2.p5.3.m1.1.5"><mi id="S2.SS2.p5.3.m1.1.5.2">κ</mi><mi id="S2.SS2.p5.3.m1.1.5.3">i</mi></msub><mrow id="S2.SS2.p5.3.m1.1.6"><mo stretchy="false" id="S2.SS2.p5.3.m1.1.6.1">(</mo><msub id="S2.SS2.p5.3.m1.1.6.2"><mi id="S2.SS2.p5.3.m1.1.6.2.2">ϕ</mi><mi id="S2.SS2.p5.3.m1.1.6.2.3">i</mi></msub><mrow id="S2.SS2.p5.3.m1.1.6.3"><mo stretchy="false" id="S2.SS2.p5.3.m1.1.6.3.1">(</mo><msub id="S2.SS2.p5.3.m1.1.6.3.2"><mi id="S2.SS2.p5.3.m1.1.6.3.2.2">γ</mi><mi id="S2.SS2.p5.3.m1.1.6.3.2.3">i</mi></msub><mo rspace="0.055em" stretchy="false" id="S2.SS2.p5.3.m1.1.6.3.3">)</mo></mrow><mo rspace="0.222em" id="S2.SS2.p5.3.m1.1.6.4">∘</mo><msub id="S2.SS2.p5.3.m1.1.6.5"><mi id="S2.SS2.p5.3.m1.1.6.5.2">ϕ</mi><mi id="S2.SS2.p5.3.m1.1.6.5.3">j</mi></msub><mrow id="S2.SS2.p5.3.m1.1.6.6"><mo stretchy="false" id="S2.SS2.p5.3.m1.1.6.6.1">(</mo><msub id="S2.SS2.p5.3.m1.1.6.6.2"><mi id="S2.SS2.p5.3.m1.1.6.6.2.2">f</mi><mi id="S2.SS2.p5.3.m1.1.6.6.2.3">θ</mi></msub><mo stretchy="false" id="S2.SS2.p5.3.m1.1.6.6.3">)</mo></mrow><mo stretchy="false" id="S2.SS2.p5.3.m1.1.6.7">)</mo></mrow><mo stretchy="false" id="S2.SS2.p5.3.m1.1.7">)</mo><mo fence="false" stretchy="false" id="S2.SS2.p5.3.m1.1.8">|</mo></mrow><annotation encoding="application/x-tex" id="S2.SS2.p5.3.m1.1c">|\kappa_{i}(\phi_{j}(\phi_{i}(f_{\theta})))-\kappa_{i}(\phi_{i}(\gamma_{i})\circ\phi_{j}(f_{\theta})))|</annotation></semantics></math>.</p>
</div>
<div id="S2.SS2.p6" class="ltx_para">
<p id="S2.SS2.p6.1" class="ltx_p">Overall, measuring Order-free Error and Order Sensitivity provides the first insights into intervention composability.
Each reduces to a simple form for point measures of the impact of composition.
While more computationally costly, the broader sweeps over hyperparameters enable clearer pictures of how interventions interact.
As shown in Section <a href="#S3" title="3 Experimental Setup ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3</span></a>, the curves produced by these sweeps also enable intuitive, visual understandings of composability.</p>
</div>
</section>
</section>
<section id="S3" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">3 </span>Experimental Setup</h2>

<div id="S3.p1" class="ltx_para">
<p id="S3.p1.1" class="ltx_p">We study the effects of sequentially applying multiple interventions to the same model on performance as measured by intervention-specific metrics and overall utility. Understanding these effects is crucial for the practical application of test-time interventions. The following sections detail our experiments to illuminate how composable various interventions are across model editing, unlearning, and compression.</p>
</div>
<section id="S3.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">3.1 </span>Intervention Datasets, Methods, and Metrics</h3>

<div id="S3.SS1.p1" class="ltx_para">
<p id="S3.SS1.p1.1" class="ltx_p">We next detail the ten techniques selected from the three categories of interventions, along with their task-specific and overall utility metrics. Appendix <a href="#A2" title="Appendix B Background on Interventions ‣ Acknowledgements ‣ 5 Conclusion ‣ 4.4 General Trends in Composability ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">B</span></a> further describes the backgrounds of the three types of intervention and Table <a href="#S3.T1" title="Table 1 ‣ 3.1 Intervention Datasets, Methods, and Metrics ‣ 3 Experimental Setup ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a> summarizes our experimental setup.</p>
</div>
<figure id="S3.T1" class="ltx_table">
<div id="S3.T1.2" class="ltx_inline-block ltx_align_center ltx_transformed_outer" style="width:433.6pt;height:95pt;vertical-align:-1.0pt;"><span class="ltx_transformed_inner" style="transform:translate(-254.0pt,55.1pt) scale(0.460467022641742,0.460467022641742) ;">
<table id="S3.T1.2.1" class="ltx_tabular ltx_align_middle">
<tr id="S3.T1.2.1.1" class="ltx_tr">
<td id="S3.T1.2.1.1.1" class="ltx_td ltx_align_left ltx_border_tt"><span id="S3.T1.2.1.1.1.1" class="ltx_text ltx_font_smallcaps">Interventions</span></td>
<td id="S3.T1.2.1.1.2" class="ltx_td ltx_align_center ltx_border_tt"><span id="S3.T1.2.1.1.2.1" class="ltx_text ltx_font_smallcaps">Methods</span></td>
<td id="S3.T1.2.1.1.3" class="ltx_td ltx_align_center ltx_border_tt"><span id="S3.T1.2.1.1.3.1" class="ltx_text ltx_font_smallcaps">Datasets</span></td>
<td id="S3.T1.2.1.1.4" class="ltx_td ltx_align_center ltx_border_tt">
<span id="S3.T1.2.1.1.4.1" class="ltx_text"></span><span id="S3.T1.2.1.1.4.2" class="ltx_text ltx_font_smallcaps"> <span id="S3.T1.2.1.1.4.2.1" class="ltx_text">
<span id="S3.T1.2.1.1.4.2.1.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.1.4.2.1.1.1" class="ltx_tr">
<span id="S3.T1.2.1.1.4.2.1.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Intervention</span></span>
<span id="S3.T1.2.1.1.4.2.1.1.2" class="ltx_tr">
<span id="S3.T1.2.1.1.4.2.1.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">Metrics</span></span>
</span></span><span id="S3.T1.2.1.1.4.2.2" class="ltx_text"></span></span>
</td>
<td id="S3.T1.2.1.1.5" class="ltx_td ltx_align_center ltx_border_tt">
<span id="S3.T1.2.1.1.5.1" class="ltx_text"></span><span id="S3.T1.2.1.1.5.2" class="ltx_text ltx_font_smallcaps"> <span id="S3.T1.2.1.1.5.2.1" class="ltx_text">
<span id="S3.T1.2.1.1.5.2.1.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.1.5.2.1.1.1" class="ltx_tr">
<span id="S3.T1.2.1.1.5.2.1.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Composability</span></span>
<span id="S3.T1.2.1.1.5.2.1.1.2" class="ltx_tr">
<span id="S3.T1.2.1.1.5.2.1.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">Metrics</span></span>
</span></span><span id="S3.T1.2.1.1.5.2.2" class="ltx_text"></span></span>
</td>
</tr>
<tr id="S3.T1.2.1.2" class="ltx_tr">
<td id="S3.T1.2.1.2.1" class="ltx_td ltx_align_left ltx_border_t">
<span id="S3.T1.2.1.2.1.1" class="ltx_text"></span><span id="S3.T1.2.1.2.1.2" class="ltx_text ltx_font_bold"> <span id="S3.T1.2.1.2.1.2.1" class="ltx_text">
<span id="S3.T1.2.1.2.1.2.1.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.2.1.2.1.1.1" class="ltx_tr">
<span id="S3.T1.2.1.2.1.2.1.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Knowledge</span></span>
<span id="S3.T1.2.1.2.1.2.1.1.2" class="ltx_tr">
<span id="S3.T1.2.1.2.1.2.1.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">Editing</span></span>
</span></span><span id="S3.T1.2.1.2.1.2.2" class="ltx_text"></span></span>
</td>
<td id="S3.T1.2.1.2.2" class="ltx_td ltx_align_center ltx_border_t">
<span id="S3.T1.2.1.2.2.1" class="ltx_text"></span> <span id="S3.T1.2.1.2.2.2" class="ltx_text">
<span id="S3.T1.2.1.2.2.2.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.2.2.2.1.1" class="ltx_tr">
<span id="S3.T1.2.1.2.2.2.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Finetuning, LoRA <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib34" title="" class="ltx_ref">34</a>]</cite>,</span></span>
<span id="S3.T1.2.1.2.2.2.1.2" class="ltx_tr">
<span id="S3.T1.2.1.2.2.2.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">MEMIT <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib20" title="" class="ltx_ref">20</a>]</cite></span></span>
</span></span><span id="S3.T1.2.1.2.2.3" class="ltx_text"></span></td>
<td id="S3.T1.2.1.2.3" class="ltx_td ltx_align_center ltx_border_t">zsRE <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib35" title="" class="ltx_ref">35</a>]</cite>
</td>
<td id="S3.T1.2.1.2.4" class="ltx_td ltx_align_center ltx_border_t">
<span id="S3.T1.2.1.2.4.1" class="ltx_text"></span> <span id="S3.T1.2.1.2.4.2" class="ltx_text">
<span id="S3.T1.2.1.2.4.2.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.2.4.2.1.1" class="ltx_tr">
<span id="S3.T1.2.1.2.4.2.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Edit Success, Edit Generalization,</span></span>
<span id="S3.T1.2.1.2.4.2.1.2" class="ltx_tr">
<span id="S3.T1.2.1.2.4.2.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">Strict Edit Locality, MMLU</span></span>
</span></span><span id="S3.T1.2.1.2.4.3" class="ltx_text"></span></td>
<td id="S3.T1.2.1.2.5" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t" rowspan="6"><span id="S3.T1.2.1.2.5.1" class="ltx_text"><span id="S3.T1.2.1.2.5.1.1" class="ltx_text"></span> <span id="S3.T1.2.1.2.5.1.2" class="ltx_text">
<span id="S3.T1.2.1.2.5.1.2.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.2.5.1.2.1.1" class="ltx_tr">
<span id="S3.T1.2.1.2.5.1.2.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Order-free Error</span></span>
<span id="S3.T1.2.1.2.5.1.2.1.2" class="ltx_tr">
<span id="S3.T1.2.1.2.5.1.2.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">(Equation <a href="#S2.E1" title="In 2.2 Composable Interventions for Language Models ‣ 2 An Evaluation Framework for Composable Interventions ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">1</span></a>)</span></span>
<span id="S3.T1.2.1.2.5.1.2.1.3" class="ltx_tr">
<span id="S3.T1.2.1.2.5.1.2.1.3.1" class="ltx_td ltx_nopad_r ltx_align_center">Order Sensitivity</span></span>
<span id="S3.T1.2.1.2.5.1.2.1.4" class="ltx_tr">
<span id="S3.T1.2.1.2.5.1.2.1.4.1" class="ltx_td ltx_nopad_r ltx_align_center">(Equation <a href="#S2.E2" title="In 2.2 Composable Interventions for Language Models ‣ 2 An Evaluation Framework for Composable Interventions ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>)</span></span>
</span></span> <span id="S3.T1.2.1.2.5.1.3" class="ltx_text"></span></span></td>
</tr>
<tr id="S3.T1.2.1.3" class="ltx_tr">
<td id="S3.T1.2.1.3.1" class="ltx_td ltx_align_left ltx_border_t">
<span id="S3.T1.2.1.3.1.1" class="ltx_text"></span><span id="S3.T1.2.1.3.1.2" class="ltx_text ltx_font_bold"> <span id="S3.T1.2.1.3.1.2.1" class="ltx_text">
<span id="S3.T1.2.1.3.1.2.1.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.3.1.2.1.1.1" class="ltx_tr">
<span id="S3.T1.2.1.3.1.2.1.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Model</span></span>
<span id="S3.T1.2.1.3.1.2.1.1.2" class="ltx_tr">
<span id="S3.T1.2.1.3.1.2.1.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">Compression</span></span>
</span></span><span id="S3.T1.2.1.3.1.2.2" class="ltx_text"></span></span>
</td>
<td id="S3.T1.2.1.3.2" class="ltx_td ltx_align_center ltx_border_t">
<span id="S3.T1.2.1.3.2.1" class="ltx_text"></span> <span id="S3.T1.2.1.3.2.2" class="ltx_text">
<span id="S3.T1.2.1.3.2.2.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.3.2.2.1.1" class="ltx_tr">
<span id="S3.T1.2.1.3.2.2.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center"><span id="S3.T1.2.1.3.2.2.1.1.1.1" class="ltx_text ltx_font_italic">Pruning:</span> Wanda <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib36" title="" class="ltx_ref">36</a>]</cite>, SparseGPT <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib16" title="" class="ltx_ref">16</a>]</cite></span></span>
<span id="S3.T1.2.1.3.2.2.1.2" class="ltx_tr">
<span id="S3.T1.2.1.3.2.2.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center"><span id="S3.T1.2.1.3.2.2.1.2.1.1" class="ltx_text ltx_font_italic">Quantization:</span> GPTQ <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib15" title="" class="ltx_ref">15</a>]</cite>, AWQ <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib37" title="" class="ltx_ref">37</a>]</cite></span></span>
</span></span><span id="S3.T1.2.1.3.2.3" class="ltx_text"></span></td>
<td id="S3.T1.2.1.3.3" class="ltx_td ltx_align_center ltx_border_t">–</td>
<td id="S3.T1.2.1.3.4" class="ltx_td ltx_align_center ltx_border_t">MMLU</td>
</tr>
<tr id="S3.T1.2.1.4" class="ltx_tr">
<td id="S3.T1.2.1.4.1" class="ltx_td ltx_align_left ltx_border_bb ltx_border_t">
<span id="S3.T1.2.1.4.1.1" class="ltx_text"></span><span id="S3.T1.2.1.4.1.2" class="ltx_text ltx_font_bold"> <span id="S3.T1.2.1.4.1.2.1" class="ltx_text">
<span id="S3.T1.2.1.4.1.2.1.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.4.1.2.1.1.1" class="ltx_tr">
<span id="S3.T1.2.1.4.1.2.1.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Machine</span></span>
<span id="S3.T1.2.1.4.1.2.1.1.2" class="ltx_tr">
<span id="S3.T1.2.1.4.1.2.1.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">Unlearning</span></span>
</span></span><span id="S3.T1.2.1.4.1.2.2" class="ltx_text"></span></span>
</td>
<td id="S3.T1.2.1.4.2" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">
<span id="S3.T1.2.1.4.2.1" class="ltx_text"></span> <span id="S3.T1.2.1.4.2.2" class="ltx_text">
<span id="S3.T1.2.1.4.2.2.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.4.2.2.1.1" class="ltx_tr">
<span id="S3.T1.2.1.4.2.2.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">Gradient Ascent (GA) <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib38" title="" class="ltx_ref">38</a>]</cite>,</span></span>
<span id="S3.T1.2.1.4.2.2.1.2" class="ltx_tr">
<span id="S3.T1.2.1.4.2.2.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">Gradient Difference (GD) <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib39" title="" class="ltx_ref">39</a>]</cite>,</span></span>
<span id="S3.T1.2.1.4.2.2.1.3" class="ltx_tr">
<span id="S3.T1.2.1.4.2.2.1.3.1" class="ltx_td ltx_nopad_r ltx_align_center">Representation Misdirection</span></span>
<span id="S3.T1.2.1.4.2.2.1.4" class="ltx_tr">
<span id="S3.T1.2.1.4.2.2.1.4.1" class="ltx_td ltx_nopad_r ltx_align_center">Unlearning (RMU) <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib40" title="" class="ltx_ref">40</a>]</cite></span></span>
</span></span><span id="S3.T1.2.1.4.2.3" class="ltx_text"></span></td>
<td id="S3.T1.2.1.4.3" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">WMDP <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib40" title="" class="ltx_ref">40</a>]</cite>
</td>
<td id="S3.T1.2.1.4.4" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">
<span id="S3.T1.2.1.4.4.1" class="ltx_text"></span> <span id="S3.T1.2.1.4.4.2" class="ltx_text">
<span id="S3.T1.2.1.4.4.2.1" class="ltx_tabular ltx_align_middle">
<span id="S3.T1.2.1.4.4.2.1.1" class="ltx_tr">
<span id="S3.T1.2.1.4.4.2.1.1.1" class="ltx_td ltx_nopad_r ltx_align_center">WMDP,</span></span>
<span id="S3.T1.2.1.4.4.2.1.2" class="ltx_tr">
<span id="S3.T1.2.1.4.4.2.1.2.1" class="ltx_td ltx_nopad_r ltx_align_center">MMLU</span></span>
</span></span><span id="S3.T1.2.1.4.4.3" class="ltx_text"></span></td>
</tr>
</table>
</span></div>
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_table"><span id="S3.T1.3.1.1" class="ltx_text" style="font-size:90%;">Table 1</span>: </span><span id="S3.T1.4.2" class="ltx_text" style="font-size:90%;">Summary of the methods, data, and metrics used in our experiments. All experiments are conducted with Llama3-8B. Composability metrics apply to all experiments.</span></figcaption>
</figure>
<div id="S3.SS1.p2" class="ltx_para">
<p id="S3.SS1.p2.1" class="ltx_p"><span id="S3.SS1.p2.1.1" class="ltx_text ltx_font_bold">Knowledge Editing.</span>
LMs’ knowledge can be fundamentally incorrect or grow outdated. Knowledge editing aims to update <span id="S3.SS1.p2.1.2" class="ltx_text ltx_font_italic">facts</span> according to LMs, typically framed as question–answering.
Here, where models are edited to return updated answers to questions by increasing the likelihood of the correct answers.
We use three editors, including the popular MEMIT <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib20" title="" class="ltx_ref">20</a>]</cite>, as well as LoRA <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib34" title="" class="ltx_ref">34</a>]</cite> and finetuning, which have recently been shown to be surprisingly successful editors despite their simplicity <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib41" title="" class="ltx_ref">41</a>, <a href="#bib.bib42" title="" class="ltx_ref">42</a>, <a href="#bib.bib43" title="" class="ltx_ref">43</a>]</cite>.
We use the zsRE <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib35" title="" class="ltx_ref">35</a>]</cite> dataset, which is a popular question-answering benchmark for knowledge editing.</p>
</div>
<div id="S3.SS1.p3" class="ltx_para">
<p id="S3.SS1.p3.1" class="ltx_p">To evaluate editors’ performance, we use three standard metrics <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib44" title="" class="ltx_ref">44</a>]</cite>: <span id="S3.SS1.p3.1.1" class="ltx_text ltx_font_italic">Edit success</span> measures whether the post-edit model successfully outputs the correct, edited response. <span id="S3.SS1.p3.1.2" class="ltx_text ltx_font_italic">Edit generalization</span> measures how well the edit generalizes to other semantically-equivalent inputs. Following recent work, we compute generalization as the average edit success across a set of 10 holdout edit rephrasings.
<span id="S3.SS1.p3.1.3" class="ltx_text ltx_font_italic">Strict Edit locality</span> measures the impact of edits on unrelated inputs as the average success on random, unedited samples.
All editing metrics are computed as the F1 score on the correct output logits as a strict measure of editing performance—intended outputs must be the most-likely tokens after editing.
Each metric’s highest possible value is 1.0.</p>
</div>
<div id="S3.SS1.p4" class="ltx_para">
<p id="S3.SS1.p4.1" class="ltx_p"><span id="S3.SS1.p4.1.1" class="ltx_text ltx_font_bold">Machine Unlearning.</span>
LMs can learn undesirable knowledge from their pretraining data, including information that is potentially dangerous <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib40" title="" class="ltx_ref">40</a>]</cite>
copyrighted <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib45" title="" class="ltx_ref">45</a>, <a href="#bib.bib46" title="" class="ltx_ref">46</a>]</cite>,
toxic <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib47" title="" class="ltx_ref">47</a>, <a href="#bib.bib48" title="" class="ltx_ref">48</a>, <a href="#bib.bib9" title="" class="ltx_ref">9</a>]</cite>,
or memorized <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib49" title="" class="ltx_ref">49</a>, <a href="#bib.bib50" title="" class="ltx_ref">50</a>, <a href="#bib.bib51" title="" class="ltx_ref">51</a>, <a href="#bib.bib52" title="" class="ltx_ref">52</a>]</cite>.
Machine unlearning methods aim to remove undesirable knowledge from LMs without regressing performance on extraneous domains <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib53" title="" class="ltx_ref">53</a>, <a href="#bib.bib27" title="" class="ltx_ref">27</a>]</cite>. We experiment with three popular and representative methods: Gradient Ascent (GA) <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib38" title="" class="ltx_ref">38</a>]</cite>, Gradient Difference (GD) <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib39" title="" class="ltx_ref">39</a>]</cite>, and Representation Misdirection for Unlearning (RMU) <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib40" title="" class="ltx_ref">40</a>]</cite>.
GA is a traditional approach that minimizes the likelihood of chosen unlearning targets.
GD augments GA by also maintaining the LM’s predictions on unrelated retention data.
RMU, the most recent method, perturbs the model’s representations for inputs related to the learning target.
Further details are in Appendix <a href="#A3.SS4" title="C.4 Machine Unlearning Details ‣ Appendix C Implementation Details ‣ Acknowledgements ‣ 5 Conclusion ‣ 4.4 General Trends in Composability ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">C.4</span></a>.</p>
</div>
<div id="S3.SS1.p5" class="ltx_para">
<p id="S3.SS1.p5.1" class="ltx_p">We evaluate unlearning with the recent QA dataset Weapons of Mass Description Proxy (WMDP) <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib40" title="" class="ltx_ref">40</a>]</cite>, updating LMs to forget potentially hazardous biosecurity and cybersecurity knowledge.
We average the performance on WMDP’s cyber and bio splits, totaling 3,260 questions. We report individual split performance in Appendix <a href="#A4" title="Appendix D Extended Results ‣ Acknowledgements ‣ 5 Conclusion ‣ 4.4 General Trends in Composability ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">D</span></a>.
Optimal unlearning yields 25% accuracy as this is random performance on the four-choice benchmark.</p>
</div>
<div id="S3.SS1.p6" class="ltx_para">
<p id="S3.SS1.p6.1" class="ltx_p"><span id="S3.SS1.p6.1.1" class="ltx_text ltx_font_bold">Model Compression.</span> LMs require significant compute resources during pretraining, finetuning, and inference.
Model compression methods aim to reduce resources needed at test-time.
We experiment with four popular, state-of-the-art weight pruning and quantization methods.
For pruning, we choose SparseGPT <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib16" title="" class="ltx_ref">16</a>]</cite> and Wanda <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib36" title="" class="ltx_ref">36</a>]</cite>.
For quantization, we choose GPTQ <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib15" title="" class="ltx_ref">15</a>]</cite> and AWQ <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib37" title="" class="ltx_ref">37</a>]</cite>.
Pruning and quantization are used for different reasons, so we generally avoid comparisons between the two approaches.
Each algorithm has a hyperparameter that controls how much to compress the model—% of zeroed weights for pruning, <span id="S3.SS1.p6.1.2" class="ltx_text ltx_font_italic">bits</span> for quantization.
Naturally, efficiency grows alongside compression, but at the expense of task performance.
We vary this hyperparameter to explore the impacts of various levels of compression, choosing sparsity levels of .25, .35, .45, .55, .65, and .75 for pruning and 2, 3, 4, and 8 bits for quantization.
The uncompressed model is 16 bits.
Editing or unlearning some compressed models requires decompression, in which case we recompress afterwards using the same compression technique.</p>
</div>
<div id="S3.SS1.p7" class="ltx_para">
<p id="S3.SS1.p7.1" class="ltx_p"><span id="S3.SS1.p7.1.1" class="ltx_text ltx_font_bold">Measuring General Model Utility.</span> All interventions aim to avoid degrading overall model utility.
While assessing general performance of LMs is an active area <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib54" title="" class="ltx_ref">54</a>, <a href="#bib.bib55" title="" class="ltx_ref">55</a>, <a href="#bib.bib56" title="" class="ltx_ref">56</a>]</cite>, we make the standard choice to evaluate question–answering accuracy on MMLU <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib57" title="" class="ltx_ref">57</a>]</cite> using the LM Eval Harness <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib58" title="" class="ltx_ref">58</a>]</cite>.
Accuracy of 25% indicates random predictions.</p>
</div>
</section>
<section id="S3.SS2" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">3.2 </span>Implementation details</h3>

<div id="S3.SS2.p1" class="ltx_para">
<p id="S3.SS2.p1.1" class="ltx_p">All experiments are performed with Llama3-8B <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib59" title="" class="ltx_ref">59</a>]</cite>, which is well-studied and highly capable for its parameter count.
All results for editing methods are averaged over 10 batches of 50 random edits from zsRE.
Further details on implementations and hyperparameters are in Appendix <a href="#A3" title="Appendix C Implementation Details ‣ Acknowledgements ‣ 5 Conclusion ‣ 4.4 General Trends in Composability ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">C</span></a>.</p>
</div>
</section>
</section>
<section id="S4" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">4 </span>Results</h2>

<div id="S4.p1" class="ltx_para">
<p id="S4.p1.1" class="ltx_p">In the following subsections, we first examine how each pair of interventions compose individually, then summarize general trends.
We measure composability using our metrics from Section <a href="#S2" title="2 An Evaluation Framework for Composable Interventions ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a> for each intervention’s criteria and report impacts of compression at different levels.
All intervention pairs are applied in both directions (<span id="S4.p1.1.1" class="ltx_text ltx_font_italic">e.g.,</span> MEMIT before AWQ and AWQ before MEMIT).
For composability metrics, we compare methods regardless of metric ranges by counting the number of times each intervention method outperforms the others (# Wins) with ties counting as losses.</p>
</div>
<section id="S4.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">4.1 </span>Composing Model Compression with Knowledge Editing</h3>

<figure id="S4.F2" class="ltx_figure">
<div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="S4.F2.sf1" class="ltx_figure ltx_figure_panel ltx_align_center"><img src="/html/2407.06483/assets/x2.png" id="S4.F2.sf1.g1" class="ltx_graphics ltx_img_square" width="461" height="377" alt="Refer to caption">
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_figure"><span id="S4.F2.sf1.2.1.1" class="ltx_text" style="font-size:90%;">(a)</span> </span></figcaption>
</figure>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="S4.F2.sf2" class="ltx_figure ltx_figure_panel ltx_align_center"><img src="/html/2407.06483/assets/x3.png" id="S4.F2.sf2.g1" class="ltx_graphics ltx_img_landscape" width="253" height="20" alt="Refer to caption">
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_figure"><span id="S4.F2.sf2.2.1.1" class="ltx_text" style="font-size:90%;">(b)</span> </span></figcaption>
</figure>
</div>
</div>
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure"><span id="S4.F2.3.1.1" class="ltx_text" style="font-size:90%;">Figure 2</span>: </span><span id="S4.F2.4.2" class="ltx_text" style="font-size:90%;">Composing Knowledge Editing with Model Compression with varying degrees of compression.
Higher values are better for all metrics.
<span id="S4.F2.4.2.1" class="ltx_text ltx_font_bold">Key Takeaways:</span> Editing post-compression generally outperforms the reverse order, as compression tends to degrade edit performance and all editors exhibit order sensitivity.
</span></figcaption>
</figure>
<figure id="S4.SS1.14" class="ltx_table">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="S4.SS1.4.4" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:122.8pt;vertical-align:-1.7pt;"><span class="ltx_transformed_inner" style="transform:translate(-62.8pt,17.5pt) scale(0.775455121845862,0.775455121845862) ;">
<table id="S4.SS1.4.4.4" class="ltx_tabular ltx_align_middle">
<tr id="S4.SS1.4.4.4.5" class="ltx_tr">
<td id="S4.SS1.4.4.4.5.1" class="ltx_td ltx_border_tt"></td>
<td id="S4.SS1.4.4.4.5.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.SS1.4.4.4.5.2.1" class="ltx_text ltx_font_bold">Edit Success</span></td>
<td id="S4.SS1.4.4.4.5.3" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.SS1.4.4.4.5.3.1" class="ltx_text ltx_font_bold">Edit Generalization</span></td>
<td id="S4.SS1.4.4.4.5.4" class="ltx_td ltx_border_tt"></td>
</tr>
<tr id="S4.SS1.4.4.4.4" class="ltx_tr">
<td id="S4.SS1.4.4.4.4.5" class="ltx_td"></td>
<td id="S4.SS1.1.1.1.1.1" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS1.1.1.1.1.1.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.SS1.1.1.1.1.1.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.1.1.1.1.1.1.m1.1a"><mo stretchy="false" id="S4.SS1.1.1.1.1.1.1.m1.1.1" xref="S4.SS1.1.1.1.1.1.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.1.1.1.1.1.1.m1.1b"><ci id="S4.SS1.1.1.1.1.1.1.m1.1.1.cmml" xref="S4.SS1.1.1.1.1.1.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.1.1.1.1.1.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS1.2.2.2.2.2" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS1.2.2.2.2.2.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.SS1.2.2.2.2.2.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.2.2.2.2.2.1.m1.1a"><mo stretchy="false" id="S4.SS1.2.2.2.2.2.1.m1.1.1" xref="S4.SS1.2.2.2.2.2.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.2.2.2.2.2.1.m1.1b"><ci id="S4.SS1.2.2.2.2.2.1.m1.1.1.cmml" xref="S4.SS1.2.2.2.2.2.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.2.2.2.2.2.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS1.3.3.3.3.3" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS1.3.3.3.3.3.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.SS1.3.3.3.3.3.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.3.3.3.3.3.1.m1.1a"><mo stretchy="false" id="S4.SS1.3.3.3.3.3.1.m1.1.1" xref="S4.SS1.3.3.3.3.3.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.3.3.3.3.3.1.m1.1b"><ci id="S4.SS1.3.3.3.3.3.1.m1.1.1.cmml" xref="S4.SS1.3.3.3.3.3.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.3.3.3.3.3.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS1.4.4.4.4.4" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS1.4.4.4.4.4.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.SS1.4.4.4.4.4.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.4.4.4.4.4.1.m1.1a"><mo stretchy="false" id="S4.SS1.4.4.4.4.4.1.m1.1.1" xref="S4.SS1.4.4.4.4.4.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.4.4.4.4.4.1.m1.1b"><ci id="S4.SS1.4.4.4.4.4.1.m1.1.1.cmml" xref="S4.SS1.4.4.4.4.4.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.4.4.4.4.4.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS1.4.4.4.4.6" class="ltx_td"></td>
</tr>
<tr id="S4.SS1.4.4.4.6" class="ltx_tr">
<td id="S4.SS1.4.4.4.6.1" class="ltx_td ltx_align_left"><span id="S4.SS1.4.4.4.6.1.1" class="ltx_text ltx_font_bold">Method</span></td>
<td id="S4.SS1.4.4.4.6.2" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.2.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.SS1.4.4.4.6.3" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.3.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.SS1.4.4.4.6.4" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.4.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.SS1.4.4.4.6.5" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.5.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.SS1.4.4.4.6.6" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.6.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.SS1.4.4.4.6.7" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.7.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.SS1.4.4.4.6.8" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.8.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.SS1.4.4.4.6.9" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.9.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.SS1.4.4.4.6.10" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.10.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.SS1.4.4.4.6.11" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.11.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.SS1.4.4.4.6.12" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.12.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.SS1.4.4.4.6.13" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.6.13.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.SS1.4.4.4.6.14" class="ltx_td ltx_align_center"><span id="S4.SS1.4.4.4.6.14.1" class="ltx_text ltx_font_italic"># Wins</span></td>
</tr>
<tr id="S4.SS1.4.4.4.7" class="ltx_tr">
<td id="S4.SS1.4.4.4.7.1" class="ltx_td ltx_align_left ltx_border_t"><span id="S4.SS1.4.4.4.7.1.1" class="ltx_text ltx_font_bold">Wanda</span></td>
<td id="S4.SS1.4.4.4.7.2" class="ltx_td ltx_align_center ltx_border_t">.01</td>
<td id="S4.SS1.4.4.4.7.3" class="ltx_td ltx_align_center ltx_border_t">.05</td>
<td id="S4.SS1.4.4.4.7.4" class="ltx_td ltx_align_center ltx_border_t">.05</td>
<td id="S4.SS1.4.4.4.7.5" class="ltx_td ltx_align_center ltx_border_t">.03</td>
<td id="S4.SS1.4.4.4.7.6" class="ltx_td ltx_align_center ltx_border_t">.24</td>
<td id="S4.SS1.4.4.4.7.7" class="ltx_td ltx_align_center ltx_border_t">.08</td>
<td id="S4.SS1.4.4.4.7.8" class="ltx_td ltx_align_center ltx_border_t">.20</td>
<td id="S4.SS1.4.4.4.7.9" class="ltx_td ltx_align_center ltx_border_t">.10</td>
<td id="S4.SS1.4.4.4.7.10" class="ltx_td ltx_align_center ltx_border_t">.46</td>
<td id="S4.SS1.4.4.4.7.11" class="ltx_td ltx_align_center ltx_border_t">.04</td>
<td id="S4.SS1.4.4.4.7.12" class="ltx_td ltx_align_center ltx_border_t">.21</td>
<td id="S4.SS1.4.4.4.7.13" class="ltx_td ltx_align_center ltx_border_t">.01</td>
<td id="S4.SS1.4.4.4.7.14" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.4.4.4.7.14.1" class="ltx_text ltx_font_bold">6</span></td>
</tr>
<tr id="S4.SS1.4.4.4.8" class="ltx_tr">
<td id="S4.SS1.4.4.4.8.1" class="ltx_td ltx_align_left"><span id="S4.SS1.4.4.4.8.1.1" class="ltx_text ltx_font_bold">SparseGPT</span></td>
<td id="S4.SS1.4.4.4.8.2" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS1.4.4.4.8.3" class="ltx_td ltx_align_center">.09</td>
<td id="S4.SS1.4.4.4.8.4" class="ltx_td ltx_align_center">.14</td>
<td id="S4.SS1.4.4.4.8.5" class="ltx_td ltx_align_center">.03</td>
<td id="S4.SS1.4.4.4.8.6" class="ltx_td ltx_align_center">.04</td>
<td id="S4.SS1.4.4.4.8.7" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS1.4.4.4.8.8" class="ltx_td ltx_align_center">.23</td>
<td id="S4.SS1.4.4.4.8.9" class="ltx_td ltx_align_center">.14</td>
<td id="S4.SS1.4.4.4.8.10" class="ltx_td ltx_align_center">.48</td>
<td id="S4.SS1.4.4.4.8.11" class="ltx_td ltx_align_center">.02</td>
<td id="S4.SS1.4.4.4.8.12" class="ltx_td ltx_align_center">.03</td>
<td id="S4.SS1.4.4.4.8.13" class="ltx_td ltx_align_center">.07</td>
<td id="S4.SS1.4.4.4.8.14" class="ltx_td ltx_align_center">4</td>
</tr>
<tr id="S4.SS1.4.4.4.9" class="ltx_tr">
<td id="S4.SS1.4.4.4.9.1" class="ltx_td ltx_align_left">
<span id="S4.SS1.4.4.4.9.1.1" class="ltx_ERROR undefined">\cdashline</span>1-14

<span id="S4.SS1.4.4.4.9.1.2" class="ltx_text ltx_font_bold">AWQ</span>
</td>
<td id="S4.SS1.4.4.4.9.2" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS1.4.4.4.9.3" class="ltx_td ltx_align_center">.07</td>
<td id="S4.SS1.4.4.4.9.4" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.4.4.4.9.5" class="ltx_td ltx_align_center">.04</td>
<td id="S4.SS1.4.4.4.9.6" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS1.4.4.4.9.7" class="ltx_td ltx_align_center">.08</td>
<td id="S4.SS1.4.4.4.9.8" class="ltx_td ltx_align_center">.16</td>
<td id="S4.SS1.4.4.4.9.9" class="ltx_td ltx_align_center">.11</td>
<td id="S4.SS1.4.4.4.9.10" class="ltx_td ltx_align_center">.26</td>
<td id="S4.SS1.4.4.4.9.11" class="ltx_td ltx_align_center">.05</td>
<td id="S4.SS1.4.4.4.9.12" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.4.4.4.9.13" class="ltx_td ltx_align_center">.14</td>
<td id="S4.SS1.4.4.4.9.14" class="ltx_td ltx_align_center"><span id="S4.SS1.4.4.4.9.14.1" class="ltx_text ltx_font_bold">12</span></td>
</tr>
<tr id="S4.SS1.4.4.4.10" class="ltx_tr">
<td id="S4.SS1.4.4.4.10.1" class="ltx_td ltx_align_left"><span id="S4.SS1.4.4.4.10.1.1" class="ltx_text ltx_font_bold">GPTQ</span></td>
<td id="S4.SS1.4.4.4.10.2" class="ltx_td ltx_align_center">.21</td>
<td id="S4.SS1.4.4.4.10.3" class="ltx_td ltx_align_center">.16</td>
<td id="S4.SS1.4.4.4.10.4" class="ltx_td ltx_align_center">.42</td>
<td id="S4.SS1.4.4.4.10.5" class="ltx_td ltx_align_center">.36</td>
<td id="S4.SS1.4.4.4.10.6" class="ltx_td ltx_align_center">.03</td>
<td id="S4.SS1.4.4.4.10.7" class="ltx_td ltx_align_center">.37</td>
<td id="S4.SS1.4.4.4.10.8" class="ltx_td ltx_align_center">.40</td>
<td id="S4.SS1.4.4.4.10.9" class="ltx_td ltx_align_center">.20</td>
<td id="S4.SS1.4.4.4.10.10" class="ltx_td ltx_align_center">.59</td>
<td id="S4.SS1.4.4.4.10.11" class="ltx_td ltx_align_center">.36</td>
<td id="S4.SS1.4.4.4.10.12" class="ltx_td ltx_align_center">.11</td>
<td id="S4.SS1.4.4.4.10.13" class="ltx_td ltx_align_center">.27</td>
<td id="S4.SS1.4.4.4.10.14" class="ltx_td ltx_align_center">0</td>
</tr>
<tr id="S4.SS1.4.4.4.11" class="ltx_tr">
<td id="S4.SS1.4.4.4.11.1" class="ltx_td ltx_align_left ltx_border_bb ltx_border_t"><span id="S4.SS1.4.4.4.11.1.1" class="ltx_text ltx_font_italic"># Wins</span></td>
<td id="S4.SS1.4.4.4.11.2" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS1.4.4.4.11.2.1" class="ltx_text ltx_font_bold">2</span></td>
<td id="S4.SS1.4.4.4.11.3" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS1.4.4.4.11.4" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS1.4.4.4.11.5" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS1.4.4.4.11.6" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS1.4.4.4.11.6.1" class="ltx_text ltx_font_bold">2</span></td>
<td id="S4.SS1.4.4.4.11.7" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS1.4.4.4.11.8" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.4.4.4.11.9" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS1.4.4.4.11.9.1" class="ltx_text ltx_font_bold">4</span></td>
<td id="S4.SS1.4.4.4.11.10" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.4.4.4.11.11" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS1.4.4.4.11.12" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS1.4.4.4.11.12.1" class="ltx_text ltx_font_bold">3</span></td>
<td id="S4.SS1.4.4.4.11.13" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS1.4.4.4.11.14" class="ltx_td ltx_border_bb ltx_border_t"></td>
</tr>
</table>
</span></div>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_2">
<div id="S4.SS1.8.8" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:122.8pt;vertical-align:-1.7pt;"><span class="ltx_transformed_inner" style="transform:translate(-62.8pt,17.5pt) scale(0.775455121845862,0.775455121845862) ;">
<table id="S4.SS1.8.8.4" class="ltx_tabular ltx_align_middle">
<tr id="S4.SS1.8.8.4.5" class="ltx_tr">
<td id="S4.SS1.8.8.4.5.1" class="ltx_td ltx_border_tt"></td>
<td id="S4.SS1.8.8.4.5.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.SS1.8.8.4.5.2.1" class="ltx_text ltx_font_bold">Strict Edit Locality</span></td>
<td id="S4.SS1.8.8.4.5.3" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.SS1.8.8.4.5.3.1" class="ltx_text ltx_font_bold">MMLU</span></td>
<td id="S4.SS1.8.8.4.5.4" class="ltx_td ltx_border_tt"></td>
</tr>
<tr id="S4.SS1.8.8.4.4" class="ltx_tr">
<td id="S4.SS1.8.8.4.4.5" class="ltx_td"></td>
<td id="S4.SS1.5.5.1.1.1" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS1.5.5.1.1.1.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.SS1.5.5.1.1.1.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.5.5.1.1.1.1.m1.1a"><mo stretchy="false" id="S4.SS1.5.5.1.1.1.1.m1.1.1" xref="S4.SS1.5.5.1.1.1.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.5.5.1.1.1.1.m1.1b"><ci id="S4.SS1.5.5.1.1.1.1.m1.1.1.cmml" xref="S4.SS1.5.5.1.1.1.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.5.5.1.1.1.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS1.6.6.2.2.2" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS1.6.6.2.2.2.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.SS1.6.6.2.2.2.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.6.6.2.2.2.1.m1.1a"><mo stretchy="false" id="S4.SS1.6.6.2.2.2.1.m1.1.1" xref="S4.SS1.6.6.2.2.2.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.6.6.2.2.2.1.m1.1b"><ci id="S4.SS1.6.6.2.2.2.1.m1.1.1.cmml" xref="S4.SS1.6.6.2.2.2.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.6.6.2.2.2.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS1.7.7.3.3.3" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS1.7.7.3.3.3.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.SS1.7.7.3.3.3.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.7.7.3.3.3.1.m1.1a"><mo stretchy="false" id="S4.SS1.7.7.3.3.3.1.m1.1.1" xref="S4.SS1.7.7.3.3.3.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.7.7.3.3.3.1.m1.1b"><ci id="S4.SS1.7.7.3.3.3.1.m1.1.1.cmml" xref="S4.SS1.7.7.3.3.3.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.7.7.3.3.3.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS1.8.8.4.4.4" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS1.8.8.4.4.4.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.SS1.8.8.4.4.4.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.8.8.4.4.4.1.m1.1a"><mo stretchy="false" id="S4.SS1.8.8.4.4.4.1.m1.1.1" xref="S4.SS1.8.8.4.4.4.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.8.8.4.4.4.1.m1.1b"><ci id="S4.SS1.8.8.4.4.4.1.m1.1.1.cmml" xref="S4.SS1.8.8.4.4.4.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.8.8.4.4.4.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS1.8.8.4.4.6" class="ltx_td"></td>
</tr>
<tr id="S4.SS1.8.8.4.6" class="ltx_tr">
<td id="S4.SS1.8.8.4.6.1" class="ltx_td ltx_align_left"><span id="S4.SS1.8.8.4.6.1.1" class="ltx_text ltx_font_bold">Method</span></td>
<td id="S4.SS1.8.8.4.6.2" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.2.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.SS1.8.8.4.6.3" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.3.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.SS1.8.8.4.6.4" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.4.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.SS1.8.8.4.6.5" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.5.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.SS1.8.8.4.6.6" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.6.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.SS1.8.8.4.6.7" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.7.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.SS1.8.8.4.6.8" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.8.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.SS1.8.8.4.6.9" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.9.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.SS1.8.8.4.6.10" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.10.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.SS1.8.8.4.6.11" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.11.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.SS1.8.8.4.6.12" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.12.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.SS1.8.8.4.6.13" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS1.8.8.4.6.13.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.SS1.8.8.4.6.14" class="ltx_td ltx_align_center"><span id="S4.SS1.8.8.4.6.14.1" class="ltx_text ltx_font_italic"># Wins</span></td>
</tr>
<tr id="S4.SS1.8.8.4.7" class="ltx_tr">
<td id="S4.SS1.8.8.4.7.1" class="ltx_td ltx_align_left ltx_border_t"><span id="S4.SS1.8.8.4.7.1.1" class="ltx_text ltx_font_bold">Wanda</span></td>
<td id="S4.SS1.8.8.4.7.2" class="ltx_td ltx_align_center ltx_border_t">.89</td>
<td id="S4.SS1.8.8.4.7.3" class="ltx_td ltx_align_center ltx_border_t">.97</td>
<td id="S4.SS1.8.8.4.7.4" class="ltx_td ltx_align_center ltx_border_t">.96</td>
<td id="S4.SS1.8.8.4.7.5" class="ltx_td ltx_align_center ltx_border_t">.04</td>
<td id="S4.SS1.8.8.4.7.6" class="ltx_td ltx_align_center ltx_border_t">.00</td>
<td id="S4.SS1.8.8.4.7.7" class="ltx_td ltx_align_center ltx_border_t">.01</td>
<td id="S4.SS1.8.8.4.7.8" class="ltx_td ltx_align_center ltx_border_t">.40</td>
<td id="S4.SS1.8.8.4.7.9" class="ltx_td ltx_align_center ltx_border_t">.41</td>
<td id="S4.SS1.8.8.4.7.10" class="ltx_td ltx_align_center ltx_border_t">.40</td>
<td id="S4.SS1.8.8.4.7.11" class="ltx_td ltx_align_center ltx_border_t">.00</td>
<td id="S4.SS1.8.8.4.7.12" class="ltx_td ltx_align_center ltx_border_t">.00</td>
<td id="S4.SS1.8.8.4.7.13" class="ltx_td ltx_align_center ltx_border_t">.00</td>
<td id="S4.SS1.8.8.4.7.14" class="ltx_td ltx_align_center ltx_border_t">0</td>
</tr>
<tr id="S4.SS1.8.8.4.8" class="ltx_tr">
<td id="S4.SS1.8.8.4.8.1" class="ltx_td ltx_align_left"><span id="S4.SS1.8.8.4.8.1.1" class="ltx_text ltx_font_bold">SparseGPT</span></td>
<td id="S4.SS1.8.8.4.8.2" class="ltx_td ltx_align_center">.89</td>
<td id="S4.SS1.8.8.4.8.3" class="ltx_td ltx_align_center">.96</td>
<td id="S4.SS1.8.8.4.8.4" class="ltx_td ltx_align_center">.96</td>
<td id="S4.SS1.8.8.4.8.5" class="ltx_td ltx_align_center">.03</td>
<td id="S4.SS1.8.8.4.8.6" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.8.7" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.8.8" class="ltx_td ltx_align_center">.40</td>
<td id="S4.SS1.8.8.4.8.9" class="ltx_td ltx_align_center">.40</td>
<td id="S4.SS1.8.8.4.8.10" class="ltx_td ltx_align_center">.40</td>
<td id="S4.SS1.8.8.4.8.11" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.8.12" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.8.13" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.8.14" class="ltx_td ltx_align_center"><span id="S4.SS1.8.8.4.8.14.1" class="ltx_text ltx_font_bold">4</span></td>
</tr>
<tr id="S4.SS1.8.8.4.9" class="ltx_tr">
<td id="S4.SS1.8.8.4.9.1" class="ltx_td ltx_align_left">
<span id="S4.SS1.8.8.4.9.1.1" class="ltx_ERROR undefined">\cdashline</span>1-14

<span id="S4.SS1.8.8.4.9.1.2" class="ltx_text ltx_font_bold">AWQ</span>
</td>
<td id="S4.SS1.8.8.4.9.2" class="ltx_td ltx_align_center">.85</td>
<td id="S4.SS1.8.8.4.9.3" class="ltx_td ltx_align_center">.96</td>
<td id="S4.SS1.8.8.4.9.4" class="ltx_td ltx_align_center">.93</td>
<td id="S4.SS1.8.8.4.9.5" class="ltx_td ltx_align_center">.04</td>
<td id="S4.SS1.8.8.4.9.6" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.9.7" class="ltx_td ltx_align_center">.03</td>
<td id="S4.SS1.8.8.4.9.8" class="ltx_td ltx_align_center">.41</td>
<td id="S4.SS1.8.8.4.9.9" class="ltx_td ltx_align_center">.41</td>
<td id="S4.SS1.8.8.4.9.10" class="ltx_td ltx_align_center">.41</td>
<td id="S4.SS1.8.8.4.9.11" class="ltx_td ltx_align_center">.02</td>
<td id="S4.SS1.8.8.4.9.12" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS1.8.8.4.9.13" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.9.14" class="ltx_td ltx_align_center"><span id="S4.SS1.8.8.4.9.14.1" class="ltx_text ltx_font_bold">5</span></td>
</tr>
<tr id="S4.SS1.8.8.4.10" class="ltx_tr">
<td id="S4.SS1.8.8.4.10.1" class="ltx_td ltx_align_left"><span id="S4.SS1.8.8.4.10.1.1" class="ltx_text ltx_font_bold">GPTQ</span></td>
<td id="S4.SS1.8.8.4.10.2" class="ltx_td ltx_align_center">.89</td>
<td id="S4.SS1.8.8.4.10.3" class="ltx_td ltx_align_center">.96</td>
<td id="S4.SS1.8.8.4.10.4" class="ltx_td ltx_align_center">.95</td>
<td id="S4.SS1.8.8.4.10.5" class="ltx_td ltx_align_center">.07</td>
<td id="S4.SS1.8.8.4.10.6" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.10.7" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS1.8.8.4.10.8" class="ltx_td ltx_align_center">.41</td>
<td id="S4.SS1.8.8.4.10.9" class="ltx_td ltx_align_center">.41</td>
<td id="S4.SS1.8.8.4.10.10" class="ltx_td ltx_align_center">.42</td>
<td id="S4.SS1.8.8.4.10.11" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS1.8.8.4.10.12" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS1.8.8.4.10.13" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS1.8.8.4.10.14" class="ltx_td ltx_align_center">2</td>
</tr>
<tr id="S4.SS1.8.8.4.11" class="ltx_tr">
<td id="S4.SS1.8.8.4.11.1" class="ltx_td ltx_align_left ltx_border_bb ltx_border_t"><span id="S4.SS1.8.8.4.11.1.1" class="ltx_text ltx_font_italic"># Wins</span></td>
<td id="S4.SS1.8.8.4.11.2" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS1.8.8.4.11.2.1" class="ltx_text ltx_font_bold">4</span></td>
<td id="S4.SS1.8.8.4.11.3" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.8.8.4.11.4" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.8.8.4.11.5" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.8.8.4.11.6" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS1.8.8.4.11.6.1" class="ltx_text ltx_font_bold">4</span></td>
<td id="S4.SS1.8.8.4.11.7" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.8.8.4.11.8" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.8.8.4.11.9" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.8.8.4.11.10" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.8.8.4.11.11" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS1.8.8.4.11.12" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS1.8.8.4.11.13" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS1.8.8.4.11.14" class="ltx_td ltx_border_bb ltx_border_t"></td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="S4.SS1.14.12.1.1" class="ltx_text" style="font-size:90%;">Table 2</span>: </span><span id="S4.SS1.14.13.2" class="ltx_text" style="font-size:90%;">Composability metrics for Knowledge Editing composed with Model Compression. Order-free Error (↓) measures best-case error caused by interventions, while Order Sensitivity (↓) measures unwanted effects of intervention ordering.
# Wins denotes the number of times each method outperforms other methods of its intervention category, counting all ties as losses.
We use 25% sparsity and 4-bit quantization to showcase practical compression levels.
<span id="S4.SS1.14.13.2.1" class="ltx_text ltx_font_bold">Key Takeaways:</span> AWQ composes better than GPTQ and LoRA is surprisingly non-composable.</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_2">
<p id="S4.SS1.14.14" class="ltx_p ltx_figure_panel">Our results are shown in Figure <a href="#S4.F2" title="Figure 2 ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>, where we find that model compression and knowledge editing interact in meaningful ways at different levels of compression.
We report three editing metrics and MMLU to measure general model degradation. All results are averaged (mean) over 10 batches of 50 edits each.
Illustrated below, these experiments reveal that there is a large gap in composability of existing methods. Our findings are as follows.</p>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1"><svg id="S4.SS1.9.pic1" class="ltx_picture ltx_figure_panel" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S4.SS1.9.pic1.1.1.1.1.1" class="ltx_text">1</span></foreignObject></g></g></svg></div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<p id="S4.SS1.14.15" class="ltx_p ltx_figure_panel"><span id="S4.SS1.14.15.1" class="ltx_text ltx_font_bold">Model compression degrades editing performance.</span>
Across Figure <a href="#S4.F2" title="Figure 2 ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>, we see that edits made by MEMIT, LoRA, and Finetuning deteriorate as compression increases.
In weight pruning, the editing metrics, Success, Generalization, and Locality, steadily drop with higher sparsity.
For quantization, the steepest drops occur between 2 and 4 Bits, and GPTQ decays editing performance faster than AWQ.
While general performance decay from compression is unsurprising, the steepness of the decay differs dramatically between editors and compression methods.
Decay curves also depend on the order of interventions.
We also note that when compression is applied after editing, the <span id="S4.SS1.14.15.2" class="ltx_text ltx_font_italic">only</span> thing that can impact the editing metrics is the compression.
Interestingly, Finetuning is also a surprisingly strong baseline at higher levels of compression, often achieving better editing performance than MEMIT or LoRA.
Overall, these results suggests that model compression can limit a model’s editability.</p>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1"><svg id="S4.SS1.10.pic2" class="ltx_picture ltx_figure_panel" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S4.SS1.10.pic2.1.1.1.1.1" class="ltx_text">2</span></foreignObject></g></g></svg></div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<p id="S4.SS1.14.16" class="ltx_p ltx_figure_panel"><span id="S4.SS1.14.16.1" class="ltx_text ltx_font_bold">Editing performance hinges on the order of interventions.</span>
The shaded area between the curves in Figure <a href="#S4.F2" title="Figure 2 ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a> shows the impact of ordering.
Interestingly, the ordering is highly impactful in most cases and varies dramatically between editing methods.
For pruning methods (left side), compressing first is generally better than editing first.
For example, pruning after Finetuning or LoRA destroys their edits much faster than if the model had been compressed first.
For quantization (right side), the order also clearly matters.
However, the best order differs by quantization method.
For GPTQ, editing should be done first.
For AWQ, compression should be done first.
This finding suggests that these interventions interact in meaningful ways and successful composition will benefit from tailored, composable methods that have expected behaviors.</p>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1"><svg id="S4.SS1.11.pic3" class="ltx_picture ltx_figure_panel" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S4.SS1.11.pic3.1.1.1.1.1" class="ltx_text">3</span></foreignObject></g></g></svg></div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<p id="S4.SS1.13.10" class="ltx_p ltx_figure_panel"><span id="S4.SS1.13.10.1" class="ltx_text ltx_font_bold">Composability can vary within the same intervention category.</span>
We next summarize the general composability of each pair of interventions in Table <a href="#S4.SS1" title="4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4.1</span></a>, using the metrics proposed in Section <a href="#S2" title="2 An Evaluation Framework for Composable Interventions ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>: Order-free Error (<math id="S4.SS1.12.9.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.12.9.m1.1a"><mo stretchy="false" id="S4.SS1.12.9.m1.1.1" xref="S4.SS1.12.9.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.12.9.m1.1b"><ci id="S4.SS1.12.9.m1.1.1.cmml" xref="S4.SS1.12.9.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.12.9.m1.1c">\downarrow</annotation></semantics></math>) measures best-case composition error, while Order Sensitivity (<math id="S4.SS1.13.10.m2.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS1.13.10.m2.1a"><mo stretchy="false" id="S4.SS1.13.10.m2.1.1" xref="S4.SS1.13.10.m2.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS1.13.10.m2.1b"><ci id="S4.SS1.13.10.m2.1.1.cmml" xref="S4.SS1.13.10.m2.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS1.13.10.m2.1c">\downarrow</annotation></semantics></math>) measures unwanted effects of intervention ordering.
These metrics let us compare methods according to composability, and we find that 1) MEMIT is generally the most composable editor, totaling 14 wins compared to 9 by Finetuning and 4 by LoRA, 2) SparseGPT is the most composable pruning method with 8 wins compared to Wanda’s 6, and 3) AWQ far surpasses GPTQ, achieving 17 wins compared to GPTQ’s 2.
However, many wins are quite close and composability can change dramatically between metrics.
For example, Finetuning has better Order-free Error than MEMIT on 2/3 metrics but underforms MEMIT on Order Sensitivity for all metrics, so MEMIT’s edits are more robust.
These results indicate a need to use many metrics to explicitly target and evaluate composability.</p>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1"><svg id="S4.SS1.14.pic4" class="ltx_picture ltx_figure_panel" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S4.SS1.14.pic4.1.1.1.1.1" class="ltx_text">4</span></foreignObject></g></g></svg></div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<p id="S4.SS1.14.17" class="ltx_p ltx_figure_panel"><span id="S4.SS1.14.17.1" class="ltx_text ltx_font_bold">Overall utility evaluations fail to measure composability.</span> Despite similar MMLU scores, methods vary significantly in composability. For example, Table <a href="#S4.SS1" title="4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4.1</span></a> shows that MEMIT and Wanda achieve near perfect Order Sensitivity on MMLU but have 0.24 Order Sensitivity on Edit Success.
This is further backed by Figure <a href="#S4.F2" title="Figure 2 ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">2</span></a>, where all compositions look very similar according to MMLU, but not according to editing metrics.
The disparity between overall utility and composability metrics underscores the need for more nuanced evaluations explicitly targeting composability as a design criterion.</p>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<section id="S4.SS2" class="ltx_subsection ltx_figure_panel">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">4.2 </span>Composing Model Compression with Machine Unlearning</h3>

<div id="S4.SS2.p1" class="ltx_para">
<p id="S4.SS2.p1.1" class="ltx_p">We next compose three model compression and three machine unlearning interventions.
We measure unlearning success with WMDP (Section <a href="#S3.SS1" title="3.1 Intervention Datasets, Methods, and Metrics ‣ 3 Experimental Setup ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3.1</span></a>).</p>
</div>
<figure id="S4.F3" class="ltx_figure">
<div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="S4.F3.sf1" class="ltx_figure ltx_figure_panel ltx_align_center"><img src="/html/2407.06483/assets/x4.png" id="S4.F3.sf1.g1" class="ltx_graphics ltx_img_landscape" width="461" height="241" alt="Refer to caption">
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_figure"><span id="S4.F3.sf1.2.1.1" class="ltx_text" style="font-size:90%;">(a)</span> </span></figcaption>
</figure>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="S4.F3.sf2" class="ltx_figure ltx_figure_panel ltx_align_center"><img src="/html/2407.06483/assets/x5.png" id="S4.F3.sf2.g1" class="ltx_graphics ltx_img_landscape" width="276" height="19" alt="Refer to caption">
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_figure"><span id="S4.F3.sf2.2.1.1" class="ltx_text" style="font-size:90%;">(b)</span> </span></figcaption>
</figure>
</div>
</div>
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_figure"><span id="S4.F3.4.1.1" class="ltx_text" style="font-size:90%;">Figure 3</span>: </span><span id="S4.F3.5.2" class="ltx_text" style="font-size:90%;">Composing Machine Unlearning with Model Compression at different levels of compression.
Lower values are better for WMDP, while higher values are better for MMLU.
<span id="S4.F3.5.2.1" class="ltx_text ltx_font_bold">Key Takeaways:</span> Unlearning should generally be applied <span id="S4.F3.5.2.2" class="ltx_text ltx_font_italic">before</span> compression, and performance varies significantly by composition.</span></figcaption>
</figure>
<div id="S4.SS2.p2" class="ltx_para ltx_noindent">
<svg id="S4.SS2.p2.pic1" class="ltx_picture" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S4.SS2.p2.pic1.1.1.1.1.1" class="ltx_text">5</span></foreignObject></g></g></svg>
<p id="S4.SS2.p2.1" class="ltx_p"><span id="S4.SS2.p2.1.1" class="ltx_text ltx_font_bold">Compression hinders unlearning.</span>
Our main results in Figure <a href="#S4.F3" title="Figure 3 ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">3</span></a> show that applying unlearning to a compressed model can degrade performance. With RMU, pruning the model before unlearning can significantly regress unlearning performance, especially at higher sparsity levels. We observe a similar trend with quantization when composing RMU and GPTQ. With GD, we similarly find that pruning after unlearning is best, though quantizing first can be optimal. We observe the opposite trend with GA, though GA’s poor unlearning performance, as seen in other studies <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib39" title="" class="ltx_ref">39</a>, <a href="#bib.bib60" title="" class="ltx_ref">60</a>, <a href="#bib.bib61" title="" class="ltx_ref">61</a>]</cite>, is a possible confounder. We overall find that unlearning a compressed model is more challenging but that the degree to which unlearning after compression hurts performance is highly contingent on the composition and compression level.</p>
</div>
<div id="S4.SS2.p3" class="ltx_para ltx_noindent">
<svg id="S4.SS2.p3.pic1" class="ltx_picture" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S4.SS2.p3.pic1.1.1.1.1.1" class="ltx_text">6</span></foreignObject></g></g></svg>
<p id="S4.SS2.p3.1" class="ltx_p"><span id="S4.SS2.p3.1.1" class="ltx_text ltx_font_bold">Order Sensitivity can determine overall composability.</span>
Table <a href="#S4.SS3" title="4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4.3</span></a> reports the composability metrics of each unlearning technique. As measured by Order-free Error on WMDP, GD and RMU outperform GA, which struggles to unlearn without significantly degrading utility. GD exhibits higher WMDP Order Sensitivity for pruning than quantization, whereas RMU has low Order Sensitivity across all techniques except when quantized by GPTQ. Despite both GD and RMU’s successful unlearning, RMU has lower overall Order Sensitivity than GD, suggesting that while task performance is critical, it alone does not fully capture the practical challenges of applying interventions—RMU is more robust to order.</p>
</div>
<section id="S4.SS3" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">4.3 </span>Composing Knowledge Editing with Machine Unlearning</h3>

<div id="S4.SS3.p1" class="ltx_para">
<p id="S4.SS3.p1.1" class="ltx_p">We next examine the composability between editing and unlearning. Both categories of interventions aim to modify the model’s knowledge without degrading overall utility. The need to keep LMs factually updated while forgetting undesirable knowledge makes pairing editing and unlearning a practical composition.</p>
</div>
<figure id="S4.SS3.5" class="ltx_table">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_2">
<div id="S4.SS3.4.4" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:144.5pt;vertical-align:-2.0pt;"><span class="ltx_transformed_inner" style="transform:translate(-20.8pt,6.8pt) scale(0.912323112857356,0.912323112857356) ;">
<table id="S4.SS3.4.4.4" class="ltx_tabular ltx_align_middle">
<tr id="S4.SS3.4.4.4.5" class="ltx_tr">
<td id="S4.SS3.4.4.4.5.1" class="ltx_td ltx_border_tt"></td>
<td id="S4.SS3.4.4.4.5.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.SS3.4.4.4.5.2.1" class="ltx_text ltx_font_bold">WMDP (Unlearning)</span></td>
<td id="S4.SS3.4.4.4.5.3" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.SS3.4.4.4.5.3.1" class="ltx_text ltx_font_bold">MMLU</span></td>
<td id="S4.SS3.4.4.4.5.4" class="ltx_td ltx_border_tt"></td>
</tr>
<tr id="S4.SS3.4.4.4.4" class="ltx_tr">
<td id="S4.SS3.4.4.4.4.5" class="ltx_td"></td>
<td id="S4.SS3.1.1.1.1.1" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS3.1.1.1.1.1.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.SS3.1.1.1.1.1.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS3.1.1.1.1.1.1.m1.1a"><mo stretchy="false" id="S4.SS3.1.1.1.1.1.1.m1.1.1" xref="S4.SS3.1.1.1.1.1.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS3.1.1.1.1.1.1.m1.1b"><ci id="S4.SS3.1.1.1.1.1.1.m1.1.1.cmml" xref="S4.SS3.1.1.1.1.1.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS3.1.1.1.1.1.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS3.2.2.2.2.2" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS3.2.2.2.2.2.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.SS3.2.2.2.2.2.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS3.2.2.2.2.2.1.m1.1a"><mo stretchy="false" id="S4.SS3.2.2.2.2.2.1.m1.1.1" xref="S4.SS3.2.2.2.2.2.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS3.2.2.2.2.2.1.m1.1b"><ci id="S4.SS3.2.2.2.2.2.1.m1.1.1.cmml" xref="S4.SS3.2.2.2.2.2.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS3.2.2.2.2.2.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS3.3.3.3.3.3" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS3.3.3.3.3.3.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.SS3.3.3.3.3.3.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS3.3.3.3.3.3.1.m1.1a"><mo stretchy="false" id="S4.SS3.3.3.3.3.3.1.m1.1.1" xref="S4.SS3.3.3.3.3.3.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS3.3.3.3.3.3.1.m1.1b"><ci id="S4.SS3.3.3.3.3.3.1.m1.1.1.cmml" xref="S4.SS3.3.3.3.3.3.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS3.3.3.3.3.3.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS3.4.4.4.4.4" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.SS3.4.4.4.4.4.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.SS3.4.4.4.4.4.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.SS3.4.4.4.4.4.1.m1.1a"><mo stretchy="false" id="S4.SS3.4.4.4.4.4.1.m1.1.1" xref="S4.SS3.4.4.4.4.4.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.SS3.4.4.4.4.4.1.m1.1b"><ci id="S4.SS3.4.4.4.4.4.1.m1.1.1.cmml" xref="S4.SS3.4.4.4.4.4.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.SS3.4.4.4.4.4.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.SS3.4.4.4.4.6" class="ltx_td"></td>
</tr>
<tr id="S4.SS3.4.4.4.6" class="ltx_tr">
<td id="S4.SS3.4.4.4.6.1" class="ltx_td ltx_align_left"><span id="S4.SS3.4.4.4.6.1.1" class="ltx_text ltx_font_bold">Method</span></td>
<td id="S4.SS3.4.4.4.6.2" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.2.1" class="ltx_text ltx_font_bold">GA</span></td>
<td id="S4.SS3.4.4.4.6.3" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.3.1" class="ltx_text ltx_font_bold">GD</span></td>
<td id="S4.SS3.4.4.4.6.4" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.4.1" class="ltx_text ltx_font_bold">RMU</span></td>
<td id="S4.SS3.4.4.4.6.5" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.5.1" class="ltx_text ltx_font_bold">GA</span></td>
<td id="S4.SS3.4.4.4.6.6" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.6.1" class="ltx_text ltx_font_bold">GD</span></td>
<td id="S4.SS3.4.4.4.6.7" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.7.1" class="ltx_text ltx_font_bold">RMU</span></td>
<td id="S4.SS3.4.4.4.6.8" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.8.1" class="ltx_text ltx_font_bold">GA</span></td>
<td id="S4.SS3.4.4.4.6.9" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.9.1" class="ltx_text ltx_font_bold">GD</span></td>
<td id="S4.SS3.4.4.4.6.10" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.10.1" class="ltx_text ltx_font_bold">RMU</span></td>
<td id="S4.SS3.4.4.4.6.11" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.11.1" class="ltx_text ltx_font_bold">GA</span></td>
<td id="S4.SS3.4.4.4.6.12" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.12.1" class="ltx_text ltx_font_bold">GD</span></td>
<td id="S4.SS3.4.4.4.6.13" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.SS3.4.4.4.6.13.1" class="ltx_text ltx_font_bold">RMU</span></td>
<td id="S4.SS3.4.4.4.6.14" class="ltx_td ltx_align_center"><span id="S4.SS3.4.4.4.6.14.1" class="ltx_text ltx_font_italic"># Wins</span></td>
</tr>
<tr id="S4.SS3.4.4.4.7" class="ltx_tr">
<td id="S4.SS3.4.4.4.7.1" class="ltx_td ltx_align_left ltx_border_t"><span id="S4.SS3.4.4.4.7.1.1" class="ltx_text ltx_font_bold">Wanda</span></td>
<td id="S4.SS3.4.4.4.7.2" class="ltx_td ltx_align_center ltx_border_t">.46</td>
<td id="S4.SS3.4.4.4.7.3" class="ltx_td ltx_align_center ltx_border_t">.35</td>
<td id="S4.SS3.4.4.4.7.4" class="ltx_td ltx_align_center ltx_border_t">.29</td>
<td id="S4.SS3.4.4.4.7.5" class="ltx_td ltx_align_center ltx_border_t">.03</td>
<td id="S4.SS3.4.4.4.7.6" class="ltx_td ltx_align_center ltx_border_t">.17</td>
<td id="S4.SS3.4.4.4.7.7" class="ltx_td ltx_align_center ltx_border_t">.03</td>
<td id="S4.SS3.4.4.4.7.8" class="ltx_td ltx_align_center ltx_border_t">.46</td>
<td id="S4.SS3.4.4.4.7.9" class="ltx_td ltx_align_center ltx_border_t">.42</td>
<td id="S4.SS3.4.4.4.7.10" class="ltx_td ltx_align_center ltx_border_t">.43</td>
<td id="S4.SS3.4.4.4.7.11" class="ltx_td ltx_align_center ltx_border_t">.02</td>
<td id="S4.SS3.4.4.4.7.12" class="ltx_td ltx_align_center ltx_border_t">.01</td>
<td id="S4.SS3.4.4.4.7.13" class="ltx_td ltx_align_center ltx_border_t">.01</td>
<td id="S4.SS3.4.4.4.7.14" class="ltx_td ltx_align_center ltx_border_t">4</td>
</tr>
<tr id="S4.SS3.4.4.4.8" class="ltx_tr">
<td id="S4.SS3.4.4.4.8.1" class="ltx_td ltx_align_left"><span id="S4.SS3.4.4.4.8.1.1" class="ltx_text ltx_font_bold">SparseGPT</span></td>
<td id="S4.SS3.4.4.4.8.2" class="ltx_td ltx_align_center">.43</td>
<td id="S4.SS3.4.4.4.8.3" class="ltx_td ltx_align_center">.28</td>
<td id="S4.SS3.4.4.4.8.4" class="ltx_td ltx_align_center">.28</td>
<td id="S4.SS3.4.4.4.8.5" class="ltx_td ltx_align_center">.03</td>
<td id="S4.SS3.4.4.4.8.6" class="ltx_td ltx_align_center">.22</td>
<td id="S4.SS3.4.4.4.8.7" class="ltx_td ltx_align_center">.02</td>
<td id="S4.SS3.4.4.4.8.8" class="ltx_td ltx_align_center">.49</td>
<td id="S4.SS3.4.4.4.8.9" class="ltx_td ltx_align_center">.42</td>
<td id="S4.SS3.4.4.4.8.10" class="ltx_td ltx_align_center">.43</td>
<td id="S4.SS3.4.4.4.8.11" class="ltx_td ltx_align_center">.06</td>
<td id="S4.SS3.4.4.4.8.12" class="ltx_td ltx_align_center">.09</td>
<td id="S4.SS3.4.4.4.8.13" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS3.4.4.4.8.14" class="ltx_td ltx_align_center">4</td>
</tr>
<tr id="S4.SS3.4.4.4.9" class="ltx_tr">
<td id="S4.SS3.4.4.4.9.1" class="ltx_td ltx_align_left">
<span id="S4.SS3.4.4.4.9.1.1" class="ltx_ERROR undefined">\cdashline</span>1-14

<span id="S4.SS3.4.4.4.9.1.2" class="ltx_text ltx_font_bold">AWQ</span>
</td>
<td id="S4.SS3.4.4.4.9.2" class="ltx_td ltx_align_center">.43</td>
<td id="S4.SS3.4.4.4.9.3" class="ltx_td ltx_align_center">.24</td>
<td id="S4.SS3.4.4.4.9.4" class="ltx_td ltx_align_center">.27</td>
<td id="S4.SS3.4.4.4.9.5" class="ltx_td ltx_align_center">.02</td>
<td id="S4.SS3.4.4.4.9.6" class="ltx_td ltx_align_center">.00</td>
<td id="S4.SS3.4.4.4.9.7" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS3.4.4.4.9.8" class="ltx_td ltx_align_center">.53</td>
<td id="S4.SS3.4.4.4.9.9" class="ltx_td ltx_align_center">.58</td>
<td id="S4.SS3.4.4.4.9.10" class="ltx_td ltx_align_center">.44</td>
<td id="S4.SS3.4.4.4.9.11" class="ltx_td ltx_align_center">.02</td>
<td id="S4.SS3.4.4.4.9.12" class="ltx_td ltx_align_center">.07</td>
<td id="S4.SS3.4.4.4.9.13" class="ltx_td ltx_align_center">.01</td>
<td id="S4.SS3.4.4.4.9.14" class="ltx_td ltx_align_center"><span id="S4.SS3.4.4.4.9.14.1" class="ltx_text ltx_font_bold">5</span></td>
</tr>
<tr id="S4.SS3.4.4.4.10" class="ltx_tr">
<td id="S4.SS3.4.4.4.10.1" class="ltx_td ltx_align_left"><span id="S4.SS3.4.4.4.10.1.1" class="ltx_text ltx_font_bold">GPTQ</span></td>
<td id="S4.SS3.4.4.4.10.2" class="ltx_td ltx_align_center">.34</td>
<td id="S4.SS3.4.4.4.10.3" class="ltx_td ltx_align_center">.24</td>
<td id="S4.SS3.4.4.4.10.4" class="ltx_td ltx_align_center">.27</td>
<td id="S4.SS3.4.4.4.10.5" class="ltx_td ltx_align_center">.09</td>
<td id="S4.SS3.4.4.4.10.6" class="ltx_td ltx_align_center">.02</td>
<td id="S4.SS3.4.4.4.10.7" class="ltx_td ltx_align_center">.18</td>
<td id="S4.SS3.4.4.4.10.8" class="ltx_td ltx_align_center">.52</td>
<td id="S4.SS3.4.4.4.10.9" class="ltx_td ltx_align_center">.44</td>
<td id="S4.SS3.4.4.4.10.10" class="ltx_td ltx_align_center">.42</td>
<td id="S4.SS3.4.4.4.10.11" class="ltx_td ltx_align_center">.09</td>
<td id="S4.SS3.4.4.4.10.12" class="ltx_td ltx_align_center">.24</td>
<td id="S4.SS3.4.4.4.10.13" class="ltx_td ltx_align_center">.05</td>
<td id="S4.SS3.4.4.4.10.14" class="ltx_td ltx_align_center">4</td>
</tr>
<tr id="S4.SS3.4.4.4.11" class="ltx_tr">
<td id="S4.SS3.4.4.4.11.1" class="ltx_td ltx_align_left ltx_border_bb ltx_border_t"><span id="S4.SS3.4.4.4.11.1.1" class="ltx_text ltx_font_italic"># Wins</span></td>
<td id="S4.SS3.4.4.4.11.2" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS3.4.4.4.11.3" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS3.4.4.4.11.3.1" class="ltx_text ltx_font_bold">2</span></td>
<td id="S4.SS3.4.4.4.11.4" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS3.4.4.4.11.5" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS3.4.4.4.11.6" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS3.4.4.4.11.6.1" class="ltx_text ltx_font_bold">2</span></td>
<td id="S4.SS3.4.4.4.11.7" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.SS3.4.4.4.11.8" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS3.4.4.4.11.9" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">2</td>
<td id="S4.SS3.4.4.4.11.10" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">2</td>
<td id="S4.SS3.4.4.4.11.11" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS3.4.4.4.11.12" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.SS3.4.4.4.11.13" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.SS3.4.4.4.11.13.1" class="ltx_text ltx_font_bold">3</span></td>
<td id="S4.SS3.4.4.4.11.14" class="ltx_td ltx_border_bb ltx_border_t"></td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="S4.SS3.5.6.1.1" class="ltx_text" style="font-size:90%;">Table 3</span>: </span><span id="S4.SS3.5.7.2" class="ltx_text" style="font-size:90%;">Composability metrics for machine unlearning and model compression. <span id="S4.SS3.5.7.2.1" class="ltx_text ltx_font_bold">Key Takeaways:</span> GD and RMU have overall comparable performance and far outperform GA. RMU is less sensitive to ordering than GD.
</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="S4.T4" class="ltx_table ltx_figure_panel">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="S4.T4.4" class="ltx_inline-block ltx_figure_panel ltx_align_center ltx_transformed_outer" style="width:433.6pt;height:103.6pt;vertical-align:-1.7pt;"><span class="ltx_transformed_inner" style="transform:translate(-54.7pt,12.9pt) scale(0.79839909542722,0.79839909542722) ;">
<table id="S4.T4.4.4" class="ltx_tabular ltx_align_middle">
<tr id="S4.T4.4.4.5" class="ltx_tr">
<td id="S4.T4.4.4.5.1" class="ltx_td ltx_border_tt"></td>
<td id="S4.T4.4.4.5.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.T4.4.4.5.2.1" class="ltx_text ltx_font_bold">Edit Success</span></td>
<td id="S4.T4.4.4.5.3" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.T4.4.4.5.3.1" class="ltx_text ltx_font_bold">Edit Generalization</span></td>
<td id="S4.T4.4.4.5.4" class="ltx_td ltx_border_tt"></td>
</tr>
<tr id="S4.T4.4.4.4" class="ltx_tr">
<td id="S4.T4.4.4.4.5" class="ltx_td"></td>
<td id="S4.T4.1.1.1.1" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.T4.1.1.1.1.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.T4.1.1.1.1.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.T4.1.1.1.1.1.m1.1a"><mo stretchy="false" id="S4.T4.1.1.1.1.1.m1.1.1" xref="S4.T4.1.1.1.1.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.T4.1.1.1.1.1.m1.1b"><ci id="S4.T4.1.1.1.1.1.m1.1.1.cmml" xref="S4.T4.1.1.1.1.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.T4.1.1.1.1.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.T4.2.2.2.2" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.T4.2.2.2.2.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.T4.2.2.2.2.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.T4.2.2.2.2.1.m1.1a"><mo stretchy="false" id="S4.T4.2.2.2.2.1.m1.1.1" xref="S4.T4.2.2.2.2.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.T4.2.2.2.2.1.m1.1b"><ci id="S4.T4.2.2.2.2.1.m1.1.1.cmml" xref="S4.T4.2.2.2.2.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.T4.2.2.2.2.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.T4.3.3.3.3" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.T4.3.3.3.3.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.T4.3.3.3.3.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.T4.3.3.3.3.1.m1.1a"><mo stretchy="false" id="S4.T4.3.3.3.3.1.m1.1.1" xref="S4.T4.3.3.3.3.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.T4.3.3.3.3.1.m1.1b"><ci id="S4.T4.3.3.3.3.1.m1.1.1.cmml" xref="S4.T4.3.3.3.3.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.T4.3.3.3.3.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.T4.4.4.4.4" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.T4.4.4.4.4.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.T4.4.4.4.4.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.T4.4.4.4.4.1.m1.1a"><mo stretchy="false" id="S4.T4.4.4.4.4.1.m1.1.1" xref="S4.T4.4.4.4.4.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.T4.4.4.4.4.1.m1.1b"><ci id="S4.T4.4.4.4.4.1.m1.1.1.cmml" xref="S4.T4.4.4.4.4.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.T4.4.4.4.4.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.T4.4.4.4.6" class="ltx_td"></td>
</tr>
<tr id="S4.T4.4.4.6" class="ltx_tr">
<td id="S4.T4.4.4.6.1" class="ltx_td ltx_align_left"><span id="S4.T4.4.4.6.1.1" class="ltx_text ltx_font_bold">Method</span></td>
<td id="S4.T4.4.4.6.2" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.2.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.T4.4.4.6.3" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.3.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.T4.4.4.6.4" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.4.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.T4.4.4.6.5" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.5.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.T4.4.4.6.6" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.6.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.T4.4.4.6.7" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.7.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.T4.4.4.6.8" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.8.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.T4.4.4.6.9" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.9.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.T4.4.4.6.10" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.10.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.T4.4.4.6.11" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.11.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.T4.4.4.6.12" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.12.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.T4.4.4.6.13" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.4.4.6.13.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.T4.4.4.6.14" class="ltx_td ltx_align_center"># Wins</td>
</tr>
<tr id="S4.T4.4.4.7" class="ltx_tr">
<td id="S4.T4.4.4.7.1" class="ltx_td ltx_align_left ltx_border_t"><span id="S4.T4.4.4.7.1.1" class="ltx_text ltx_font_bold">GA</span></td>
<td id="S4.T4.4.4.7.2" class="ltx_td ltx_align_center ltx_border_t">.93</td>
<td id="S4.T4.4.4.7.3" class="ltx_td ltx_align_center ltx_border_t">.52</td>
<td id="S4.T4.4.4.7.4" class="ltx_td ltx_align_center ltx_border_t">.00</td>
<td id="S4.T4.4.4.7.5" class="ltx_td ltx_align_center ltx_border_t">.07</td>
<td id="S4.T4.4.4.7.6" class="ltx_td ltx_align_center ltx_border_t">.48</td>
<td id="S4.T4.4.4.7.7" class="ltx_td ltx_align_center ltx_border_t">1.0</td>
<td id="S4.T4.4.4.7.8" class="ltx_td ltx_align_center ltx_border_t">.96</td>
<td id="S4.T4.4.4.7.9" class="ltx_td ltx_align_center ltx_border_t">.59</td>
<td id="S4.T4.4.4.7.10" class="ltx_td ltx_align_center ltx_border_t">.22</td>
<td id="S4.T4.4.4.7.11" class="ltx_td ltx_align_center ltx_border_t">.04</td>
<td id="S4.T4.4.4.7.12" class="ltx_td ltx_align_center ltx_border_t">.41</td>
<td id="S4.T4.4.4.7.13" class="ltx_td ltx_align_center ltx_border_t">.78</td>
<td id="S4.T4.4.4.7.14" class="ltx_td ltx_align_center ltx_border_t">1</td>
</tr>
<tr id="S4.T4.4.4.8" class="ltx_tr">
<td id="S4.T4.4.4.8.1" class="ltx_td ltx_align_left"><span id="S4.T4.4.4.8.1.1" class="ltx_text ltx_font_bold">GD</span></td>
<td id="S4.T4.4.4.8.2" class="ltx_td ltx_align_center">.01</td>
<td id="S4.T4.4.4.8.3" class="ltx_td ltx_align_center">.07</td>
<td id="S4.T4.4.4.8.4" class="ltx_td ltx_align_center">.00</td>
<td id="S4.T4.4.4.8.5" class="ltx_td ltx_align_center">.67</td>
<td id="S4.T4.4.4.8.6" class="ltx_td ltx_align_center">.40</td>
<td id="S4.T4.4.4.8.7" class="ltx_td ltx_align_center">.56</td>
<td id="S4.T4.4.4.8.8" class="ltx_td ltx_align_center">.19</td>
<td id="S4.T4.4.4.8.9" class="ltx_td ltx_align_center">.11</td>
<td id="S4.T4.4.4.8.10" class="ltx_td ltx_align_center">.29</td>
<td id="S4.T4.4.4.8.11" class="ltx_td ltx_align_center">.56</td>
<td id="S4.T4.4.4.8.12" class="ltx_td ltx_align_center">.41</td>
<td id="S4.T4.4.4.8.13" class="ltx_td ltx_align_center">.48</td>
<td id="S4.T4.4.4.8.14" class="ltx_td ltx_align_center">0</td>
</tr>
<tr id="S4.T4.4.4.9" class="ltx_tr">
<td id="S4.T4.4.4.9.1" class="ltx_td ltx_align_left"><span id="S4.T4.4.4.9.1.1" class="ltx_text ltx_font_bold">RMU</span></td>
<td id="S4.T4.4.4.9.2" class="ltx_td ltx_align_center">.00</td>
<td id="S4.T4.4.4.9.3" class="ltx_td ltx_align_center">.03</td>
<td id="S4.T4.4.4.9.4" class="ltx_td ltx_align_center">.00</td>
<td id="S4.T4.4.4.9.5" class="ltx_td ltx_align_center">.01</td>
<td id="S4.T4.4.4.9.6" class="ltx_td ltx_align_center">.01</td>
<td id="S4.T4.4.4.9.7" class="ltx_td ltx_align_center">.00</td>
<td id="S4.T4.4.4.9.8" class="ltx_td ltx_align_center">.18</td>
<td id="S4.T4.4.4.9.9" class="ltx_td ltx_align_center">.07</td>
<td id="S4.T4.4.4.9.10" class="ltx_td ltx_align_center">.29</td>
<td id="S4.T4.4.4.9.11" class="ltx_td ltx_align_center">.03</td>
<td id="S4.T4.4.4.9.12" class="ltx_td ltx_align_center">.04</td>
<td id="S4.T4.4.4.9.13" class="ltx_td ltx_align_center">.04</td>
<td id="S4.T4.4.4.9.14" class="ltx_td ltx_align_center"><span id="S4.T4.4.4.9.14.1" class="ltx_text ltx_font_bold">10</span></td>
</tr>
<tr id="S4.T4.4.4.10" class="ltx_tr">
<td id="S4.T4.4.4.10.1" class="ltx_td ltx_align_left ltx_border_bb ltx_border_t"><span id="S4.T4.4.4.10.1.1" class="ltx_text ltx_font_italic"># Wins</span></td>
<td id="S4.T4.4.4.10.2" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.T4.4.4.10.3" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.T4.4.4.10.4" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.T4.4.4.10.4.1" class="ltx_text ltx_font_bold">3</span></td>
<td id="S4.T4.4.4.10.5" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.4.4.10.6" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.4.4.10.7" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.4.4.10.8" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.T4.4.4.10.9" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.T4.4.4.10.9.1" class="ltx_text ltx_font_bold">2</span></td>
<td id="S4.T4.4.4.10.10" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.4.4.10.11" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.T4.4.4.10.11.1" class="ltx_text ltx_font_bold">2</span></td>
<td id="S4.T4.4.4.10.12" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.4.4.10.13" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.T4.4.4.10.14" class="ltx_td ltx_border_bb ltx_border_t"></td>
</tr>
</table>
</span></div>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="S4.T4.8" class="ltx_inline-block ltx_figure_panel ltx_align_center ltx_transformed_outer" style="width:433.6pt;height:103.6pt;vertical-align:-1.7pt;"><span class="ltx_transformed_inner" style="transform:translate(-54.7pt,12.9pt) scale(0.79839909542722,0.79839909542722) ;">
<table id="S4.T4.8.4" class="ltx_tabular ltx_align_middle">
<tr id="S4.T4.8.4.5" class="ltx_tr">
<td id="S4.T4.8.4.5.1" class="ltx_td ltx_border_tt"></td>
<td id="S4.T4.8.4.5.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.T4.8.4.5.2.1" class="ltx_text ltx_font_bold">WMDP (Unlearning)</span></td>
<td id="S4.T4.8.4.5.3" class="ltx_td ltx_align_center ltx_border_tt" colspan="6"><span id="S4.T4.8.4.5.3.1" class="ltx_text ltx_font_bold">MMLU</span></td>
<td id="S4.T4.8.4.5.4" class="ltx_td ltx_border_tt"></td>
</tr>
<tr id="S4.T4.8.4.4" class="ltx_tr">
<td id="S4.T4.8.4.4.5" class="ltx_td"></td>
<td id="S4.T4.5.1.1.1" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.T4.5.1.1.1.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.T4.5.1.1.1.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.T4.5.1.1.1.1.m1.1a"><mo stretchy="false" id="S4.T4.5.1.1.1.1.m1.1.1" xref="S4.T4.5.1.1.1.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.T4.5.1.1.1.1.m1.1b"><ci id="S4.T4.5.1.1.1.1.m1.1.1.cmml" xref="S4.T4.5.1.1.1.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.T4.5.1.1.1.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.T4.6.2.2.2" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.T4.6.2.2.2.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.T4.6.2.2.2.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.T4.6.2.2.2.1.m1.1a"><mo stretchy="false" id="S4.T4.6.2.2.2.1.m1.1.1" xref="S4.T4.6.2.2.2.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.T4.6.2.2.2.1.m1.1b"><ci id="S4.T4.6.2.2.2.1.m1.1.1.cmml" xref="S4.T4.6.2.2.2.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.T4.6.2.2.2.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.T4.7.3.3.3" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.T4.7.3.3.3.1" class="ltx_text ltx_font_bold">Order-free Error (<math id="S4.T4.7.3.3.3.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.T4.7.3.3.3.1.m1.1a"><mo stretchy="false" id="S4.T4.7.3.3.3.1.m1.1.1" xref="S4.T4.7.3.3.3.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.T4.7.3.3.3.1.m1.1b"><ci id="S4.T4.7.3.3.3.1.m1.1.1.cmml" xref="S4.T4.7.3.3.3.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.T4.7.3.3.3.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.T4.8.4.4.4" class="ltx_td ltx_align_center ltx_border_t" colspan="3"><span id="S4.T4.8.4.4.4.1" class="ltx_text ltx_font_bold">Order Sensitivity (<math id="S4.T4.8.4.4.4.1.m1.1" class="ltx_Math" alttext="\downarrow" display="inline"><semantics id="S4.T4.8.4.4.4.1.m1.1a"><mo stretchy="false" id="S4.T4.8.4.4.4.1.m1.1.1" xref="S4.T4.8.4.4.4.1.m1.1.1.cmml">↓</mo><annotation-xml encoding="MathML-Content" id="S4.T4.8.4.4.4.1.m1.1b"><ci id="S4.T4.8.4.4.4.1.m1.1.1.cmml" xref="S4.T4.8.4.4.4.1.m1.1.1">↓</ci></annotation-xml><annotation encoding="application/x-tex" id="S4.T4.8.4.4.4.1.m1.1c">\downarrow</annotation></semantics></math>)</span></td>
<td id="S4.T4.8.4.4.6" class="ltx_td"></td>
</tr>
<tr id="S4.T4.8.4.6" class="ltx_tr">
<td id="S4.T4.8.4.6.1" class="ltx_td ltx_align_left"><span id="S4.T4.8.4.6.1.1" class="ltx_text ltx_font_bold">Method</span></td>
<td id="S4.T4.8.4.6.2" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.2.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.T4.8.4.6.3" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.3.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.T4.8.4.6.4" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.4.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.T4.8.4.6.5" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.5.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.T4.8.4.6.6" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.6.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.T4.8.4.6.7" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.7.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.T4.8.4.6.8" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.8.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.T4.8.4.6.9" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.9.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.T4.8.4.6.10" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.10.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.T4.8.4.6.11" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.11.1" class="ltx_text ltx_font_bold">FT</span></td>
<td id="S4.T4.8.4.6.12" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.12.1" class="ltx_text ltx_font_bold">MEMIT</span></td>
<td id="S4.T4.8.4.6.13" class="ltx_td ltx_align_center ltx_border_t"><span id="S4.T4.8.4.6.13.1" class="ltx_text ltx_font_bold">LoRA</span></td>
<td id="S4.T4.8.4.6.14" class="ltx_td ltx_align_center"><span id="S4.T4.8.4.6.14.1" class="ltx_text ltx_font_italic"># Wins</span></td>
</tr>
<tr id="S4.T4.8.4.7" class="ltx_tr">
<td id="S4.T4.8.4.7.1" class="ltx_td ltx_align_left ltx_border_t"><span id="S4.T4.8.4.7.1.1" class="ltx_text ltx_font_bold">GA</span></td>
<td id="S4.T4.8.4.7.2" class="ltx_td ltx_align_center ltx_border_t">.47</td>
<td id="S4.T4.8.4.7.3" class="ltx_td ltx_align_center ltx_border_t">.40</td>
<td id="S4.T4.8.4.7.4" class="ltx_td ltx_align_center ltx_border_t">.28</td>
<td id="S4.T4.8.4.7.5" class="ltx_td ltx_align_center ltx_border_t">.00</td>
<td id="S4.T4.8.4.7.6" class="ltx_td ltx_align_center ltx_border_t">.05</td>
<td id="S4.T4.8.4.7.7" class="ltx_td ltx_align_center ltx_border_t">.07</td>
<td id="S4.T4.8.4.7.8" class="ltx_td ltx_align_center ltx_border_t">.47</td>
<td id="S4.T4.8.4.7.9" class="ltx_td ltx_align_center ltx_border_t">.51</td>
<td id="S4.T4.8.4.7.10" class="ltx_td ltx_align_center ltx_border_t">.64</td>
<td id="S4.T4.8.4.7.11" class="ltx_td ltx_align_center ltx_border_t">.01</td>
<td id="S4.T4.8.4.7.12" class="ltx_td ltx_align_center ltx_border_t">.04</td>
<td id="S4.T4.8.4.7.13" class="ltx_td ltx_align_center ltx_border_t">.07</td>
<td id="S4.T4.8.4.7.14" class="ltx_td ltx_align_center ltx_border_t">1</td>
</tr>
<tr id="S4.T4.8.4.8" class="ltx_tr">
<td id="S4.T4.8.4.8.1" class="ltx_td ltx_align_left"><span id="S4.T4.8.4.8.1.1" class="ltx_text ltx_font_bold">GD</span></td>
<td id="S4.T4.8.4.8.2" class="ltx_td ltx_align_center">.29</td>
<td id="S4.T4.8.4.8.3" class="ltx_td ltx_align_center">.26</td>
<td id="S4.T4.8.4.8.4" class="ltx_td ltx_align_center">.30</td>
<td id="S4.T4.8.4.8.5" class="ltx_td ltx_align_center">.00</td>
<td id="S4.T4.8.4.8.6" class="ltx_td ltx_align_center">.02</td>
<td id="S4.T4.8.4.8.7" class="ltx_td ltx_align_center">.24</td>
<td id="S4.T4.8.4.8.8" class="ltx_td ltx_align_center">.41</td>
<td id="S4.T4.8.4.8.9" class="ltx_td ltx_align_center">.42</td>
<td id="S4.T4.8.4.8.10" class="ltx_td ltx_align_center">.41</td>
<td id="S4.T4.8.4.8.11" class="ltx_td ltx_align_center">.18</td>
<td id="S4.T4.8.4.8.12" class="ltx_td ltx_align_center">.22</td>
<td id="S4.T4.8.4.8.13" class="ltx_td ltx_align_center">.14</td>
<td id="S4.T4.8.4.8.14" class="ltx_td ltx_align_center">4</td>
</tr>
<tr id="S4.T4.8.4.9" class="ltx_tr">
<td id="S4.T4.8.4.9.1" class="ltx_td ltx_align_left"><span id="S4.T4.8.4.9.1.1" class="ltx_text ltx_font_bold">RMU</span></td>
<td id="S4.T4.8.4.9.2" class="ltx_td ltx_align_center">.28</td>
<td id="S4.T4.8.4.9.3" class="ltx_td ltx_align_center">.29</td>
<td id="S4.T4.8.4.9.4" class="ltx_td ltx_align_center">.29</td>
<td id="S4.T4.8.4.9.5" class="ltx_td ltx_align_center">.04</td>
<td id="S4.T4.8.4.9.6" class="ltx_td ltx_align_center">.01</td>
<td id="S4.T4.8.4.9.7" class="ltx_td ltx_align_center">.00</td>
<td id="S4.T4.8.4.9.8" class="ltx_td ltx_align_center">.43</td>
<td id="S4.T4.8.4.9.9" class="ltx_td ltx_align_center">.44</td>
<td id="S4.T4.8.4.9.10" class="ltx_td ltx_align_center">.44</td>
<td id="S4.T4.8.4.9.11" class="ltx_td ltx_align_center">.01</td>
<td id="S4.T4.8.4.9.12" class="ltx_td ltx_align_center">.00</td>
<td id="S4.T4.8.4.9.13" class="ltx_td ltx_align_center">.04</td>
<td id="S4.T4.8.4.9.14" class="ltx_td ltx_align_center"><span id="S4.T4.8.4.9.14.1" class="ltx_text ltx_font_bold">5</span></td>
</tr>
<tr id="S4.T4.8.4.10" class="ltx_tr">
<td id="S4.T4.8.4.10.1" class="ltx_td ltx_align_left ltx_border_bb ltx_border_t"><span id="S4.T4.8.4.10.1.1" class="ltx_text ltx_font_italic"># Wins</span></td>
<td id="S4.T4.8.4.10.2" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.8.4.10.3" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.8.4.10.4" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.8.4.10.5" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.T4.8.4.10.5.1" class="ltx_text ltx_font_bold">2</span></td>
<td id="S4.T4.8.4.10.6" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.T4.8.4.10.7" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.8.4.10.8" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t"><span id="S4.T4.8.4.10.8.1" class="ltx_text ltx_font_bold">2</span></td>
<td id="S4.T4.8.4.10.9" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.T4.8.4.10.10" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">0</td>
<td id="S4.T4.8.4.10.11" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.8.4.10.12" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.8.4.10.13" class="ltx_td ltx_align_center ltx_border_bb ltx_border_t">1</td>
<td id="S4.T4.8.4.10.14" class="ltx_td ltx_border_bb ltx_border_t"></td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_table"><span id="S4.T4.11.1.1" class="ltx_text" style="font-size:90%;">Table 4</span>: </span><span id="S4.T4.12.2" class="ltx_text" style="font-size:90%;">Composability metrics for Machine Unlearning composed with Model Editing. We study compression at 25% sparsity and 4-bit quantization. Note that lower values of WMDP MCS indicate more unlearning and therefore better performance while higher values are better for the other metrics’ MCS. <span id="S4.T4.12.2.1" class="ltx_text ltx_font_bold">Key Takeaways:</span> Editing and unlearning are composable for GD and RMU, with RMU having the lowest Order Sensitivity. These results demonstrate that different techniques can modify LMs’ knowledge sequentially.
</span></figcaption>
</figure>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1"><svg id="S4.SS3.5.pic1" class="ltx_picture ltx_figure_panel" height="13.38" overflow="visible" version="1.1" width="13.38"><g transform="translate(0,13.38) matrix(1 0 0 -1 0 0) translate(6.69,0) translate(0,6.69)" fill="#000000" stroke="#000000" stroke-width="0.4pt"><path d="M 6.42 0 C 6.42 3.54 3.54 6.42 0 6.42 C -3.54 6.42 -6.42 3.54 -6.42 0 C -6.42 -3.54 -3.54 -6.42 0 -6.42 C 3.54 -6.42 6.42 -3.54 6.42 0 Z M 0 0" style="fill:none"></path><g transform="matrix(1.0 0.0 0.0 1.0 -3.46 -4.46)" fill="#000000" stroke="#000000"><foreignObject width="6.92" height="8.92" transform="matrix(1 0 0 -1 0 16.6)" overflow="visible"><span id="S4.SS3.5.pic1.1.1.1.1.1" class="ltx_text">7</span></foreignObject></g></g></svg></div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<p id="S4.SS3.5.8" class="ltx_p ltx_figure_panel"><span id="S4.SS3.5.8.1" class="ltx_text ltx_font_bold">Editing and unlearning are highly composable for some unlearning methods.</span>
Table <a href="#S4.T4" title="Table 4 ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4</span></a> shows that the editing metrics’ Order-free Error and Order Sensitivity vary greatly between unlearning techniques. RMU has a lower overall Order-free Error and Order Sensitivity than every other unlearning technique across all editors. Similar to Section <a href="#S4.SS2" title="4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4.2</span></a>, we find that GD can perform comparably to RMU but with a worse Order Sensitivity. GA’s catastrophic forgetting makes it unsuitable for composition. Excluding GA, editing does not disrupt the model’s ability to forget the unlearning target, possibly due to knowledge editing’s relatively surgical modification to the LM and targeting knowledge semantically distinct from the WMDP unlearning target.
Together, these results imply that editing and unlearning can be composable, but composability depends most on the unlearning technique.</p>
</div>
<div class="ltx_flex_break"></div>
<div class="ltx_flex_cell ltx_flex_size_1">
<section id="S4.SS4" class="ltx_subsection ltx_figure_panel">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">4.4 </span>General Trends in Composability</h3>

<div id="S4.SS4.p1" class="ltx_para">
<p id="S4.SS4.p1.1" class="ltx_p">We finally summarize trends revealed when considering all findings.</p>
</div>
<div id="S4.SS4.p2" class="ltx_para ltx_noindent">
<p id="S4.SS4.p2.1" class="ltx_p"><span id="S4.SS4.p2.1.1" class="ltx_text ltx_font_bold">Compression consistently hinders other interventions.</span>
Compressing an edited model can regress editing performance. Applying unlearning to a compressed model is often more challenging than unlearning the base model. These trends suggest that compression may modify how models encode knowledge in a way that makes targeted updates more challenging. Previous works have studied how compression can cause models to forget some knowledge <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib62" title="" class="ltx_ref">62</a>, <a href="#bib.bib63" title="" class="ltx_ref">63</a>, <a href="#bib.bib64" title="" class="ltx_ref">64</a>, <a href="#bib.bib65" title="" class="ltx_ref">65</a>, <a href="#bib.bib66" title="" class="ltx_ref">66</a>]</cite>. Others have studied how editing modifies model internals in a way that makes the model more fragile <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib67" title="" class="ltx_ref">67</a>, <a href="#bib.bib68" title="" class="ltx_ref">68</a>]</cite>. However, the relationships between <span id="S4.SS4.p2.1.2" class="ltx_text ltx_font_italic">how models are compressed</span> and <span id="S4.SS4.p2.1.3" class="ltx_text ltx_font_italic">how knowledge is modified</span> remain underexplored. An improved understanding of these dynamics may help guide more composable interventions and shed further light on how LMs encode knowledge.</p>
</div>
<div id="S4.SS4.p3" class="ltx_para ltx_noindent">
<p id="S4.SS4.p3.1" class="ltx_p"><span id="S4.SS4.p3.1.1" class="ltx_text ltx_font_bold">RMU is the most composable unlearning technique.</span>
RMU has comparable unlearning performance to GD without regressing overall utility as harshly and while being more order invariant. A possible factor is the RMU training objective, which directs activations for inputs related to the unrelating to a random direction in activation space while minimally modifying unrelated activations.
RMU achieves strong performance, even when only modifying the LM’s third layer.
These results suggest that intervening on model representations <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib69" title="" class="ltx_ref">69</a>]</cite> is a promising direction for effective and composable interventions.</p>
</div>
<div id="S4.SS4.p4" class="ltx_para ltx_noindent">
<p id="S4.SS4.p4.1" class="ltx_p"><span id="S4.SS4.p4.1.1" class="ltx_text ltx_font_bold">MMLU is insufficient for measuring composability.</span>
MMLU is a common measure of a model’s general performance.
However, similar scores on MMLU can mask significant differences in intervention performance.
Across the board, interventions appear far more composable on MMLU than other metrics.
We therefore advocate for thorough multi-metric and multi-dataset evaluations for composable interventions.</p>
</div>
<section id="S5" class="ltx_section">
<h2 class="ltx_title ltx_title_section">
<span class="ltx_tag ltx_tag_section">5 </span>Conclusion</h2>

<div id="S5.p1" class="ltx_para">
<p id="S5.p1.1" class="ltx_p">We conduct an extensive study of composing test-time interventions for language models.
Such interventions are an important step towards keeping expensively trained models up-to-date over extended deployments with respect to factuality, efficiency, and social responsibility.
Our study lays the foundation for a new range of intervention methods that are <span id="S5.p1.1.1" class="ltx_text ltx_font_italic">composable</span> with one another, proposing two metrics for measuring the composability of interventions.
Our setup aligns with the practical need to update the same models multiple times as use cases evolve and training remains expensive.
We deploy this new framework to investigate composability between popular interventions spanning knowledge editing, machine unlearning, and model compression.
We show that these interventions interact in non-trivial ways, pointing to concrete directions for future work.
We further discover guidance for current practice: model compression harms other interventions, unlearning compressed models is especially hard, and the order in which interventions are applied changes their outcomes substantially.
Our composability metrics also pave the way for formal investigations and practical principles.
Finally, our framework is general and naturally extends to a broad range of test-time interventions.</p>
</div>
<section id="Sx1" class="ltx_section">
<h2 class="ltx_title ltx_title_section">Acknowledgements</h2>

<div id="Sx1.p1" class="ltx_para">
<p id="Sx1.p1.1" class="ltx_p">We thank the University of Virginia Research Computing team for providing access to excellent high-performance computing resources.</p>
</div>
<section id="bib" class="ltx_bibliography">
<h2 class="ltx_title ltx_title_bibliography">References</h2>

<ul class="ltx_biblist">
<li id="bib.bib1" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[1]</span>
<span class="ltx_bibblock">
K. Singhal, T. Tu, J. Gottweis, R. Sayres, E. Wulczyn, L. Hou, K. Clark, S. Pfohl, H. Cole-Lewis, D. Neal, <span id="bib.bib1.1.1" class="ltx_text ltx_font_italic">et al.</span>, “Towards expert-level medical question answering with large language models,” <span id="bib.bib1.2.2" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2305.09617</span>, 2023.

</span>
</li>
<li id="bib.bib2" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[2]</span>
<span class="ltx_bibblock">
S. Wu, O. Irsoy, S. Lu, V. Dabravolski, M. Dredze, S. Gehrmann, P. Kambadur, D. Rosenberg, and G. Mann, “Bloomberggpt: A large language model for finance,” <span id="bib.bib2.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2303.17564</span>, 2023.

</span>
</li>
<li id="bib.bib3" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[3]</span>
<span class="ltx_bibblock">
R. Taylor, M. Kardas, G. Cucurull, T. Scialom, A. Hartshorn, E. Saravia, A. Poulton, V. Kerkez, and R. Stojnic, “Galactica: A large language model for science,” <span id="bib.bib3.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2211.09085</span>, 2022.

</span>
</li>
<li id="bib.bib4" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[4]</span>
<span class="ltx_bibblock">
S. Zhong, Z. Huang, S. Gao, W. Wen, L. Lin, M. Zitnik, and P. Zhou, “Let’s think outside the box: Exploring leap-of-thought in large language models with creative humor generation,” <span id="bib.bib4.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2312.02439</span>, 2023.

</span>
</li>
<li id="bib.bib5" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[5]</span>
<span class="ltx_bibblock">
Z. Ji, N. Lee, R. Frieske, T. Yu, D. Su, Y. Xu, E. Ishii, Y. J. Bang, A. Madotto, and P. Fung, “Survey of hallucination in natural language generation,” <span id="bib.bib5.1.1" class="ltx_text ltx_font_italic">ACM Computing Surveys</span>, vol. 55, no. 12, pp. 1–38, 2023.

</span>
</li>
<li id="bib.bib6" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[6]</span>
<span class="ltx_bibblock">
Y. Zhao, J. Zhang, I. Chern, S. Gao, P. Liu, J. He, <span id="bib.bib6.1.1" class="ltx_text ltx_font_italic">et al.</span>, “Felm: Benchmarking factuality evaluation of large language models,” <span id="bib.bib6.2.2" class="ltx_text ltx_font_italic">Advances in Neural Information Processing Systems</span>, vol. 36, 2023.

</span>
</li>
<li id="bib.bib7" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[7]</span>
<span class="ltx_bibblock">
J. Mendelsohn, R. L. Bras, Y. Choi, and M. Sap, “From dogwhistles to bullhorns: Unveiling coded rhetoric with language models,” <span id="bib.bib7.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2305.17174</span>, 2023.

</span>
</li>
<li id="bib.bib8" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[8]</span>
<span class="ltx_bibblock">
D. Jain, P. Kumar, S. Gehman, X. Zhou, T. Hartvigsen, and M. Sap, “PolygloToxicityPrompts: Multilingual evaluation of neural toxic degeneration in large language models,” <span id="bib.bib8.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2405.09373</span>, 2024.

</span>
</li>
<li id="bib.bib9" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[9]</span>
<span class="ltx_bibblock">
T. Hartvigsen, S. Gabriel, H. Palangi, M. Sap, D. Ray, and E. Kamar, “Toxigen: A large-scale machine-generated dataset for adversarial and implicit hate speech detection,” in <span id="bib.bib9.1.1" class="ltx_text ltx_font_italic">Annual Meeting of the Association for Computational Linguistics</span>, 2022.

</span>
</li>
<li id="bib.bib10" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[10]</span>
<span class="ltx_bibblock">
The President of the United States, “Executive order 14110 of october 30, 2023: Safe, secure, and trustworthy development and use of artificial intelligence.” Federal Register, 2023.

</span>
<span class="ltx_bibblock">Published November 1, 2023.

</span>
</li>
<li id="bib.bib11" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[11]</span>
<span class="ltx_bibblock">
J. Tack, J. Kim, E. Mitchell, J. Shin, Y. W. Teh, and J. R. Schwarz, “Online adaptation of language models with a memory of amortized contexts,” <span id="bib.bib11.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2403.04317</span>, 2024.

</span>
</li>
<li id="bib.bib12" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[12]</span>
<span class="ltx_bibblock">
M. Grynbaum and R. Mac, “The times sues openai and microsoft over a.i. use of copyrighted work,” <span id="bib.bib12.1.1" class="ltx_text ltx_font_italic">The New York Times</span>, 2023.

</span>
</li>
<li id="bib.bib13" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[13]</span>
<span class="ltx_bibblock">
J. Kaddour, J. Harris, M. Mozes, H. Bradley, R. Raileanu, and R. McHardy, “Challenges and applications of large language models,” <span id="bib.bib13.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2307.10169</span>, 2023.

</span>
</li>
<li id="bib.bib14" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[14]</span>
<span class="ltx_bibblock">
X. Zhu, J. Li, Y. Liu, C. Ma, and W. Wang, “A survey on model compression for large language models,” <span id="bib.bib14.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2308.07633</span>, 2023.

</span>
</li>
<li id="bib.bib15" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[15]</span>
<span class="ltx_bibblock">
E. Frantar, S. Ashkboos, T. Hoefler, and D. Alistarh, “GPTQ: Accurate post-training quantization for generative pre-trained transformers,” in <span id="bib.bib15.1.1" class="ltx_text ltx_font_italic">International Conference on Learning Representations</span>, 2023.

</span>
</li>
<li id="bib.bib16" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[16]</span>
<span class="ltx_bibblock">
E. Frantar and D. Alistarh, “SparseGPT: Massive language models can be accurately pruned in one-shot,” in <span id="bib.bib16.1.1" class="ltx_text ltx_font_italic">International Conference on Machine Learning</span>, 2023.

</span>
</li>
<li id="bib.bib17" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[17]</span>
<span class="ltx_bibblock">
V. Mazzia, A. Pedrani, A. Caciolai, K. Rottmann, and D. Bernardi, “A survey on knowledge editing of neural networks,” <span id="bib.bib17.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2310.19704</span>, 2023.

</span>
</li>
<li id="bib.bib18" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[18]</span>
<span class="ltx_bibblock">
T. Hartvigsen, S. Sankaranarayanan, H. Palangi, Y. Kim, and M. Ghassemi, “Aging with grace: Lifelong model editing with discrete key-value adaptors,” in <span id="bib.bib18.1.1" class="ltx_text ltx_font_italic">Advances in Neural Information Processing Systems</span>, 2023.

</span>
</li>
<li id="bib.bib19" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[19]</span>
<span class="ltx_bibblock">
K. Meng, D. Bau, A. Andonian, and Y. Belinkov, “Locating and editing factual associations in gpt,” in <span id="bib.bib19.1.1" class="ltx_text ltx_font_italic">Advances in Neural Information Processing Systems</span>, 2022.

</span>
</li>
<li id="bib.bib20" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[20]</span>
<span class="ltx_bibblock">
K. Meng, A. S. Sharma, A. Andonian, Y. Belinkov, and D. Bau, “Mass-editing memory in a transformer,” in <span id="bib.bib20.1.1" class="ltx_text ltx_font_italic">International Conference on Learning Representations</span>, 2023.

</span>
</li>
<li id="bib.bib21" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[21]</span>
<span class="ltx_bibblock">
L. Yu, Q. Chen, J. Zhou, and L. He, “Melo: Enhancing model editing with neuron-indexed dynamic lora,” in <span id="bib.bib21.1.1" class="ltx_text ltx_font_italic">Proceedings of the AAAI Conference on Artificial Intelligence</span>, vol. 38, pp. 19449–19457, 2024.

</span>
</li>
<li id="bib.bib22" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[22]</span>
<span class="ltx_bibblock">
C. Tan, G. Zhang, and J. Fu, “Massive editing for large language models via meta learning,” in <span id="bib.bib22.1.1" class="ltx_text ltx_font_italic">International Conference on Learning Representations</span>, 2024.

</span>
</li>
<li id="bib.bib23" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[23]</span>
<span class="ltx_bibblock">
J. Welbl, A. Glaese, J. Uesato, S. Dathathri, J. Mellor, L. A. Hendricks, K. Anderson, P. Kohli, B. Coppin, and P.-S. Huang, “Challenges in detoxifying language models,” in <span id="bib.bib23.1.1" class="ltx_text ltx_font_italic">Findings of the Association for Computational Linguistics: EMNLP 2021</span>, pp. 2447–2469, 2021.

</span>
</li>
<li id="bib.bib24" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[24]</span>
<span class="ltx_bibblock">
C. Yu, S. Jeoung, A. Kasi, P. Yu, and H. Ji, “Unlearning bias in language models by partitioning gradients,” in <span id="bib.bib24.1.1" class="ltx_text ltx_font_italic">Findings of the Association for Computational Linguistics: ACL 2023</span>, pp. 6032–6048, 2023.

</span>
</li>
<li id="bib.bib25" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[25]</span>
<span class="ltx_bibblock">
B. Wang, W. Ping, C. Xiao, P. Xu, M. Patwary, M. Shoeybi, B. Li, A. Anandkumar, and B. Catanzaro, “Exploring the limits of domain-adaptive training for detoxifying large-scale language models,” <span id="bib.bib25.1.1" class="ltx_text ltx_font_italic">Advances in Neural Information Processing Systems</span>, vol. 35, pp. 35811–35824, 2022.

</span>
</li>
<li id="bib.bib26" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[26]</span>
<span class="ltx_bibblock">
X. Suau, P. Delobelle, K. Metcalf, A. Joulin, N. Apostoloff, L. Zappella, and P. Rodriguez, “Whispering experts: Neural interventions for toxicity mitigation in language models,” in <span id="bib.bib26.1.1" class="ltx_text ltx_font_italic">Forty-first International Conference on Machine Learning</span>.

</span>
</li>
<li id="bib.bib27" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[27]</span>
<span class="ltx_bibblock">
S. Liu, Y. Yao, J. Jia, S. Casper, N. Baracaldo, P. Hase, X. Xu, Y. Yao, H. Li, K. R. Varshney, M. Bansal, S. Koyejo, and Y. Liu, “Rethinking machine unlearning for large language models,” <span id="bib.bib27.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2402.08787, 2024.

</span>
</li>
<li id="bib.bib28" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[28]</span>
<span class="ltx_bibblock">
A. Lynch, P. Guo, A. Ewart, S. Casper, and D. Hadfield-Menell, “Eight methods to evaluate robust unlearning in llms,” <span id="bib.bib28.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2402.16835</span>, 2024.

</span>
</li>
<li id="bib.bib29" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[29]</span>
<span class="ltx_bibblock">
R. Eldan and M. Russinovich, “Who’s harry potter? approximate unlearning in llms,” <span id="bib.bib29.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2310.02238, 2023.

</span>
</li>
<li id="bib.bib30" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[30]</span>
<span class="ltx_bibblock">
C. Matzken, S. Eger, and I. Habernal, “Trade-offs between fairness and privacy in language modeling,” <span id="bib.bib30.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2305.14936</span>, 2023.

</span>
</li>
<li id="bib.bib31" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[31]</span>
<span class="ltx_bibblock">
Z. Xu, Z. Liu, B. Chen, Y. Tang, J. Wang, K. Zhou, X. Hu, and A. Shrivastava, “Compress, then prompt: Improving accuracy-efficiency trade-off of llm inference with transferable prompt,” <span id="bib.bib31.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2305.11186</span>, 2023.

</span>
</li>
<li id="bib.bib32" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[32]</span>
<span class="ltx_bibblock">
Y. Li, Y. Yu, C. Liang, P. He, N. Karampatziakis, W. Chen, and T. Zhao, “LoftQ: Lora-fine-tuning-aware quantization for large language models,” in <span id="bib.bib32.1.1" class="ltx_text ltx_font_italic">International Conference on Learning Representations</span>, 2024.

</span>
</li>
<li id="bib.bib33" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[33]</span>
<span class="ltx_bibblock">
H. Touvron, L. Martin, K. Stone, P. Albert, A. Almahairi, Y. Babaei, N. Bashlykov, S. Batra, P. Bhargava, S. Bhosale, <span id="bib.bib33.1.1" class="ltx_text ltx_font_italic">et al.</span>, “Llama 2: Open foundation and fine-tuned chat models,” <span id="bib.bib33.2.2" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2307.09288</span>, 2023.

</span>
</li>
<li id="bib.bib34" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[34]</span>
<span class="ltx_bibblock">
E. J. Hu, Y. Shen, P. Wallis, Z. Allen-Zhu, Y. Li, S. Wang, L. Wang, and W. Chen, “Lora: Low-rank adaptation of large language models,” <span id="bib.bib34.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2106.09685</span>, 2021.

</span>
</li>
<li id="bib.bib35" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[35]</span>
<span class="ltx_bibblock">
O. Levy, M. Seo, E. Choi, and L. Zettlemoyer, “Zero-shot relation extraction via reading comprehension,” in <span id="bib.bib35.1.1" class="ltx_text ltx_font_italic">Proceedings of the 21st Conference on Computational Natural Language Learning (CoNLL 2017)</span>, pp. 333–342, 2017.

</span>
</li>
<li id="bib.bib36" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[36]</span>
<span class="ltx_bibblock">
M. Sun, Z. Liu, A. Bair, and J. Z. Kolter, “A simple and effective pruning approach for large language models,” <span id="bib.bib36.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2306.11695</span>, 2023.

</span>
</li>
<li id="bib.bib37" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[37]</span>
<span class="ltx_bibblock">
J. Lin, J. Tang, H. Tang, S. Yang, X. Dang, and S. Han, “AWQ: Activation-aware weight quantization for llm compression and acceleration,” <span id="bib.bib37.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2306.00978</span>, 2023.

</span>
</li>
<li id="bib.bib38" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[38]</span>
<span class="ltx_bibblock">
J. Jang, D. Yoon, S. Yang, S. Cha, M. Lee, L. Logeswaran, and M. Seo, “Knowledge unlearning for mitigating privacy risks in language models,” in <span id="bib.bib38.1.1" class="ltx_text ltx_font_italic">Annual Meeting of the Association for Computational Linguistics</span>, 2022.

</span>
</li>
<li id="bib.bib39" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[39]</span>
<span class="ltx_bibblock">
P. Maini, Z. Feng, A. Schwarzschild, Z. C. Lipton, and J. Z. Kolter, “Tofu: A task of fictitious unlearning for llms,” <span id="bib.bib39.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2401.06121, 2024.

</span>
</li>
<li id="bib.bib40" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[40]</span>
<span class="ltx_bibblock">
N. Li, A. Pan, A. Gopal, S. Yue, D. Berrios, A. Gatti, J. D. Li, A.-K. Dombrowski, S. Goel, L. Phan, G. Mukobi, N. Helm-Burger, R. Lababidi, L. Justen, A. B. Liu, M. Chen, I. Barrass, O. Zhang, X. Zhu, R. Tamirisa, B. Bharathi, A. Khoja, Z. Zhao, A. Herbert-Voss, C. B. Breuer, S. Marks, O. Patel, A. Zou, M. Mazeika, Z. Wang, P. Oswal, W. Liu, A. A. Hunt, J. Tienken-Harder, K. Y. Shih, K. Talley, J. Guan, R. Kaplan, I. Steneker, D. Campbell, B. Jokubaitis, A. Levinson, J. Wang, W. Qian, K. K. Karmakar, S. Basart, S. Fitz, M. Levine, P. Kumaraguru, U. Tupakula, V. Varadharajan, Y. Shoshitaishvili, J. Ba, K. M. Esvelt, A. Wang, and D. Hendrycks, “The wmdp benchmark: Measuring and reducing malicious use with unlearning,” 2024.

</span>
</li>
<li id="bib.bib41" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[41]</span>
<span class="ltx_bibblock">
G. Gangadhar and K. Stratos, “Model editing by pure fine-tuning,” 2024.

</span>
</li>
<li id="bib.bib42" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[42]</span>
<span class="ltx_bibblock">
C.-H. Hsueh, P. K.-M. Huang, T.-H. Lin, C.-W. Liao, H.-C. Fang, C.-W. Huang, and Y.-N. Chen, “Editing the mind of giants: An in-depth exploration of pitfalls of knowledge editing in large language models,” <span id="bib.bib42.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2406.01436</span>, 2024.

</span>
</li>
<li id="bib.bib43" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[43]</span>
<span class="ltx_bibblock">
W. Hua, J. Guo, M. Dong, H. Zhu, P. Ng, and Z. Wang, “Propagation and pitfalls: Reasoning-based assessment of knowledge editing through counterfactual tasks,” <span id="bib.bib43.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2401.17585</span>, 2024.

</span>
</li>
<li id="bib.bib44" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[44]</span>
<span class="ltx_bibblock">
Y. Yao, P. Wang, B. Tian, S. Cheng, Z. Li, S. Deng, H. Chen, and N. Zhang, “Editing large language models: Problems, methods, and opportunities,” 2023.

</span>
</li>
<li id="bib.bib45" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[45]</span>
<span class="ltx_bibblock">
A. Karamolegkou, J. Li, L. Zhou, and A. Sogaard, “Copyright violations and large language models,” <span id="bib.bib45.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2310.13771, 2023.

</span>
</li>
<li id="bib.bib46" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[46]</span>
<span class="ltx_bibblock">
M. Meeus, I. Shilov, M. Faysse, and Y.-A. de Montjoye, “Copyright traps for large language models,” <span id="bib.bib46.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2402.09363, 2024.

</span>
</li>
<li id="bib.bib47" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[47]</span>
<span class="ltx_bibblock">
E. Sheng, K.-W. Chang, P. Natarajan, and N. Peng, “The woman worked as a babysitter: On biases in language generation,” <span id="bib.bib47.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/1909.01326, 2019.

</span>
</li>
<li id="bib.bib48" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[48]</span>
<span class="ltx_bibblock">
S. Gehman, S. Gururangan, M. Sap, Y. Choi, and N. A. Smith, “Realtoxicityprompts: Evaluating neural toxic degeneration in language models,” in <span id="bib.bib48.1.1" class="ltx_text ltx_font_italic">Findings</span>, 2020.

</span>
</li>
<li id="bib.bib49" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[49]</span>
<span class="ltx_bibblock">
N. Carlini, C. Liu, Ú. Erlingsson, J. Kos, and D. X. Song, “The secret sharer: Evaluating and testing unintended memorization in neural networks,” in <span id="bib.bib49.1.1" class="ltx_text ltx_font_italic">USENIX Security Symposium</span>, 2018.

</span>
</li>
<li id="bib.bib50" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[50]</span>
<span class="ltx_bibblock">
D. Ippolito, F. Tramèr, M. Nasr, C. Zhang, M. Jagielski, K. Lee, C. A. Choquette-Choo, and N. Carlini, “Preventing verbatim memorization in language models gives a false sense of privacy,” <span id="bib.bib50.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2210.17546, 2022.

</span>
</li>
<li id="bib.bib51" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[51]</span>
<span class="ltx_bibblock">
S. Biderman, U. S. Prashanth, L. Sutawika, H. Schoelkopf, Q. G. Anthony, S. Purohit, and E. Raf, “Emergent and predictable memorization in large language models,” <span id="bib.bib51.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2304.11158, 2023.

</span>
</li>
<li id="bib.bib52" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[52]</span>
<span class="ltx_bibblock">
U. S. Prashanth, A. Deng, K. O’Brien, V. JyothirS, M. A. Khan, J. Borkar, C. A. Choquette-Choo, J. R. Fuehne, S. Biderman, T. Ke, K. Lee, and N. Saphra, “Recite, reconstruct, recollect: Memorization in lms as a multifaceted phenomenon,” 2024.

</span>
</li>
<li id="bib.bib53" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[53]</span>
<span class="ltx_bibblock">
Y. Cao and J. Yang, “Towards making systems forget with machine unlearning,” <span id="bib.bib53.1.1" class="ltx_text ltx_font_italic">2015 IEEE Symposium on Security and Privacy</span>, pp. 463–480, 2015.

</span>
</li>
<li id="bib.bib54" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[54]</span>
<span class="ltx_bibblock">
M. Dehghani, Y. Tay, A. A. Gritsenko, Z. Zhao, N. Houlsby, F. Diaz, D. Metzler, and O. Vinyals, “The benchmark lottery,” <span id="bib.bib54.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2107.07002, 2021.

</span>
</li>
<li id="bib.bib55" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[55]</span>
<span class="ltx_bibblock">
S. Biderman, H. Schoelkopf, L. Sutawika, L. Gao, J. Tow, B. Abbasi, A. F. Aji, P. S. Ammanamanchi, S. Black, J. Clive, A. DiPofi, J. Etxaniz, B. Fattori, J. Z. Forde, C. Foster, M. Jaiswal, W. Y. Lee, H. Li, C. Lovering, N. Muennighoff, E. Pavlick, J. Phang, A. Skowron, S. Tan, X. Tang, K. A. Wang, G. I. Winata, F. Yvon, and A. Zou, “Lessons from the trenches on reproducible evaluation of language models,” 2024.

</span>
</li>
<li id="bib.bib56" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[56]</span>
<span class="ltx_bibblock">
V. Gupta, D. Pantoja, C. Ross, A. Williams, and M. Ung, “Changing answer order can decrease mmlu accuracy,” 2024.

</span>
</li>
<li id="bib.bib57" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[57]</span>
<span class="ltx_bibblock">
D. Hendrycks, C. Burns, S. Basart, A. Zou, M. Mazeika, D. X. Song, and J. Steinhardt, “Measuring massive multitask language understanding,” <span id="bib.bib57.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2009.03300, 2020.

</span>
</li>
<li id="bib.bib58" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[58]</span>
<span class="ltx_bibblock">
L. Gao, J. Tow, B. Abbasi, S. Biderman, S. Black, A. DiPofi, C. Foster, L. Golding, J. Hsu, A. Le Noac’h, H. Li, K. McDonell, N. Muennighoff, C. Ociepa, J. Phang, L. Reynolds, H. Schoelkopf, A. Skowron, L. Sutawika, E. Tang, A. Thite, B. Wang, K. Wang, and A. Zou, “A framework for few-shot language model evaluation,” 12 2023.

</span>
</li>
<li id="bib.bib59" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[59]</span>
<span class="ltx_bibblock">
AI@Meta, “Llama 3 model card,” 2024.

</span>
</li>
<li id="bib.bib60" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[60]</span>
<span class="ltx_bibblock">
R. Zhang, L. Lin, Y. Bai, and S. Mei, “Negative preference optimization: From catastrophic collapse to effective unlearning,” <span id="bib.bib60.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2404.05868, 2024.

</span>
</li>
<li id="bib.bib61" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[61]</span>
<span class="ltx_bibblock">
Q. Wang, B. Han, P. Yang, J. Zhu, T. Liu, and M. Sugiyama, “Unlearning with control: Assessing real-world utility for large language model unlearning,” 2024.

</span>
</li>
<li id="bib.bib62" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[62]</span>
<span class="ltx_bibblock">
S. Hooker, A. C. Courville, G. Clark, Y. Dauphin, and A. Frome, “What do compressed deep neural networks forget,” <span id="bib.bib62.1.1" class="ltx_text ltx_font_italic">arXiv: Learning</span>, 2019.

</span>
</li>
<li id="bib.bib63" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[63]</span>
<span class="ltx_bibblock">
M. Du, S. Mukherjee, Y. Cheng, M. Shokouhi, X. Hu, and A. H. Awadallah, “What do compressed large language models forget? robustness challenges in model compression,” <span id="bib.bib63.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2110.08419, 2021.

</span>
</li>
<li id="bib.bib64" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[64]</span>
<span class="ltx_bibblock">
A. H. Azeemi, I. A. Qazi, and A. A. Raza, “Data pruning for efficient model pruning in neural machine translation,” in <span id="bib.bib64.1.1" class="ltx_text ltx_font_italic">Conference on Empirical Methods in Natural Language Processing</span>, 2023.

</span>
</li>
<li id="bib.bib65" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[65]</span>
<span class="ltx_bibblock">
D. N. M. Hoang, M. Cho, T. Merth, M. Rastegari, and Z. Wang, “Do compressed llms forget knowledge? an experimental study with practical implications,” 2023.

</span>
</li>
<li id="bib.bib66" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[66]</span>
<span class="ltx_bibblock">
S. Sai, S. Namburi, M. N. Sreedhar, S. Srinivasan, and F. Sala, “The cost of compression: Investigating the impact of compression on parametric knowledge in language models,” <span id="bib.bib66.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2312.00960, 2023.

</span>
</li>
<li id="bib.bib67" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[67]</span>
<span class="ltx_bibblock">
D. Brown, C. Godfrey, C. Nizinski, J. Tu, and H. Kvinge, “Edit at your own risk: evaluating the robustness of edited models to distribution shifts,” 2023.

</span>
</li>
<li id="bib.bib68" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[68]</span>
<span class="ltx_bibblock">
J.-C. Gu, H. Xu, J.-Y. Ma, P. Lu, Z.-H. Ling, K.-W. Chang, and N. Peng, “Model editing harms general abilities of large language models: Regularization to the rescue,” 2024.

</span>
</li>
<li id="bib.bib69" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[69]</span>
<span class="ltx_bibblock">
A. Zou, L. Phan, S. Chen, J. Campbell, P. Guo, R. Ren, A. Pan, X. Yin, M. Mazeika, A.-K. Dombrowski, S. Goel, N. Li, M. J. Byun, Z. Wang, A. Mallen, S. Basart, S. Koyejo, D. Song, M. Fredrikson, Z. Kolter, and D. Hendrycks, “Representation engineering: A top-down approach to ai transparency,” <span id="bib.bib69.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2310.01405, 2023.

</span>
</li>
<li id="bib.bib70" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[70]</span>
<span class="ltx_bibblock">
S. Wang, Y. Zhu, H. Liu, Z. Zheng, C. Chen, and J. Li, “Knowledge editing for large language models: A survey,” <span id="bib.bib70.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2310.16218</span>, 2023.

</span>
</li>
<li id="bib.bib71" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[71]</span>
<span class="ltx_bibblock">
E. Mitchell, C. Lin, A. Bosselut, C. D. Manning, and C. Finn, “Memory-based model editing at scale,” in <span id="bib.bib71.1.1" class="ltx_text ltx_font_italic">International Conference on Machine Learning</span>, PMLR, 2022.

</span>
</li>
<li id="bib.bib72" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[72]</span>
<span class="ltx_bibblock">
A. Sinitsin, V. Plokhotnyuk, D. Pyrkin, S. Popov, and A. Babenko, “Editable neural networks,” in <span id="bib.bib72.1.1" class="ltx_text ltx_font_italic">International Conference on Learning Representations</span>, 2020.

</span>
</li>
<li id="bib.bib73" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[73]</span>
<span class="ltx_bibblock">
E. Mitchell, C. Lin, A. Bosselut, C. Finn, and C. D. Manning, “Fast model editing at scale,” in <span id="bib.bib73.1.1" class="ltx_text ltx_font_italic">International Conference on Learning Representations</span>, 2022.

</span>
</li>
<li id="bib.bib74" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[74]</span>
<span class="ltx_bibblock">
Z. Zhong, Z. Wu, C. D. Manning, C. Potts, and D. Chen, “Mquake: Assessing knowledge editing in language models via multi-hop questions,” in <span id="bib.bib74.1.1" class="ltx_text ltx_font_italic">Proceedings of the 2023 Conference on Empirical Methods in Natural Language Processing</span>, 2023.

</span>
</li>
<li id="bib.bib75" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[75]</span>
<span class="ltx_bibblock">
R. Cohen, E. Biran, O. Yoran, A. Globerson, and M. Geva, “Evaluating the ripple effects of knowledge editing in language models,” <span id="bib.bib75.1.1" class="ltx_text ltx_font_italic">Transactions of the Association for Computational Linguistics</span>, vol. 12, pp. 283–298, 2024.

</span>
</li>
<li id="bib.bib76" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[76]</span>
<span class="ltx_bibblock">
N. De Cao, W. Aziz, and I. Titov, “Editing factual knowledge in language models,” in <span id="bib.bib76.1.1" class="ltx_text ltx_font_italic">Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing</span>, pp. 6491–6506, 2021.

</span>
</li>
<li id="bib.bib77" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[77]</span>
<span class="ltx_bibblock">
Z. Huang, Y. Shen, X. Zhang, J. Zhou, W. Rong, and Z. Xiong, “Transformer-patcher: One mistake worth one neuron,” in <span id="bib.bib77.1.1" class="ltx_text ltx_font_italic">International Conference on Learning Representations</span>, 2023.

</span>
</li>
<li id="bib.bib78" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[78]</span>
<span class="ltx_bibblock">
J.-C. Gu, H.-X. Xu, J.-Y. Ma, P. Lu, Z.-H. Ling, K.-W. Chang, and N. Peng, “Model editing can hurt general abilities of large language models,” <span id="bib.bib78.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2401.04700</span>, 2024.

</span>
</li>
<li id="bib.bib79" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[79]</span>
<span class="ltx_bibblock">
J. Hoelscher-Obermaier, J. Persson, E. Kran, I. Konstas, and F. Barez, “Detecting edit failures in large language models: An improved specificity benchmark,” <span id="bib.bib79.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2305.17553</span>, 2023.

</span>
</li>
<li id="bib.bib80" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[80]</span>
<span class="ltx_bibblock">
Z. Li, N. Zhang, Y. Yao, M. Wang, X. Chen, and H. Chen, “Unveiling the pitfalls of knowledge editing for large language models,” <span id="bib.bib80.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2310.02129</span>, 2023.

</span>
</li>
<li id="bib.bib81" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[81]</span>
<span class="ltx_bibblock">
R. Hazra, S. Layek, S. Banerjee, and S. Poria, “Sowing the wind, reaping the whirlwind: The impact of editing language models,” <span id="bib.bib81.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2401.10647</span>, 2024.

</span>
</li>
<li id="bib.bib82" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[82]</span>
<span class="ltx_bibblock">
D. Brown, C. Godfrey, C. Nizinski, J. Tu, and H. Kvinge, “Robustness of edited neural networks,” <span id="bib.bib82.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2303.00046</span>, 2023.

</span>
</li>
<li id="bib.bib83" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[83]</span>
<span class="ltx_bibblock">
P. Hase, M. Bansal, B. Kim, and A. Ghandeharioun, “Does localization inform editing? surprising differences in causality-based localization vs. knowledge editing in language models,” in <span id="bib.bib83.1.1" class="ltx_text ltx_font_italic">Advances in Neural Information Processing Systems</span>, 2023.

</span>
</li>
<li id="bib.bib84" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[84]</span>
<span class="ltx_bibblock">
D. Powell, W. Gerych, and T. Hartvigsen, “TAXI: Evaluating categorical knowledge editing for language models,” in <span id="bib.bib84.1.1" class="ltx_text ltx_font_italic">Findings of the Association for Computational Linguistics: ACL 2024</span>, 2024.

</span>
</li>
<li id="bib.bib85" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[85]</span>
<span class="ltx_bibblock">
T. B. Shaik, X. Tao, H. Xie, L. Li, X. Zhu, and Q. Li, “Exploring the landscape of machine unlearning: A comprehensive survey and taxonomy,” <span id="bib.bib85.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2305.06360, 2023.

</span>
</li>
<li id="bib.bib86" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[86]</span>
<span class="ltx_bibblock">
J. Zhao, Z. Deng, D. Madras, J. Zou, and M. Ren, “Learning and forgetting unsafe examples in large language models,” <span id="bib.bib86.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2312.12736, 2023.

</span>
</li>
<li id="bib.bib87" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[87]</span>
<span class="ltx_bibblock">
Z. Liu, G. Dou, Z. Tan, Y. Tian, and M. Jiang, “Towards safer large language models through machine unlearning,” <span id="bib.bib87.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2402.10058, 2024.

</span>
</li>
<li id="bib.bib88" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[88]</span>
<span class="ltx_bibblock">
J. Jia, J. Liu, P. Ram, Y. Yao, G. Liu, Y. Liu, P. Sharma, and S. Liu, “Model sparsity can simplify machine unlearning,” in <span id="bib.bib88.1.1" class="ltx_text ltx_font_italic">Neural Information Processing Systems</span>, 2023.

</span>
</li>
<li id="bib.bib89" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[89]</span>
<span class="ltx_bibblock">
G. Xiao, J. Lin, M. Seznec, H. Wu, J. Demouth, and S. Han, “Smoothquant: Accurate and efficient post-training quantization for large language models,” in <span id="bib.bib89.1.1" class="ltx_text ltx_font_italic">International Conference on Machine Learning</span>, pp. 38087–38099, PMLR, 2023.

</span>
</li>
<li id="bib.bib90" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[90]</span>
<span class="ltx_bibblock">
L. Yin, Y. Wu, Z. Zhang, C.-Y. Hsieh, Y. Wang, Y. Jia, M. Pechenizkiy, Y. Liang, Z. Wang, and S. Liu, “Outlier weighed layerwise sparsity (owl): A missing secret sauce for pruning llms to high sparsity,” <span id="bib.bib90.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2310.05175</span>, 2023.

</span>
</li>
<li id="bib.bib91" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[91]</span>
<span class="ltx_bibblock">
X. Ma, G. Fang, and X. Wang, “Llm-pruner: On the structural pruning of large language models,” <span id="bib.bib91.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2305.11627</span>, 2023.

</span>
</li>
<li id="bib.bib92" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[92]</span>
<span class="ltx_bibblock">
P. Wang, Z. Wang, Z. Li, Y. Gao, B. Yin, and X. Ren, “Scott: Self-consistent chain-of-thought distillation,” <span id="bib.bib92.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2305.01879</span>, 2023.

</span>
</li>
<li id="bib.bib93" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[93]</span>
<span class="ltx_bibblock">
S. Saha, P. Hase, and M. Bansal, “Can language models teach weaker agents? teacher explanations improve students via theory of mind,” <span id="bib.bib93.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2306.09299</span>, 2023.

</span>
</li>
<li id="bib.bib94" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[94]</span>
<span class="ltx_bibblock">
C.-Y. Hsieh, C.-L. Li, C.-K. Yeh, H. Nakhost, Y. Fujii, A. Ratner, R. Krishna, C.-Y. Lee, and T. Pfister, “Distilling step-by-step! outperforming larger language models with less training data and smaller model sizes,” <span id="bib.bib94.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2305.02301</span>, 2023.

</span>
</li>
<li id="bib.bib95" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[95]</span>
<span class="ltx_bibblock">
M. Xu, Y. L. Xu, and D. P. Mandic, “Tensorgpt: Efficient compression of the embedding layer in llms based on the tensor-train decomposition,” <span id="bib.bib95.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2307.00526</span>, 2023.

</span>
</li>
<li id="bib.bib96" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[96]</span>
<span class="ltx_bibblock">
P. Sharma, J. T. Ash, and D. Misra, “The truth is in there: Improving reasoning in language models with layer-selective rank reduction,” <span id="bib.bib96.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2312.13558</span>, 2023.

</span>
</li>
<li id="bib.bib97" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[97]</span>
<span class="ltx_bibblock">
Z. Yao, R. Yazdani Aminabadi, M. Zhang, X. Wu, C. Li, and Y. He, “Zeroquant: Efficient and affordable post-training quantization for large-scale transformers,” <span id="bib.bib97.1.1" class="ltx_text ltx_font_italic">Advances in Neural Information Processing Systems</span>, vol. 35, pp. 27168–27183, 2022.

</span>
</li>
<li id="bib.bib98" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[98]</span>
<span class="ltx_bibblock">
T. Dettmers, M. Lewis, Y. Belkada, and L. Zettlemoyer, “Llm. int8 (): 8-bit matrix multiplication for transformers at scale,” <span id="bib.bib98.1.1" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2208.07339</span>, 2022.

</span>
</li>
<li id="bib.bib99" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[99]</span>
<span class="ltx_bibblock">
P. Wang, N. Zhang, X. Xie, Y. Yao, B. Tian, M. Wang, Z. Xi, S. Cheng, K. Liu, G. Zheng, <span id="bib.bib99.1.1" class="ltx_text ltx_font_italic">et al.</span>, “Easyedit: An easy-to-use knowledge editing framework for large language models,” <span id="bib.bib99.2.2" class="ltx_text ltx_font_italic">arXiv preprint arXiv:2308.07269</span>, 2023.

</span>
</li>
<li id="bib.bib100" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[100]</span>
<span class="ltx_bibblock">
S. Merity, C. Xiong, J. Bradbury, and R. Socher, “Pointer sentinel mixture models,” in <span id="bib.bib100.1.1" class="ltx_text ltx_font_italic">International Conference on Learning Representations</span>, 2016.

</span>
</li>
<li id="bib.bib101" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[101]</span>
<span class="ltx_bibblock">
A. Golatkar, A. Achille, and S. Soatto, “Eternal sunshine of the spotless net: Selective forgetting in deep networks,” <span id="bib.bib101.1.1" class="ltx_text ltx_font_italic">2020 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)</span>, pp. 9301–9309, 2019.

</span>
</li>
<li id="bib.bib102" class="ltx_bibitem">
<span class="ltx_tag ltx_role_refnum ltx_tag_bibitem">[102]</span>
<span class="ltx_bibblock">
J. Yao, E. Chien, M. Du, X. Niu, T. Wang, Z. Cheng, and X. Yue, “Machine unlearning of pre-trained large language models,” <span id="bib.bib102.1.1" class="ltx_text ltx_font_italic">ArXiv</span>, vol. abs/2402.15159, 2024.

</span>
</li>
</ul>
</section>
<div class="ltx_pagination ltx_role_newpage"></div>
<section id="Ax1" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">Appendix</h2>

</section>
<section id="A1" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">
<span class="ltx_tag ltx_tag_appendix">Appendix A </span>Limitations</h2>

<div id="A1.p1" class="ltx_para">
<p id="A1.p1.1" class="ltx_p">While our work provides the first look at how state-of-the-art methods compose, there are far more possible combinations of interventions, methods, models, and datasets than we can consider in one work.
So some popular methods are omitted.
We also prioritize broad experiments with many different intervention methods.
We hope future works will closely examine individual methods to study, improve, and extend beyond the mechanisms that drive their composability.
Lastly, we only experiment with Llama3-8B.
While our code is extendable and Llama3-8B is popular, it is unknown whether our findings generalize between language models.
We hope our codebase empowers others to explore more model sizes and architectures in future research.

</p>
</div>
</section>
<section id="A2" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">
<span class="ltx_tag ltx_tag_appendix">Appendix B </span>Background on Interventions</h2>

<div id="A2.p1" class="ltx_para">
<p id="A2.p1.1" class="ltx_p">Designing interventions for language models is a highly-active research area.
We focus on three key types of intervention.</p>
</div>
<div id="A2.p2" class="ltx_para ltx_noindent">
<p id="A2.p2.1" class="ltx_p"><span id="A2.p2.1.1" class="ltx_text ltx_font_bold">Knowledge Editing</span>.
Knowledge editors aim to address factuality decay in LLMs: As the world changes, some facts an LLM learned during training become inaccurate <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib17" title="" class="ltx_ref">17</a>, <a href="#bib.bib70" title="" class="ltx_ref">70</a>]</cite>.
For example, a model trained before 2020 would still predict <span id="A2.p2.1.2" class="ltx_text ltx_font_typewriter">The latest pandemic in the US is</span> <math id="A2.p2.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A2.p2.1.m1.1a"><mo stretchy="false" id="A2.p2.1.m1.1.1" xref="A2.p2.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A2.p2.1.m1.1b"><ci id="A2.p2.1.m1.1.1.cmml" xref="A2.p2.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A2.p2.1.m1.1c">\rightarrow</annotation></semantics></math> “<span id="A2.p2.1.3" class="ltx_text ltx_font_typewriter">Swine Flu</span>” until updated to predict “<span id="A2.p2.1.4" class="ltx_text ltx_font_typewriter">COVID</span>”.
There are four general approaches to model editing: 1) Memory-based methods that cache knowledge <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib71" title="" class="ltx_ref">71</a>, <a href="#bib.bib18" title="" class="ltx_ref">18</a>]</cite>, 2) Locate-then-Edit methods that selectively finetune parameter subsets <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib19" title="" class="ltx_ref">19</a>, <a href="#bib.bib20" title="" class="ltx_ref">20</a>]</cite>, 3) Hypernetwork methods that predict new model weights <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib72" title="" class="ltx_ref">72</a>, <a href="#bib.bib73" title="" class="ltx_ref">73</a>, <a href="#bib.bib22" title="" class="ltx_ref">22</a>]</cite>, and 4) Prompt editors that add facts to prompts <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib74" title="" class="ltx_ref">74</a>, <a href="#bib.bib75" title="" class="ltx_ref">75</a>]</cite>.
Most editors update singular facts <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib73" title="" class="ltx_ref">73</a>, <a href="#bib.bib76" title="" class="ltx_ref">76</a>, <a href="#bib.bib72" title="" class="ltx_ref">72</a>, <a href="#bib.bib19" title="" class="ltx_ref">19</a>]</cite>, though recent works apply many simultaneously <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib20" title="" class="ltx_ref">20</a>, <a href="#bib.bib71" title="" class="ltx_ref">71</a>, <a href="#bib.bib22" title="" class="ltx_ref">22</a>]</cite>.
Others works embrace the practical need for <span id="A2.p2.1.5" class="ltx_text ltx_font_italic">sequential</span> edits <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib77" title="" class="ltx_ref">77</a>, <a href="#bib.bib18" title="" class="ltx_ref">18</a>]</cite>.
Recent works have also begun investigating unintended impacts of editing on pre-trained models <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib78" title="" class="ltx_ref">78</a>, <a href="#bib.bib79" title="" class="ltx_ref">79</a>, <a href="#bib.bib80" title="" class="ltx_ref">80</a>, <a href="#bib.bib81" title="" class="ltx_ref">81</a>, <a href="#bib.bib82" title="" class="ltx_ref">82</a>, <a href="#bib.bib83" title="" class="ltx_ref">83</a>]</cite>, while others are exploring multi-hop editing <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib84" title="" class="ltx_ref">84</a>, <a href="#bib.bib74" title="" class="ltx_ref">74</a>, <a href="#bib.bib75" title="" class="ltx_ref">75</a>]</cite>.
However, these works study impacts on model performance, not interactions with other types of interventions.</p>
</div>
<div id="A2.p3" class="ltx_para ltx_noindent">
<p id="A2.p3.1" class="ltx_p"><span id="A2.p3.1.1" class="ltx_text ltx_font_bold">Machine Unlearning.</span> Machine Unlearning (MU) refers to a broad set of techniques for modifying a model post-training to remove the influence of specific training examples or other undesirable properties<span id="footnote4" class="ltx_note ltx_role_footnote"><sup class="ltx_note_mark">4</sup><span class="ltx_note_outer"><span class="ltx_note_content"><sup class="ltx_note_mark">4</sup><span class="ltx_tag ltx_tag_note">4</span>Most MU techniques for LMs are considered approximate unlearning in that they aim to modify the model to approximate the behavior of it not being trained on the unlearning target without retraining from scratch.</span></span></span> <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib53" title="" class="ltx_ref">53</a>, <a href="#bib.bib85" title="" class="ltx_ref">85</a>, <a href="#bib.bib27" title="" class="ltx_ref">27</a>]</cite>. These undesirable properties are commonly referred to as the unlearning target. Crucially, MU must excel at causing the model to forget the unlearning target without adversely affecting overall utility by reducing performance on other domains <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib78" title="" class="ltx_ref">78</a>]</cite>. Another requirement is that MU must cause models to deeply forget knowledge such that the unlearning target can not be elicited by adversaries or probing <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib28" title="" class="ltx_ref">28</a>]</cite>. With the rise of generative models, recent work has explored MU for forgetting potentially copyrighted work <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib29" title="" class="ltx_ref">29</a>]</cite>, information about individuals <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib39" title="" class="ltx_ref">39</a>]</cite>, and potentially dangerous knowledge <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib86" title="" class="ltx_ref">86</a>, <a href="#bib.bib87" title="" class="ltx_ref">87</a>, <a href="#bib.bib40" title="" class="ltx_ref">40</a>]</cite>. Existing works have studied which types of examples models forget when compressed <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib62" title="" class="ltx_ref">62</a>]</cite>, whether compression hinders unlearning <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib88" title="" class="ltx_ref">88</a>]</cite>, and whether compression in itself can be used as an unlearning technique <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib88" title="" class="ltx_ref">88</a>]</cite>. While related to knowledge editing, the two areas have largely developed independently.</p>
</div>
<div id="A2.p4" class="ltx_para ltx_noindent">
<p id="A2.p4.1" class="ltx_p"><span id="A2.p4.1.1" class="ltx_text ltx_font_bold">Model Compression</span>.
To mitigate the huge computational and memory demands of LLMs, plenty of classical techniques in model compression have been explored, such as quantization <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib15" title="" class="ltx_ref">15</a>, <a href="#bib.bib37" title="" class="ltx_ref">37</a>, <a href="#bib.bib89" title="" class="ltx_ref">89</a>]</cite>, network pruning <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib16" title="" class="ltx_ref">16</a>, <a href="#bib.bib36" title="" class="ltx_ref">36</a>, <a href="#bib.bib90" title="" class="ltx_ref">90</a>, <a href="#bib.bib91" title="" class="ltx_ref">91</a>]</cite>, knowledge distillation <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib92" title="" class="ltx_ref">92</a>, <a href="#bib.bib93" title="" class="ltx_ref">93</a>, <a href="#bib.bib94" title="" class="ltx_ref">94</a>]</cite>, and low-rank factorization <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib34" title="" class="ltx_ref">34</a>, <a href="#bib.bib95" title="" class="ltx_ref">95</a>, <a href="#bib.bib96" title="" class="ltx_ref">96</a>]</cite>. We focus on quantization and network pruning,
which both make in-place updates to pre-trained model weights.
Quantization involves lowering the bit-precision of model parameters, effectively shrinking model size and expediting inference. Early works, such as ZeroQuant <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib97" title="" class="ltx_ref">97</a>]</cite> and LLM.int8 <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib98" title="" class="ltx_ref">98</a>]</cite>, focused on fine-tuning quantization granularity, showing promising results predominantly at higher bit-widths, like 8-bit. More recent innovations like GPTQ <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib15" title="" class="ltx_ref">15</a>]</cite> and AWQ <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib37" title="" class="ltx_ref">37</a>]</cite> have successfully pushed the boundaries by reducing bit-width to as low as 3 or 4 bits per weight, with negligible performance degradation.
Pruning is another compression methods, which aims to eliminate redundant components like weights and channels.
Techniques like SparseGPT <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib16" title="" class="ltx_ref">16</a>]</cite> and Wanda <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib36" title="" class="ltx_ref">36</a>]</cite> successfully maintain much of an LLM’s performance, even when discarding approximately 50% of its weights. OWL <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib90" title="" class="ltx_ref">90</a>]</cite> achieves higher levels of sparsity by reallocating sparsities across layers.</p>
</div>
</section>
<section id="A3" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">
<span class="ltx_tag ltx_tag_appendix">Appendix C </span>Implementation Details</h2>

<div id="A3.p1" class="ltx_para">
<p id="A3.p1.1" class="ltx_p">In this section we outline the set up and hyperparameters used for knowledge editing, unlearning, and compression.</p>
</div>
<section id="A3.SS1" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">C.1 </span>Knowledge Editing Details</h3>

<section id="A3.SS1.SSS1" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.1.1 </span>Knowledge Editing Metrics</h4>

<div id="A3.SS1.SSS1.p1" class="ltx_para">
<p id="A3.SS1.SSS1.p1.1" class="ltx_p">We adopt three widely-used editing metrics and three compression metrics to measure the effectiveness of these interventions. The editing metrics are:</p>
</div>
<div id="A3.SS1.SSS1.p2" class="ltx_para">
<p id="A3.SS1.SSS1.p2.1" class="ltx_p"><math id="A3.SS1.SSS1.p2.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS1.SSS1.p2.1.m1.1a"><mo id="A3.SS1.SSS1.p2.1.m1.1.1" xref="A3.SS1.SSS1.p2.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS1.p2.1.m1.1b"><ci id="A3.SS1.SSS1.p2.1.m1.1.1.cmml" xref="A3.SS1.SSS1.p2.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS1.p2.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS1.SSS1.p2.1.1" class="ltx_text ltx_font_italic">Edit Success:</span> Assesses the post-edit model’s ability to provide the correct, expected answer. An issue arises here with the different levels of verbosity for the models. An unedited model might return the correct token, but use a differently phrased answer or arrange the correct tokens differently. We therefore adopt and F1-style measure used in recent work <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib18" title="" class="ltx_ref">18</a>]</cite> for this.</p>
</div>
<div id="A3.SS1.SSS1.p3" class="ltx_para">
<p id="A3.SS1.SSS1.p3.3" class="ltx_p">First, we determine the common tokens between the generated response and the ground truth, which are represented by the sets <math id="A3.SS1.SSS1.p3.1.m1.1" class="ltx_Math" alttext="G" display="inline"><semantics id="A3.SS1.SSS1.p3.1.m1.1a"><mi id="A3.SS1.SSS1.p3.1.m1.1.1" xref="A3.SS1.SSS1.p3.1.m1.1.1.cmml">G</mi><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS1.p3.1.m1.1b"><ci id="A3.SS1.SSS1.p3.1.m1.1.1.cmml" xref="A3.SS1.SSS1.p3.1.m1.1.1">𝐺</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS1.p3.1.m1.1c">G</annotation></semantics></math> (generated response tokens without special tokens) and <math id="A3.SS1.SSS1.p3.2.m2.1" class="ltx_Math" alttext="T" display="inline"><semantics id="A3.SS1.SSS1.p3.2.m2.1a"><mi id="A3.SS1.SSS1.p3.2.m2.1.1" xref="A3.SS1.SSS1.p3.2.m2.1.1.cmml">T</mi><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS1.p3.2.m2.1b"><ci id="A3.SS1.SSS1.p3.2.m2.1.1.cmml" xref="A3.SS1.SSS1.p3.2.m2.1.1">𝑇</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS1.p3.2.m2.1c">T</annotation></semantics></math> (target tokens without padding), respectively. The common tokens are given by <math id="A3.SS1.SSS1.p3.3.m3.1" class="ltx_Math" alttext="C=G\cap T" display="inline"><semantics id="A3.SS1.SSS1.p3.3.m3.1a"><mrow id="A3.SS1.SSS1.p3.3.m3.1.1" xref="A3.SS1.SSS1.p3.3.m3.1.1.cmml"><mi id="A3.SS1.SSS1.p3.3.m3.1.1.2" xref="A3.SS1.SSS1.p3.3.m3.1.1.2.cmml">C</mi><mo id="A3.SS1.SSS1.p3.3.m3.1.1.1" xref="A3.SS1.SSS1.p3.3.m3.1.1.1.cmml">=</mo><mrow id="A3.SS1.SSS1.p3.3.m3.1.1.3" xref="A3.SS1.SSS1.p3.3.m3.1.1.3.cmml"><mi id="A3.SS1.SSS1.p3.3.m3.1.1.3.2" xref="A3.SS1.SSS1.p3.3.m3.1.1.3.2.cmml">G</mi><mo id="A3.SS1.SSS1.p3.3.m3.1.1.3.1" xref="A3.SS1.SSS1.p3.3.m3.1.1.3.1.cmml">∩</mo><mi id="A3.SS1.SSS1.p3.3.m3.1.1.3.3" xref="A3.SS1.SSS1.p3.3.m3.1.1.3.3.cmml">T</mi></mrow></mrow><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS1.p3.3.m3.1b"><apply id="A3.SS1.SSS1.p3.3.m3.1.1.cmml" xref="A3.SS1.SSS1.p3.3.m3.1.1"><eq id="A3.SS1.SSS1.p3.3.m3.1.1.1.cmml" xref="A3.SS1.SSS1.p3.3.m3.1.1.1"></eq><ci id="A3.SS1.SSS1.p3.3.m3.1.1.2.cmml" xref="A3.SS1.SSS1.p3.3.m3.1.1.2">𝐶</ci><apply id="A3.SS1.SSS1.p3.3.m3.1.1.3.cmml" xref="A3.SS1.SSS1.p3.3.m3.1.1.3"><intersect id="A3.SS1.SSS1.p3.3.m3.1.1.3.1.cmml" xref="A3.SS1.SSS1.p3.3.m3.1.1.3.1"></intersect><ci id="A3.SS1.SSS1.p3.3.m3.1.1.3.2.cmml" xref="A3.SS1.SSS1.p3.3.m3.1.1.3.2">𝐺</ci><ci id="A3.SS1.SSS1.p3.3.m3.1.1.3.3.cmml" xref="A3.SS1.SSS1.p3.3.m3.1.1.3.3">𝑇</ci></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS1.p3.3.m3.1c">C=G\cap T</annotation></semantics></math>.</p>
</div>
<div id="A3.SS1.SSS1.p4" class="ltx_para">
<p id="A3.SS1.SSS1.p4.1" class="ltx_p">If <math id="A3.SS1.SSS1.p4.1.m1.1" class="ltx_Math" alttext="C" display="inline"><semantics id="A3.SS1.SSS1.p4.1.m1.1a"><mi id="A3.SS1.SSS1.p4.1.m1.1.1" xref="A3.SS1.SSS1.p4.1.m1.1.1.cmml">C</mi><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS1.p4.1.m1.1b"><ci id="A3.SS1.SSS1.p4.1.m1.1.1.cmml" xref="A3.SS1.SSS1.p4.1.m1.1.1">𝐶</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS1.p4.1.m1.1c">C</annotation></semantics></math> is empty is empty, the F1 score is set to 0, as there are no common elements to compare.</p>
</div>
<div id="A3.SS1.SSS1.p5" class="ltx_para">
<p id="A3.SS1.SSS1.p5.1" class="ltx_p">Otherwise, we calculate precision and recall. Precision is the ratio of the number of common tokens to the number of tokens in the generated response, and recall is the ratio of the number of common tokens to the number of target tokens. They are defined as:</p>
<table id="A4.EGx1" class="ltx_equationgroup ltx_eqn_align ltx_eqn_table">

<tbody id="A3.Ex1"><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_td ltx_align_right ltx_eqn_cell"><span id="A3.Ex1.2.1.1.1" class="ltx_text ltx_markedasmath">Precision</span></td>
<td class="ltx_td ltx_align_left ltx_eqn_cell"><math id="A3.Ex1.m2.3" class="ltx_Math" alttext="\displaystyle=\frac{|C|}{|G|}," display="inline"><semantics id="A3.Ex1.m2.3a"><mrow id="A3.Ex1.m2.3.3.1" xref="A3.Ex1.m2.3.3.1.1.cmml"><mrow id="A3.Ex1.m2.3.3.1.1" xref="A3.Ex1.m2.3.3.1.1.cmml"><mi id="A3.Ex1.m2.3.3.1.1.2" xref="A3.Ex1.m2.3.3.1.1.2.cmml"></mi><mo id="A3.Ex1.m2.3.3.1.1.1" xref="A3.Ex1.m2.3.3.1.1.1.cmml">=</mo><mstyle displaystyle="true" id="A3.Ex1.m2.2.2" xref="A3.Ex1.m2.2.2.cmml"><mfrac id="A3.Ex1.m2.2.2a" xref="A3.Ex1.m2.2.2.cmml"><mrow id="A3.Ex1.m2.1.1.1.3" xref="A3.Ex1.m2.1.1.1.2.cmml"><mo stretchy="false" id="A3.Ex1.m2.1.1.1.3.1" xref="A3.Ex1.m2.1.1.1.2.1.cmml">|</mo><mi id="A3.Ex1.m2.1.1.1.1" xref="A3.Ex1.m2.1.1.1.1.cmml">C</mi><mo stretchy="false" id="A3.Ex1.m2.1.1.1.3.2" xref="A3.Ex1.m2.1.1.1.2.1.cmml">|</mo></mrow><mrow id="A3.Ex1.m2.2.2.2.3" xref="A3.Ex1.m2.2.2.2.2.cmml"><mo stretchy="false" id="A3.Ex1.m2.2.2.2.3.1" xref="A3.Ex1.m2.2.2.2.2.1.cmml">|</mo><mi id="A3.Ex1.m2.2.2.2.1" xref="A3.Ex1.m2.2.2.2.1.cmml">G</mi><mo stretchy="false" id="A3.Ex1.m2.2.2.2.3.2" xref="A3.Ex1.m2.2.2.2.2.1.cmml">|</mo></mrow></mfrac></mstyle></mrow><mo id="A3.Ex1.m2.3.3.1.2" xref="A3.Ex1.m2.3.3.1.1.cmml">,</mo></mrow><annotation-xml encoding="MathML-Content" id="A3.Ex1.m2.3b"><apply id="A3.Ex1.m2.3.3.1.1.cmml" xref="A3.Ex1.m2.3.3.1"><eq id="A3.Ex1.m2.3.3.1.1.1.cmml" xref="A3.Ex1.m2.3.3.1.1.1"></eq><csymbol cd="latexml" id="A3.Ex1.m2.3.3.1.1.2.cmml" xref="A3.Ex1.m2.3.3.1.1.2">absent</csymbol><apply id="A3.Ex1.m2.2.2.cmml" xref="A3.Ex1.m2.2.2"><divide id="A3.Ex1.m2.2.2.3.cmml" xref="A3.Ex1.m2.2.2"></divide><apply id="A3.Ex1.m2.1.1.1.2.cmml" xref="A3.Ex1.m2.1.1.1.3"><abs id="A3.Ex1.m2.1.1.1.2.1.cmml" xref="A3.Ex1.m2.1.1.1.3.1"></abs><ci id="A3.Ex1.m2.1.1.1.1.cmml" xref="A3.Ex1.m2.1.1.1.1">𝐶</ci></apply><apply id="A3.Ex1.m2.2.2.2.2.cmml" xref="A3.Ex1.m2.2.2.2.3"><abs id="A3.Ex1.m2.2.2.2.2.1.cmml" xref="A3.Ex1.m2.2.2.2.3.1"></abs><ci id="A3.Ex1.m2.2.2.2.1.cmml" xref="A3.Ex1.m2.2.2.2.1">𝐺</ci></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.Ex1.m2.3c">\displaystyle=\frac{|C|}{|G|},</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
<tbody id="A3.Ex2"><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_td ltx_align_right ltx_eqn_cell"><span id="A3.Ex2.2.1.1.1" class="ltx_text ltx_markedasmath">Recall</span></td>
<td class="ltx_td ltx_align_left ltx_eqn_cell"><math id="A3.Ex2.m2.3" class="ltx_Math" alttext="\displaystyle=\frac{|C|}{|T|}." display="inline"><semantics id="A3.Ex2.m2.3a"><mrow id="A3.Ex2.m2.3.3.1" xref="A3.Ex2.m2.3.3.1.1.cmml"><mrow id="A3.Ex2.m2.3.3.1.1" xref="A3.Ex2.m2.3.3.1.1.cmml"><mi id="A3.Ex2.m2.3.3.1.1.2" xref="A3.Ex2.m2.3.3.1.1.2.cmml"></mi><mo id="A3.Ex2.m2.3.3.1.1.1" xref="A3.Ex2.m2.3.3.1.1.1.cmml">=</mo><mstyle displaystyle="true" id="A3.Ex2.m2.2.2" xref="A3.Ex2.m2.2.2.cmml"><mfrac id="A3.Ex2.m2.2.2a" xref="A3.Ex2.m2.2.2.cmml"><mrow id="A3.Ex2.m2.1.1.1.3" xref="A3.Ex2.m2.1.1.1.2.cmml"><mo stretchy="false" id="A3.Ex2.m2.1.1.1.3.1" xref="A3.Ex2.m2.1.1.1.2.1.cmml">|</mo><mi id="A3.Ex2.m2.1.1.1.1" xref="A3.Ex2.m2.1.1.1.1.cmml">C</mi><mo stretchy="false" id="A3.Ex2.m2.1.1.1.3.2" xref="A3.Ex2.m2.1.1.1.2.1.cmml">|</mo></mrow><mrow id="A3.Ex2.m2.2.2.2.3" xref="A3.Ex2.m2.2.2.2.2.cmml"><mo stretchy="false" id="A3.Ex2.m2.2.2.2.3.1" xref="A3.Ex2.m2.2.2.2.2.1.cmml">|</mo><mi id="A3.Ex2.m2.2.2.2.1" xref="A3.Ex2.m2.2.2.2.1.cmml">T</mi><mo stretchy="false" id="A3.Ex2.m2.2.2.2.3.2" xref="A3.Ex2.m2.2.2.2.2.1.cmml">|</mo></mrow></mfrac></mstyle></mrow><mo lspace="0em" id="A3.Ex2.m2.3.3.1.2" xref="A3.Ex2.m2.3.3.1.1.cmml">.</mo></mrow><annotation-xml encoding="MathML-Content" id="A3.Ex2.m2.3b"><apply id="A3.Ex2.m2.3.3.1.1.cmml" xref="A3.Ex2.m2.3.3.1"><eq id="A3.Ex2.m2.3.3.1.1.1.cmml" xref="A3.Ex2.m2.3.3.1.1.1"></eq><csymbol cd="latexml" id="A3.Ex2.m2.3.3.1.1.2.cmml" xref="A3.Ex2.m2.3.3.1.1.2">absent</csymbol><apply id="A3.Ex2.m2.2.2.cmml" xref="A3.Ex2.m2.2.2"><divide id="A3.Ex2.m2.2.2.3.cmml" xref="A3.Ex2.m2.2.2"></divide><apply id="A3.Ex2.m2.1.1.1.2.cmml" xref="A3.Ex2.m2.1.1.1.3"><abs id="A3.Ex2.m2.1.1.1.2.1.cmml" xref="A3.Ex2.m2.1.1.1.3.1"></abs><ci id="A3.Ex2.m2.1.1.1.1.cmml" xref="A3.Ex2.m2.1.1.1.1">𝐶</ci></apply><apply id="A3.Ex2.m2.2.2.2.2.cmml" xref="A3.Ex2.m2.2.2.2.3"><abs id="A3.Ex2.m2.2.2.2.2.1.cmml" xref="A3.Ex2.m2.2.2.2.3.1"></abs><ci id="A3.Ex2.m2.2.2.2.1.cmml" xref="A3.Ex2.m2.2.2.2.1">𝑇</ci></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.Ex2.m2.3c">\displaystyle=\frac{|C|}{|T|}.</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
</div>
<div id="A3.SS1.SSS1.p6" class="ltx_para">
<p id="A3.SS1.SSS1.p6.2" class="ltx_p">The F1 score, which is the harmonic mean of precision and recall, is then calculated as:</p>
<table id="A4.EGx2" class="ltx_equationgroup ltx_eqn_align ltx_eqn_table">

<tbody id="A3.Ex3"><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_td ltx_align_right ltx_eqn_cell"><math id="A3.Ex3.m1.1" class="ltx_Math" alttext="\displaystyle F1" display="inline"><semantics id="A3.Ex3.m1.1a"><mrow id="A3.Ex3.m1.1.1" xref="A3.Ex3.m1.1.1.cmml"><mi id="A3.Ex3.m1.1.1.2" xref="A3.Ex3.m1.1.1.2.cmml">F</mi><mo lspace="0em" rspace="0em" id="A3.Ex3.m1.1.1.1" xref="A3.Ex3.m1.1.1.1.cmml">​</mo><mn id="A3.Ex3.m1.1.1.3" xref="A3.Ex3.m1.1.1.3.cmml">1</mn></mrow><annotation-xml encoding="MathML-Content" id="A3.Ex3.m1.1b"><apply id="A3.Ex3.m1.1.1.cmml" xref="A3.Ex3.m1.1.1"><times id="A3.Ex3.m1.1.1.1.cmml" xref="A3.Ex3.m1.1.1.1"></times><ci id="A3.Ex3.m1.1.1.2.cmml" xref="A3.Ex3.m1.1.1.2">𝐹</ci><cn type="integer" id="A3.Ex3.m1.1.1.3.cmml" xref="A3.Ex3.m1.1.1.3">1</cn></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.Ex3.m1.1c">\displaystyle F1</annotation></semantics></math></td>
<td class="ltx_td ltx_align_left ltx_eqn_cell"><math id="A3.Ex3.m2.1" class="ltx_Math" alttext="\displaystyle=\frac{2\times\text{Precision}\times\text{Recall}}{\text{Precision}+\text{Recall}}," display="inline"><semantics id="A3.Ex3.m2.1a"><mrow id="A3.Ex3.m2.1.1.1" xref="A3.Ex3.m2.1.1.1.1.cmml"><mrow id="A3.Ex3.m2.1.1.1.1" xref="A3.Ex3.m2.1.1.1.1.cmml"><mi id="A3.Ex3.m2.1.1.1.1.2" xref="A3.Ex3.m2.1.1.1.1.2.cmml"></mi><mo id="A3.Ex3.m2.1.1.1.1.1" xref="A3.Ex3.m2.1.1.1.1.1.cmml">=</mo><mstyle displaystyle="true" id="A3.Ex3.m2.1.1.1.1.3" xref="A3.Ex3.m2.1.1.1.1.3.cmml"><mfrac id="A3.Ex3.m2.1.1.1.1.3a" xref="A3.Ex3.m2.1.1.1.1.3.cmml"><mrow id="A3.Ex3.m2.1.1.1.1.3.2" xref="A3.Ex3.m2.1.1.1.1.3.2.cmml"><mn id="A3.Ex3.m2.1.1.1.1.3.2.2" xref="A3.Ex3.m2.1.1.1.1.3.2.2.cmml">2</mn><mo lspace="0.222em" rspace="0.222em" id="A3.Ex3.m2.1.1.1.1.3.2.1" xref="A3.Ex3.m2.1.1.1.1.3.2.1.cmml">×</mo><mtext id="A3.Ex3.m2.1.1.1.1.3.2.3" xref="A3.Ex3.m2.1.1.1.1.3.2.3a.cmml">Precision</mtext><mo lspace="0.222em" rspace="0.222em" id="A3.Ex3.m2.1.1.1.1.3.2.1a" xref="A3.Ex3.m2.1.1.1.1.3.2.1.cmml">×</mo><mtext id="A3.Ex3.m2.1.1.1.1.3.2.4" xref="A3.Ex3.m2.1.1.1.1.3.2.4a.cmml">Recall</mtext></mrow><mrow id="A3.Ex3.m2.1.1.1.1.3.3" xref="A3.Ex3.m2.1.1.1.1.3.3.cmml"><mtext id="A3.Ex3.m2.1.1.1.1.3.3.2" xref="A3.Ex3.m2.1.1.1.1.3.3.2a.cmml">Precision</mtext><mo id="A3.Ex3.m2.1.1.1.1.3.3.1" xref="A3.Ex3.m2.1.1.1.1.3.3.1.cmml">+</mo><mtext id="A3.Ex3.m2.1.1.1.1.3.3.3" xref="A3.Ex3.m2.1.1.1.1.3.3.3a.cmml">Recall</mtext></mrow></mfrac></mstyle></mrow><mo id="A3.Ex3.m2.1.1.1.2" xref="A3.Ex3.m2.1.1.1.1.cmml">,</mo></mrow><annotation-xml encoding="MathML-Content" id="A3.Ex3.m2.1b"><apply id="A3.Ex3.m2.1.1.1.1.cmml" xref="A3.Ex3.m2.1.1.1"><eq id="A3.Ex3.m2.1.1.1.1.1.cmml" xref="A3.Ex3.m2.1.1.1.1.1"></eq><csymbol cd="latexml" id="A3.Ex3.m2.1.1.1.1.2.cmml" xref="A3.Ex3.m2.1.1.1.1.2">absent</csymbol><apply id="A3.Ex3.m2.1.1.1.1.3.cmml" xref="A3.Ex3.m2.1.1.1.1.3"><divide id="A3.Ex3.m2.1.1.1.1.3.1.cmml" xref="A3.Ex3.m2.1.1.1.1.3"></divide><apply id="A3.Ex3.m2.1.1.1.1.3.2.cmml" xref="A3.Ex3.m2.1.1.1.1.3.2"><times id="A3.Ex3.m2.1.1.1.1.3.2.1.cmml" xref="A3.Ex3.m2.1.1.1.1.3.2.1"></times><cn type="integer" id="A3.Ex3.m2.1.1.1.1.3.2.2.cmml" xref="A3.Ex3.m2.1.1.1.1.3.2.2">2</cn><ci id="A3.Ex3.m2.1.1.1.1.3.2.3a.cmml" xref="A3.Ex3.m2.1.1.1.1.3.2.3"><mtext id="A3.Ex3.m2.1.1.1.1.3.2.3.cmml" xref="A3.Ex3.m2.1.1.1.1.3.2.3">Precision</mtext></ci><ci id="A3.Ex3.m2.1.1.1.1.3.2.4a.cmml" xref="A3.Ex3.m2.1.1.1.1.3.2.4"><mtext id="A3.Ex3.m2.1.1.1.1.3.2.4.cmml" xref="A3.Ex3.m2.1.1.1.1.3.2.4">Recall</mtext></ci></apply><apply id="A3.Ex3.m2.1.1.1.1.3.3.cmml" xref="A3.Ex3.m2.1.1.1.1.3.3"><plus id="A3.Ex3.m2.1.1.1.1.3.3.1.cmml" xref="A3.Ex3.m2.1.1.1.1.3.3.1"></plus><ci id="A3.Ex3.m2.1.1.1.1.3.3.2a.cmml" xref="A3.Ex3.m2.1.1.1.1.3.3.2"><mtext id="A3.Ex3.m2.1.1.1.1.3.3.2.cmml" xref="A3.Ex3.m2.1.1.1.1.3.3.2">Precision</mtext></ci><ci id="A3.Ex3.m2.1.1.1.1.3.3.3a.cmml" xref="A3.Ex3.m2.1.1.1.1.3.3.3"><mtext id="A3.Ex3.m2.1.1.1.1.3.3.3.cmml" xref="A3.Ex3.m2.1.1.1.1.3.3.3">Recall</mtext></ci></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.Ex3.m2.1c">\displaystyle=\frac{2\times\text{Precision}\times\text{Recall}}{\text{Precision}+\text{Recall}},</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
<tbody id="A3.Ex4"><tr class="ltx_equation ltx_eqn_row ltx_align_baseline">
<td class="ltx_eqn_cell ltx_eqn_center_padleft"></td>
<td class="ltx_td ltx_eqn_cell"></td>
<td class="ltx_td ltx_align_left ltx_eqn_cell"><math id="A3.Ex4.m1.9" class="ltx_Math" alttext="\displaystyle=\frac{2\times|C|/|G|\times|C|/|T|}{|C|/|G|+|C|/|T|}," display="inline"><semantics id="A3.Ex4.m1.9a"><mrow id="A3.Ex4.m1.9.9.1" xref="A3.Ex4.m1.9.9.1.1.cmml"><mrow id="A3.Ex4.m1.9.9.1.1" xref="A3.Ex4.m1.9.9.1.1.cmml"><mi id="A3.Ex4.m1.9.9.1.1.2" xref="A3.Ex4.m1.9.9.1.1.2.cmml"></mi><mo id="A3.Ex4.m1.9.9.1.1.1" xref="A3.Ex4.m1.9.9.1.1.1.cmml">=</mo><mstyle displaystyle="true" id="A3.Ex4.m1.8.8" xref="A3.Ex4.m1.8.8.cmml"><mfrac id="A3.Ex4.m1.8.8a" xref="A3.Ex4.m1.8.8.cmml"><mrow id="A3.Ex4.m1.4.4.4" xref="A3.Ex4.m1.4.4.4.cmml"><mrow id="A3.Ex4.m1.4.4.4.6" xref="A3.Ex4.m1.4.4.4.6.cmml"><mrow id="A3.Ex4.m1.4.4.4.6.2" xref="A3.Ex4.m1.4.4.4.6.2.cmml"><mrow id="A3.Ex4.m1.4.4.4.6.2.2" xref="A3.Ex4.m1.4.4.4.6.2.2.cmml"><mn id="A3.Ex4.m1.4.4.4.6.2.2.2" xref="A3.Ex4.m1.4.4.4.6.2.2.2.cmml">2</mn><mo lspace="0.222em" rspace="0.222em" id="A3.Ex4.m1.4.4.4.6.2.2.1" xref="A3.Ex4.m1.4.4.4.6.2.2.1.cmml">×</mo><mrow id="A3.Ex4.m1.4.4.4.6.2.2.3.2" xref="A3.Ex4.m1.4.4.4.6.2.2.3.1.cmml"><mo stretchy="false" id="A3.Ex4.m1.4.4.4.6.2.2.3.2.1" xref="A3.Ex4.m1.4.4.4.6.2.2.3.1.1.cmml">|</mo><mi id="A3.Ex4.m1.1.1.1.1" xref="A3.Ex4.m1.1.1.1.1.cmml">C</mi><mo stretchy="false" id="A3.Ex4.m1.4.4.4.6.2.2.3.2.2" xref="A3.Ex4.m1.4.4.4.6.2.2.3.1.1.cmml">|</mo></mrow></mrow><mo id="A3.Ex4.m1.4.4.4.6.2.1" xref="A3.Ex4.m1.4.4.4.6.2.1.cmml">/</mo><mrow id="A3.Ex4.m1.4.4.4.6.2.3.2" xref="A3.Ex4.m1.4.4.4.6.2.3.1.cmml"><mo stretchy="false" id="A3.Ex4.m1.4.4.4.6.2.3.2.1" xref="A3.Ex4.m1.4.4.4.6.2.3.1.1.cmml">|</mo><mi id="A3.Ex4.m1.2.2.2.2" xref="A3.Ex4.m1.2.2.2.2.cmml">G</mi><mo rspace="0.055em" stretchy="false" id="A3.Ex4.m1.4.4.4.6.2.3.2.2" xref="A3.Ex4.m1.4.4.4.6.2.3.1.1.cmml">|</mo></mrow></mrow><mo rspace="0.222em" id="A3.Ex4.m1.4.4.4.6.1" xref="A3.Ex4.m1.4.4.4.6.1.cmml">×</mo><mrow id="A3.Ex4.m1.4.4.4.6.3.2" xref="A3.Ex4.m1.4.4.4.6.3.1.cmml"><mo stretchy="false" id="A3.Ex4.m1.4.4.4.6.3.2.1" xref="A3.Ex4.m1.4.4.4.6.3.1.1.cmml">|</mo><mi id="A3.Ex4.m1.3.3.3.3" xref="A3.Ex4.m1.3.3.3.3.cmml">C</mi><mo stretchy="false" id="A3.Ex4.m1.4.4.4.6.3.2.2" xref="A3.Ex4.m1.4.4.4.6.3.1.1.cmml">|</mo></mrow></mrow><mo id="A3.Ex4.m1.4.4.4.5" xref="A3.Ex4.m1.4.4.4.5.cmml">/</mo><mrow id="A3.Ex4.m1.4.4.4.7.2" xref="A3.Ex4.m1.4.4.4.7.1.cmml"><mo stretchy="false" id="A3.Ex4.m1.4.4.4.7.2.1" xref="A3.Ex4.m1.4.4.4.7.1.1.cmml">|</mo><mi id="A3.Ex4.m1.4.4.4.4" xref="A3.Ex4.m1.4.4.4.4.cmml">T</mi><mo stretchy="false" id="A3.Ex4.m1.4.4.4.7.2.2" xref="A3.Ex4.m1.4.4.4.7.1.1.cmml">|</mo></mrow></mrow><mrow id="A3.Ex4.m1.8.8.8" xref="A3.Ex4.m1.8.8.8.cmml"><mrow id="A3.Ex4.m1.8.8.8.6" xref="A3.Ex4.m1.8.8.8.6.cmml"><mrow id="A3.Ex4.m1.8.8.8.6.2.2" xref="A3.Ex4.m1.8.8.8.6.2.1.cmml"><mo stretchy="false" id="A3.Ex4.m1.8.8.8.6.2.2.1" xref="A3.Ex4.m1.8.8.8.6.2.1.1.cmml">|</mo><mi id="A3.Ex4.m1.5.5.5.1" xref="A3.Ex4.m1.5.5.5.1.cmml">C</mi><mo stretchy="false" id="A3.Ex4.m1.8.8.8.6.2.2.2" xref="A3.Ex4.m1.8.8.8.6.2.1.1.cmml">|</mo></mrow><mo id="A3.Ex4.m1.8.8.8.6.1" xref="A3.Ex4.m1.8.8.8.6.1.cmml">/</mo><mrow id="A3.Ex4.m1.8.8.8.6.3.2" xref="A3.Ex4.m1.8.8.8.6.3.1.cmml"><mo stretchy="false" id="A3.Ex4.m1.8.8.8.6.3.2.1" xref="A3.Ex4.m1.8.8.8.6.3.1.1.cmml">|</mo><mi id="A3.Ex4.m1.6.6.6.2" xref="A3.Ex4.m1.6.6.6.2.cmml">G</mi><mo stretchy="false" id="A3.Ex4.m1.8.8.8.6.3.2.2" xref="A3.Ex4.m1.8.8.8.6.3.1.1.cmml">|</mo></mrow></mrow><mo id="A3.Ex4.m1.8.8.8.5" xref="A3.Ex4.m1.8.8.8.5.cmml">+</mo><mrow id="A3.Ex4.m1.8.8.8.7" xref="A3.Ex4.m1.8.8.8.7.cmml"><mrow id="A3.Ex4.m1.8.8.8.7.2.2" xref="A3.Ex4.m1.8.8.8.7.2.1.cmml"><mo stretchy="false" id="A3.Ex4.m1.8.8.8.7.2.2.1" xref="A3.Ex4.m1.8.8.8.7.2.1.1.cmml">|</mo><mi id="A3.Ex4.m1.7.7.7.3" xref="A3.Ex4.m1.7.7.7.3.cmml">C</mi><mo stretchy="false" id="A3.Ex4.m1.8.8.8.7.2.2.2" xref="A3.Ex4.m1.8.8.8.7.2.1.1.cmml">|</mo></mrow><mo id="A3.Ex4.m1.8.8.8.7.1" xref="A3.Ex4.m1.8.8.8.7.1.cmml">/</mo><mrow id="A3.Ex4.m1.8.8.8.7.3.2" xref="A3.Ex4.m1.8.8.8.7.3.1.cmml"><mo stretchy="false" id="A3.Ex4.m1.8.8.8.7.3.2.1" xref="A3.Ex4.m1.8.8.8.7.3.1.1.cmml">|</mo><mi id="A3.Ex4.m1.8.8.8.4" xref="A3.Ex4.m1.8.8.8.4.cmml">T</mi><mo stretchy="false" id="A3.Ex4.m1.8.8.8.7.3.2.2" xref="A3.Ex4.m1.8.8.8.7.3.1.1.cmml">|</mo></mrow></mrow></mrow></mfrac></mstyle></mrow><mo id="A3.Ex4.m1.9.9.1.2" xref="A3.Ex4.m1.9.9.1.1.cmml">,</mo></mrow><annotation-xml encoding="MathML-Content" id="A3.Ex4.m1.9b"><apply id="A3.Ex4.m1.9.9.1.1.cmml" xref="A3.Ex4.m1.9.9.1"><eq id="A3.Ex4.m1.9.9.1.1.1.cmml" xref="A3.Ex4.m1.9.9.1.1.1"></eq><csymbol cd="latexml" id="A3.Ex4.m1.9.9.1.1.2.cmml" xref="A3.Ex4.m1.9.9.1.1.2">absent</csymbol><apply id="A3.Ex4.m1.8.8.cmml" xref="A3.Ex4.m1.8.8"><divide id="A3.Ex4.m1.8.8.9.cmml" xref="A3.Ex4.m1.8.8"></divide><apply id="A3.Ex4.m1.4.4.4.cmml" xref="A3.Ex4.m1.4.4.4"><divide id="A3.Ex4.m1.4.4.4.5.cmml" xref="A3.Ex4.m1.4.4.4.5"></divide><apply id="A3.Ex4.m1.4.4.4.6.cmml" xref="A3.Ex4.m1.4.4.4.6"><times id="A3.Ex4.m1.4.4.4.6.1.cmml" xref="A3.Ex4.m1.4.4.4.6.1"></times><apply id="A3.Ex4.m1.4.4.4.6.2.cmml" xref="A3.Ex4.m1.4.4.4.6.2"><divide id="A3.Ex4.m1.4.4.4.6.2.1.cmml" xref="A3.Ex4.m1.4.4.4.6.2.1"></divide><apply id="A3.Ex4.m1.4.4.4.6.2.2.cmml" xref="A3.Ex4.m1.4.4.4.6.2.2"><times id="A3.Ex4.m1.4.4.4.6.2.2.1.cmml" xref="A3.Ex4.m1.4.4.4.6.2.2.1"></times><cn type="integer" id="A3.Ex4.m1.4.4.4.6.2.2.2.cmml" xref="A3.Ex4.m1.4.4.4.6.2.2.2">2</cn><apply id="A3.Ex4.m1.4.4.4.6.2.2.3.1.cmml" xref="A3.Ex4.m1.4.4.4.6.2.2.3.2"><abs id="A3.Ex4.m1.4.4.4.6.2.2.3.1.1.cmml" xref="A3.Ex4.m1.4.4.4.6.2.2.3.2.1"></abs><ci id="A3.Ex4.m1.1.1.1.1.cmml" xref="A3.Ex4.m1.1.1.1.1">𝐶</ci></apply></apply><apply id="A3.Ex4.m1.4.4.4.6.2.3.1.cmml" xref="A3.Ex4.m1.4.4.4.6.2.3.2"><abs id="A3.Ex4.m1.4.4.4.6.2.3.1.1.cmml" xref="A3.Ex4.m1.4.4.4.6.2.3.2.1"></abs><ci id="A3.Ex4.m1.2.2.2.2.cmml" xref="A3.Ex4.m1.2.2.2.2">𝐺</ci></apply></apply><apply id="A3.Ex4.m1.4.4.4.6.3.1.cmml" xref="A3.Ex4.m1.4.4.4.6.3.2"><abs id="A3.Ex4.m1.4.4.4.6.3.1.1.cmml" xref="A3.Ex4.m1.4.4.4.6.3.2.1"></abs><ci id="A3.Ex4.m1.3.3.3.3.cmml" xref="A3.Ex4.m1.3.3.3.3">𝐶</ci></apply></apply><apply id="A3.Ex4.m1.4.4.4.7.1.cmml" xref="A3.Ex4.m1.4.4.4.7.2"><abs id="A3.Ex4.m1.4.4.4.7.1.1.cmml" xref="A3.Ex4.m1.4.4.4.7.2.1"></abs><ci id="A3.Ex4.m1.4.4.4.4.cmml" xref="A3.Ex4.m1.4.4.4.4">𝑇</ci></apply></apply><apply id="A3.Ex4.m1.8.8.8.cmml" xref="A3.Ex4.m1.8.8.8"><plus id="A3.Ex4.m1.8.8.8.5.cmml" xref="A3.Ex4.m1.8.8.8.5"></plus><apply id="A3.Ex4.m1.8.8.8.6.cmml" xref="A3.Ex4.m1.8.8.8.6"><divide id="A3.Ex4.m1.8.8.8.6.1.cmml" xref="A3.Ex4.m1.8.8.8.6.1"></divide><apply id="A3.Ex4.m1.8.8.8.6.2.1.cmml" xref="A3.Ex4.m1.8.8.8.6.2.2"><abs id="A3.Ex4.m1.8.8.8.6.2.1.1.cmml" xref="A3.Ex4.m1.8.8.8.6.2.2.1"></abs><ci id="A3.Ex4.m1.5.5.5.1.cmml" xref="A3.Ex4.m1.5.5.5.1">𝐶</ci></apply><apply id="A3.Ex4.m1.8.8.8.6.3.1.cmml" xref="A3.Ex4.m1.8.8.8.6.3.2"><abs id="A3.Ex4.m1.8.8.8.6.3.1.1.cmml" xref="A3.Ex4.m1.8.8.8.6.3.2.1"></abs><ci id="A3.Ex4.m1.6.6.6.2.cmml" xref="A3.Ex4.m1.6.6.6.2">𝐺</ci></apply></apply><apply id="A3.Ex4.m1.8.8.8.7.cmml" xref="A3.Ex4.m1.8.8.8.7"><divide id="A3.Ex4.m1.8.8.8.7.1.cmml" xref="A3.Ex4.m1.8.8.8.7.1"></divide><apply id="A3.Ex4.m1.8.8.8.7.2.1.cmml" xref="A3.Ex4.m1.8.8.8.7.2.2"><abs id="A3.Ex4.m1.8.8.8.7.2.1.1.cmml" xref="A3.Ex4.m1.8.8.8.7.2.2.1"></abs><ci id="A3.Ex4.m1.7.7.7.3.cmml" xref="A3.Ex4.m1.7.7.7.3">𝐶</ci></apply><apply id="A3.Ex4.m1.8.8.8.7.3.1.cmml" xref="A3.Ex4.m1.8.8.8.7.3.2"><abs id="A3.Ex4.m1.8.8.8.7.3.1.1.cmml" xref="A3.Ex4.m1.8.8.8.7.3.2.1"></abs><ci id="A3.Ex4.m1.8.8.8.4.cmml" xref="A3.Ex4.m1.8.8.8.4">𝑇</ci></apply></apply></apply></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.Ex4.m1.9c">\displaystyle=\frac{2\times|C|/|G|\times|C|/|T|}{|C|/|G|+|C|/|T|},</annotation></semantics></math></td>
<td class="ltx_eqn_cell ltx_eqn_center_padright"></td>
</tr></tbody>
</table>
<p id="A3.SS1.SSS1.p6.1" class="ltx_p">with the condition that if the denominator <math id="A3.SS1.SSS1.p6.1.m1.1" class="ltx_Math" alttext="(\text{Precision}+\text{Recall})" display="inline"><semantics id="A3.SS1.SSS1.p6.1.m1.1a"><mrow id="A3.SS1.SSS1.p6.1.m1.1.1.1" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.cmml"><mo stretchy="false" id="A3.SS1.SSS1.p6.1.m1.1.1.1.2" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.cmml">(</mo><mrow id="A3.SS1.SSS1.p6.1.m1.1.1.1.1" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.cmml"><mtext id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.2" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.2a.cmml">Precision</mtext><mo id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.1" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.1.cmml">+</mo><mtext id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.3" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.3a.cmml">Recall</mtext></mrow><mo stretchy="false" id="A3.SS1.SSS1.p6.1.m1.1.1.1.3" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.cmml">)</mo></mrow><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS1.p6.1.m1.1b"><apply id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.cmml" xref="A3.SS1.SSS1.p6.1.m1.1.1.1"><plus id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.1.cmml" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.1"></plus><ci id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.2a.cmml" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.2"><mtext id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.2.cmml" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.2">Precision</mtext></ci><ci id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.3a.cmml" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.3"><mtext id="A3.SS1.SSS1.p6.1.m1.1.1.1.1.3.cmml" xref="A3.SS1.SSS1.p6.1.m1.1.1.1.1.3">Recall</mtext></ci></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS1.p6.1.m1.1c">(\text{Precision}+\text{Recall})</annotation></semantics></math> is zero, the F1 score is set to 0.</p>
</div>
<div id="A3.SS1.SSS1.p7" class="ltx_para">
<p id="A3.SS1.SSS1.p7.1" class="ltx_p">The final F1 score is the average of the F1 scores for all items. If there are no F1 scores (i.e., the list of scores is empty), the average F1 score is set to 0.</p>
</div>
<div id="A3.SS1.SSS1.p8" class="ltx_para">
<p id="A3.SS1.SSS1.p8.1" class="ltx_p"><math id="A3.SS1.SSS1.p8.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS1.SSS1.p8.1.m1.1a"><mo id="A3.SS1.SSS1.p8.1.m1.1.1" xref="A3.SS1.SSS1.p8.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS1.p8.1.m1.1b"><ci id="A3.SS1.SSS1.p8.1.m1.1.1.cmml" xref="A3.SS1.SSS1.p8.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS1.p8.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS1.SSS1.p8.1.1" class="ltx_text ltx_font_italic">Edit Generalization:</span> Measures the model’s capability to genuinely update knowledge, rather than just repeating keywords. This is tested by evaluating edit success on paraphrased prompts given by the respective dataset.</p>
</div>
<div id="A3.SS1.SSS1.p9" class="ltx_para">
<p id="A3.SS1.SSS1.p9.1" class="ltx_p"><math id="A3.SS1.SSS1.p9.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS1.SSS1.p9.1.m1.1a"><mo id="A3.SS1.SSS1.p9.1.m1.1.1" xref="A3.SS1.SSS1.p9.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS1.p9.1.m1.1b"><ci id="A3.SS1.SSS1.p9.1.m1.1.1.cmml" xref="A3.SS1.SSS1.p9.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS1.p9.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS1.SSS1.p9.1.1" class="ltx_text ltx_font_italic">Edit Locality:</span> Examines the precision of edits, ensuring that only relevant, in-distribution knowledge is altered while out-of-distribution facts remain unaffected. In this setting, the F1 score is critical as testing on the pre-trained knowledge can result in varied answer phrasings.</p>
</div>
</section>
<section id="A3.SS1.SSS2" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.1.2 </span>Knowledge Editing Dataset</h4>

<div id="A3.SS1.SSS2.p1" class="ltx_para">
<p id="A3.SS1.SSS2.p1.1" class="ltx_p"><math id="A3.SS1.SSS2.p1.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS1.SSS2.p1.1.m1.1a"><mo id="A3.SS1.SSS2.p1.1.m1.1.1" xref="A3.SS1.SSS2.p1.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS1.SSS2.p1.1.m1.1b"><ci id="A3.SS1.SSS2.p1.1.m1.1.1.cmml" xref="A3.SS1.SSS2.p1.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS1.SSS2.p1.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS1.SSS2.p1.1.1" class="ltx_text ltx_font_italic">ZsRE</span> <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib35" title="" class="ltx_ref">35</a>]</cite>: This dataset focuses on context-free question-answering. Given a question related to a subject and a relation, the model’s task is to provide the correct object as the answer. <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib99" title="" class="ltx_ref">99</a>]</cite> follow the procedure outlined in <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib44" title="" class="ltx_ref">44</a>]</cite>, to improve the locality sets used. As the initial locality sets were solely based on Natural Question annotations, we use the locality sets provided by <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib99" title="" class="ltx_ref">99</a>]</cite>.</p>
</div>
<div id="A3.SS1.SSS2.p2" class="ltx_para">
<p id="A3.SS1.SSS2.p2.1" class="ltx_p">In all three datasets, original prompt rephrasings and locality sets are used to derive the respective metrics.</p>
</div>
</section>
</section>
<section id="A3.SS2" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">C.2 </span>Model Editing Details</h3>

<section id="A3.SS2.SSS1" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.2.1 </span>MEMIT Details</h4>

<div id="A3.SS2.SSS1.p1" class="ltx_para">
<p id="A3.SS2.SSS1.p1.5" class="ltx_p">We use the state-of-the-art MEMIT <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib20" title="" class="ltx_ref">20</a>]</cite> model editor, which applies batches of edits simultaneously.
The editing process was applied to layers 4 through 8 of the model, with a clamp normalization factor set at 4. The learning parameters adhered closely to the original implementation: <span id="A3.SS2.SSS1.p1.5.1" class="ltx_text ltx_markedasmath">v_num_grad_steps</span> was set to 25, accompanied by a learning rate (<span id="A3.SS2.SSS1.p1.5.2" class="ltx_text ltx_markedasmath">lr</span>) of 0.5, and using the last layer for loss calculation. Additionally, a weight decay (<span id="A3.SS2.SSS1.p1.5.3" class="ltx_text ltx_markedasmath">weight_decay</span>) of 0.001 was employed. The KL divergence contribution to the overall loss was controlled by a <math id="A3.SS2.SSS1.p1.4.m4.1" class="ltx_Math" alttext="KL\_\text{factor}" display="inline"><semantics id="A3.SS2.SSS1.p1.4.m4.1a"><mrow id="A3.SS2.SSS1.p1.4.m4.1.1" xref="A3.SS2.SSS1.p1.4.m4.1.1.cmml"><mi id="A3.SS2.SSS1.p1.4.m4.1.1.2" xref="A3.SS2.SSS1.p1.4.m4.1.1.2.cmml">K</mi><mo lspace="0em" rspace="0em" id="A3.SS2.SSS1.p1.4.m4.1.1.1" xref="A3.SS2.SSS1.p1.4.m4.1.1.1.cmml">​</mo><mi id="A3.SS2.SSS1.p1.4.m4.1.1.3" xref="A3.SS2.SSS1.p1.4.m4.1.1.3.cmml">L</mi><mo lspace="0em" rspace="0em" id="A3.SS2.SSS1.p1.4.m4.1.1.1a" xref="A3.SS2.SSS1.p1.4.m4.1.1.1.cmml">​</mo><mi mathvariant="normal" id="A3.SS2.SSS1.p1.4.m4.1.1.4" xref="A3.SS2.SSS1.p1.4.m4.1.1.4.cmml">_</mi><mo lspace="0em" rspace="0em" id="A3.SS2.SSS1.p1.4.m4.1.1.1b" xref="A3.SS2.SSS1.p1.4.m4.1.1.1.cmml">​</mo><mtext id="A3.SS2.SSS1.p1.4.m4.1.1.5" xref="A3.SS2.SSS1.p1.4.m4.1.1.5a.cmml">factor</mtext></mrow><annotation-xml encoding="MathML-Content" id="A3.SS2.SSS1.p1.4.m4.1b"><apply id="A3.SS2.SSS1.p1.4.m4.1.1.cmml" xref="A3.SS2.SSS1.p1.4.m4.1.1"><times id="A3.SS2.SSS1.p1.4.m4.1.1.1.cmml" xref="A3.SS2.SSS1.p1.4.m4.1.1.1"></times><ci id="A3.SS2.SSS1.p1.4.m4.1.1.2.cmml" xref="A3.SS2.SSS1.p1.4.m4.1.1.2">𝐾</ci><ci id="A3.SS2.SSS1.p1.4.m4.1.1.3.cmml" xref="A3.SS2.SSS1.p1.4.m4.1.1.3">𝐿</ci><ci id="A3.SS2.SSS1.p1.4.m4.1.1.4.cmml" xref="A3.SS2.SSS1.p1.4.m4.1.1.4">_</ci><ci id="A3.SS2.SSS1.p1.4.m4.1.1.5a.cmml" xref="A3.SS2.SSS1.p1.4.m4.1.1.5"><mtext id="A3.SS2.SSS1.p1.4.m4.1.1.5.cmml" xref="A3.SS2.SSS1.p1.4.m4.1.1.5">factor</mtext></ci></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.SS2.SSS1.p1.4.m4.1c">KL\_\text{factor}</annotation></semantics></math> of 0.0625. Moreover, a second momentum adjustment was enabled, with an update weight of <math id="A3.SS2.SSS1.p1.5.m5.1" class="ltx_Math" alttext="15000" display="inline"><semantics id="A3.SS2.SSS1.p1.5.m5.1a"><mn id="A3.SS2.SSS1.p1.5.m5.1.1" xref="A3.SS2.SSS1.p1.5.m5.1.1.cmml">15000</mn><annotation-xml encoding="MathML-Content" id="A3.SS2.SSS1.p1.5.m5.1b"><cn type="integer" id="A3.SS2.SSS1.p1.5.m5.1.1.cmml" xref="A3.SS2.SSS1.p1.5.m5.1.1">15000</cn></annotation-xml><annotation encoding="application/x-tex" id="A3.SS2.SSS1.p1.5.m5.1c">15000</annotation></semantics></math>, to fine-tune the optimization process. The model generated a maximum length of 40 tokens and a batch size of 50, matching the number of edits being made. 10 repeats were made for each edit and the results averaged.</p>
</div>
</section>
<section id="A3.SS2.SSS2" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.2.2 </span>LoRA Details</h4>

<div id="A3.SS2.SSS2.p1" class="ltx_para">
<p id="A3.SS2.SSS2.p1.1" class="ltx_p">We also employed LoRA as a model editor, specifically using the adalora variant for our experiments. The editing process utilized 70 gradient steps, with a learning rate of 0.005. The rank for the low-rank adaptation was set at 8, with a scaling factor of 32. There was no dropout applied to the adaptation matrices, and the normalization constraint was disabled. The target modules for the edits consisted of query and value projection layers. Consistent with the MEMIT method, the model generated sequences with a maximum length of 40 tokens and utilized a batch size of 50, matching the number of edits per batch. Each edit was repeated 10 times, and the results were averaged to ensure reliability. The data type for the computations was set to <span id="A3.SS2.SSS2.p1.1.1" class="ltx_text ltx_font_typewriter">torch.float</span>.</p>
</div>
</section>
<section id="A3.SS2.SSS3" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.2.3 </span>Fine-Tuning (FT) Details</h4>

<div id="A3.SS2.SSS3.p1" class="ltx_para">
<p id="A3.SS2.SSS3.p1.1" class="ltx_p">For fine-tuning (FT), we focused on layers 4 through 8 of the model. The specific modules targeted for editing included the MLP down-projection (<span id="A3.SS2.SSS3.p1.1.1" class="ltx_text ltx_font_typewriter">model.layers.{}.mlp.down_proj</span>), the general layer module (<span id="A3.SS2.SSS3.p1.1.2" class="ltx_text ltx_font_typewriter">model.layers.{}</span>), the MLP module (<span id="A3.SS2.SSS3.p1.1.3" class="ltx_text ltx_font_typewriter">model.layers.{}.mlp</span>), the self-attention module (<span id="A3.SS2.SSS3.p1.1.4" class="ltx_text ltx_font_typewriter">model.layers.{}.self_attn</span>), the final layer normalization (<span id="A3.SS2.SSS3.p1.1.5" class="ltx_text ltx_font_typewriter">model.norm</span>), and the language model head (<span id="A3.SS2.SSS3.p1.1.6" class="ltx_text ltx_font_typewriter">lm_head</span>).</p>
</div>
<div id="A3.SS2.SSS3.p2" class="ltx_para">
<p id="A3.SS2.SSS3.p2.1" class="ltx_p">The fine-tuning process was conducted with 25 optimization steps, using a learning rate of <math id="A3.SS2.SSS3.p2.1.m1.1" class="ltx_Math" alttext="10^{-5}" display="inline"><semantics id="A3.SS2.SSS3.p2.1.m1.1a"><msup id="A3.SS2.SSS3.p2.1.m1.1.1" xref="A3.SS2.SSS3.p2.1.m1.1.1.cmml"><mn id="A3.SS2.SSS3.p2.1.m1.1.1.2" xref="A3.SS2.SSS3.p2.1.m1.1.1.2.cmml">10</mn><mrow id="A3.SS2.SSS3.p2.1.m1.1.1.3" xref="A3.SS2.SSS3.p2.1.m1.1.1.3.cmml"><mo id="A3.SS2.SSS3.p2.1.m1.1.1.3a" xref="A3.SS2.SSS3.p2.1.m1.1.1.3.cmml">−</mo><mn id="A3.SS2.SSS3.p2.1.m1.1.1.3.2" xref="A3.SS2.SSS3.p2.1.m1.1.1.3.2.cmml">5</mn></mrow></msup><annotation-xml encoding="MathML-Content" id="A3.SS2.SSS3.p2.1.m1.1b"><apply id="A3.SS2.SSS3.p2.1.m1.1.1.cmml" xref="A3.SS2.SSS3.p2.1.m1.1.1"><csymbol cd="ambiguous" id="A3.SS2.SSS3.p2.1.m1.1.1.1.cmml" xref="A3.SS2.SSS3.p2.1.m1.1.1">superscript</csymbol><cn type="integer" id="A3.SS2.SSS3.p2.1.m1.1.1.2.cmml" xref="A3.SS2.SSS3.p2.1.m1.1.1.2">10</cn><apply id="A3.SS2.SSS3.p2.1.m1.1.1.3.cmml" xref="A3.SS2.SSS3.p2.1.m1.1.1.3"><minus id="A3.SS2.SSS3.p2.1.m1.1.1.3.1.cmml" xref="A3.SS2.SSS3.p2.1.m1.1.1.3"></minus><cn type="integer" id="A3.SS2.SSS3.p2.1.m1.1.1.3.2.cmml" xref="A3.SS2.SSS3.p2.1.m1.1.1.3.2">5</cn></apply></apply></annotation-xml><annotation encoding="application/x-tex" id="A3.SS2.SSS3.p2.1.m1.1c">10^{-5}</annotation></semantics></math>. No weight decay was applied. The normalization constraint was disabled. The optimization objective was set to target the new data and the data type for the computations was set to bfloat16.</p>
</div>
<div id="A3.SS2.SSS3.p3" class="ltx_para">
<p id="A3.SS2.SSS3.p3.1" class="ltx_p">As with the MEMIT and LoRA methods, the model generated sequences with a maximum length of 40 tokens and used a batch size of 50, matching the number of edits per batch. Each edit was repeated 10 times, and the results were averaged for consistency and reliability.</p>
</div>
</section>
</section>
<section id="A3.SS3" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">C.3 </span>Model Compression Details</h3>

<section id="A3.SS3.SSS1" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.3.1 </span>Compression Metrics</h4>

<div id="A3.SS3.SSS1.p1" class="ltx_para">
<p id="A3.SS3.SSS1.p1.1" class="ltx_p">The compression metrics include:</p>
</div>
<div id="A3.SS3.SSS1.p2" class="ltx_para">
<p id="A3.SS3.SSS1.p2.1" class="ltx_p"><math id="A3.SS3.SSS1.p2.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS3.SSS1.p2.1.m1.1a"><mo id="A3.SS3.SSS1.p2.1.m1.1.1" xref="A3.SS3.SSS1.p2.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS3.SSS1.p2.1.m1.1b"><ci id="A3.SS3.SSS1.p2.1.m1.1.1.cmml" xref="A3.SS3.SSS1.p2.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS3.SSS1.p2.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS3.SSS1.p2.1.1" class="ltx_text ltx_font_italic">Sparsity Ratio.</span> The fraction of parameters in the model that are zero, indicating the extent to which the model is sparse.</p>
</div>
<div id="A3.SS3.SSS1.p3" class="ltx_para">
<p id="A3.SS3.SSS1.p3.1" class="ltx_p"><math id="A3.SS3.SSS1.p3.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS3.SSS1.p3.1.m1.1a"><mo id="A3.SS3.SSS1.p3.1.m1.1.1" xref="A3.SS3.SSS1.p3.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS3.SSS1.p3.1.m1.1b"><ci id="A3.SS3.SSS1.p3.1.m1.1.1.cmml" xref="A3.SS3.SSS1.p3.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS3.SSS1.p3.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS3.SSS1.p3.1.1" class="ltx_text ltx_font_italic">Average Bits.</span> The mean number of bits utilized to represent each parameter in a quantized or sparsified neural network, reflecting the level of compression.</p>
</div>
</section>
<section id="A3.SS3.SSS2" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.3.2 </span>Compressor details</h4>

<div id="A3.SS3.SSS2.p1" class="ltx_para">
<p id="A3.SS3.SSS2.p1.1" class="ltx_p">We use four state-of-the-art compression methods including two pruning methods:</p>
</div>
<div id="A3.SS3.SSS2.p2" class="ltx_para">
<p id="A3.SS3.SSS2.p2.1" class="ltx_p"><math id="A3.SS3.SSS2.p2.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS3.SSS2.p2.1.m1.1a"><mo id="A3.SS3.SSS2.p2.1.m1.1.1" xref="A3.SS3.SSS2.p2.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS3.SSS2.p2.1.m1.1b"><ci id="A3.SS3.SSS2.p2.1.m1.1.1.cmml" xref="A3.SS3.SSS2.p2.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS3.SSS2.p2.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS3.SSS2.p2.1.1" class="ltx_text ltx_font_italic">SparseGPT</span> <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib16" title="" class="ltx_ref">16</a>]</cite>: an efficient one-shot pruning method tailored for large models. It converts the pruning process into solving large-scale sparse regression problems using an approximate solver. This approach enables rapid pruning on a single GPU with minimal accuracy loss, achieving 50-60% on large models.</p>
</div>
<div id="A3.SS3.SSS2.p3" class="ltx_para">
<p id="A3.SS3.SSS2.p3.1" class="ltx_p"><math id="A3.SS3.SSS2.p3.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS3.SSS2.p3.1.m1.1a"><mo id="A3.SS3.SSS2.p3.1.m1.1.1" xref="A3.SS3.SSS2.p3.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS3.SSS2.p3.1.m1.1b"><ci id="A3.SS3.SSS2.p3.1.m1.1.1.cmml" xref="A3.SS3.SSS2.p3.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS3.SSS2.p3.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS3.SSS2.p3.1.1" class="ltx_text ltx_font_italic">Wanda</span> <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib36" title="" class="ltx_ref">36</a>]</cite>: is another popular method for pruning large language models that relies on a pruning metric that combines a weight’s magnitude with the norm of its corresponding input activations, determined from a small calibration dataset. The method focuses on selectively pruning weights within individual outputs of linear layers, aiming for high sparsity levels without modifying unpruned weights. Wanda is computationally efficient, executable in a single forward pass.</p>
</div>
<div id="A3.SS3.SSS2.p4" class="ltx_para">
<p id="A3.SS3.SSS2.p4.1" class="ltx_p">and two quantization methods:</p>
</div>
<div id="A3.SS3.SSS2.p5" class="ltx_para">
<p id="A3.SS3.SSS2.p5.1" class="ltx_p"><math id="A3.SS3.SSS2.p5.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS3.SSS2.p5.1.m1.1a"><mo id="A3.SS3.SSS2.p5.1.m1.1.1" xref="A3.SS3.SSS2.p5.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS3.SSS2.p5.1.m1.1b"><ci id="A3.SS3.SSS2.p5.1.m1.1.1.cmml" xref="A3.SS3.SSS2.p5.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS3.SSS2.p5.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS3.SSS2.p5.1.1" class="ltx_text ltx_font_italic">GPTQ</span> <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib15" title="" class="ltx_ref">15</a>]</cite>: an algorithm designed for efficient weight quantization in large-scale models. It revises the weight quantization approach by quantizing weights in a fixed order rather than a greedy order, which shows minimal performance difference, especially in larger models. GPTQ introduced a novel method where each weight is quantized column-by-column, reducing computational complexity.</p>
</div>
<div id="A3.SS3.SSS2.p6" class="ltx_para">
<p id="A3.SS3.SSS2.p6.1" class="ltx_p"><math id="A3.SS3.SSS2.p6.1.m1.1" class="ltx_Math" alttext="\bullet" display="inline"><semantics id="A3.SS3.SSS2.p6.1.m1.1a"><mo id="A3.SS3.SSS2.p6.1.m1.1.1" xref="A3.SS3.SSS2.p6.1.m1.1.1.cmml">∙</mo><annotation-xml encoding="MathML-Content" id="A3.SS3.SSS2.p6.1.m1.1b"><ci id="A3.SS3.SSS2.p6.1.m1.1.1.cmml" xref="A3.SS3.SSS2.p6.1.m1.1.1">∙</ci></annotation-xml><annotation encoding="application/x-tex" id="A3.SS3.SSS2.p6.1.m1.1c">\bullet</annotation></semantics></math> <span id="A3.SS3.SSS2.p6.1.1" class="ltx_text ltx_font_italic">AWQ</span> <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib37" title="" class="ltx_ref">37</a>]</cite>: is based on the premise that not all weights are equally critical for model performance, and it identifies a small fraction of salient weights whose quantization significantly impacts model accuracy. This identification is done by analyzing activation distributions rather than weight distributions, under the rationale that weights linked to larger activation magnitudes are more crucial.</p>
</div>
</section>
</section>
<section id="A3.SS4" class="ltx_subsection">
<h3 class="ltx_title ltx_title_subsection">
<span class="ltx_tag ltx_tag_subsection">C.4 </span>Machine Unlearning Details</h3>

<div id="A3.SS4.p1" class="ltx_para">
<p id="A3.SS4.p1.1" class="ltx_p">We implement three unlearning techniques: RMU, GA, and GD. The success of each technique hinges on the careful selection of hyperparameters. We describe our parameter searches and selected values in the following sections.</p>
</div>
<section id="A3.SS4.SSS1" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.4.1 </span>Representation Misdirection Unlearning</h4>

<div id="A3.SS4.SSS1.p1" class="ltx_para">
<p id="A3.SS4.SSS1.p1.1" class="ltx_p">Introduced in <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib40" title="" class="ltx_ref">40</a>]</cite>, RMU intervenes on a specific layer of the LM to perturb activations for inputs related to the unlearning target. This technique has achieved impressive performance on the WMDP dataset, with performance on the unlearning targets dropping to near-random while overall model utility (MMLU) minimally regresses. Wikitext <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib100" title="" class="ltx_ref">100</a>]</cite> acts as the retain set.</p>
</div>
<div id="A3.SS4.SSS1.p2" class="ltx_para">
<p id="A3.SS4.SSS1.p2.1" class="ltx_p">RMU relies on multiple hyperparameters for selecting which layer to modify and how the loss should be calculated. We found that RMU is quite sensitive to hyperparameter choice, with most combinations either leaving the model unaffected or significantly harming model utility. Table <a href="#A3.T5" title="Table 5 ‣ C.4.1 Representation Misdirection Unlearning ‣ C.4 Machine Unlearning Details ‣ Appendix C Implementation Details ‣ Acknowledgements ‣ 5 Conclusion ‣ 4.4 General Trends in Composability ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">5</span></a> details the results from our hyperparameters search. We otherwise use the RMU repo’s<span id="footnote5" class="ltx_note ltx_role_footnote"><sup class="ltx_note_mark">5</sup><span class="ltx_note_outer"><span class="ltx_note_content"><sup class="ltx_note_mark">5</sup><span class="ltx_tag ltx_tag_note">5</span>https://github.com/centerforaisafety/wmdp</span></span></span> default settings.</p>
</div>
<figure id="A3.T5" class="ltx_table">
<div id="A3.T5.2" class="ltx_inline-block ltx_align_center ltx_transformed_outer" style="width:433.6pt;height:73.7pt;vertical-align:-0.0pt;"><span class="ltx_transformed_inner" style="transform:translate(2.2pt,-0.4pt) scale(1.01011338988195,1.01011338988195) ;">
<table id="A3.T5.2.1" class="ltx_tabular ltx_align_middle">
<tr id="A3.T5.2.1.1" class="ltx_tr">
<td id="A3.T5.2.1.1.1" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T5.2.1.1.1.1" class="ltx_text ltx_font_bold">Parameter</span></td>
<td id="A3.T5.2.1.1.2" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T5.2.1.1.2.1" class="ltx_text ltx_font_bold">Description</span></td>
<td id="A3.T5.2.1.1.3" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T5.2.1.1.3.1" class="ltx_text ltx_font_bold">Search</span></td>
<td id="A3.T5.2.1.1.4" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T5.2.1.1.4.1" class="ltx_text ltx_font_bold">Selected</span></td>
</tr>
<tr id="A3.T5.2.1.2" class="ltx_tr">
<td id="A3.T5.2.1.2.1" class="ltx_td ltx_align_left ltx_border_t">Alpha</td>
<td id="A3.T5.2.1.2.2" class="ltx_td ltx_align_left ltx_border_t">The weight for the retain loss</td>
<td id="A3.T5.2.1.2.3" class="ltx_td ltx_align_left ltx_border_t">[1, 10, 100, 1000, 10000]</td>
<td id="A3.T5.2.1.2.4" class="ltx_td ltx_align_left ltx_border_t">1000</td>
</tr>
<tr id="A3.T5.2.1.3" class="ltx_tr">
<td id="A3.T5.2.1.3.1" class="ltx_td ltx_align_left">Layer</td>
<td id="A3.T5.2.1.3.2" class="ltx_td ltx_align_left">The layer to modify</td>
<td id="A3.T5.2.1.3.3" class="ltx_td ltx_align_left">[3, 17]</td>
<td id="A3.T5.2.1.3.4" class="ltx_td ltx_align_left">3</td>
</tr>
<tr id="A3.T5.2.1.4" class="ltx_tr">
<td id="A3.T5.2.1.4.1" class="ltx_td ltx_align_left ltx_border_bb">Maximum Batches</td>
<td id="A3.T5.2.1.4.2" class="ltx_td ltx_align_left ltx_border_bb">Training duration RMU in batches</td>
<td id="A3.T5.2.1.4.3" class="ltx_td ltx_align_left ltx_border_bb">[100, 150, 200, 250, 300]</td>
<td id="A3.T5.2.1.4.4" class="ltx_td ltx_align_left ltx_border_bb">250</td>
</tr>
</table>
</span></div>
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_table"><span id="A3.T5.4.1.1" class="ltx_text" style="font-size:90%;">Table 5</span>: </span><span id="A3.T5.5.2" class="ltx_text ltx_font_bold" style="font-size:90%;">RMU Hyperparameter Search.<span id="A3.T5.5.2.1" class="ltx_text ltx_font_medium"> Details for the grid search over possible RMU settings. Hyperparameters were selected using grid search.</span></span></figcaption>
</figure>
</section>
<section id="A3.SS4.SSS2" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.4.2 </span>Gradient Ascent</h4>

<div id="A3.SS4.SSS2.p1" class="ltx_para">
<p id="A3.SS4.SSS2.p1.1" class="ltx_p">GA, a simple and widely studied unlearning method <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib101" title="" class="ltx_ref">101</a>, <a href="#bib.bib61" title="" class="ltx_ref">61</a>, <a href="#bib.bib39" title="" class="ltx_ref">39</a>, <a href="#bib.bib102" title="" class="ltx_ref">102</a>, <a href="#bib.bib60" title="" class="ltx_ref">60</a>]</cite>, compels the model to minimize the likelihood of the correct token. The underlying intuition is that models will learn to avoid generating tokens associated with the unlearning target. We only train the final 16 layers of the model due to memory constraints.</p>
</div>
<figure id="A3.T6" class="ltx_table">
<div id="A3.T6.2" class="ltx_inline-block ltx_align_center ltx_transformed_outer" style="width:433.6pt;height:51.3pt;vertical-align:-1.9pt;"><span class="ltx_transformed_inner" style="transform:translate(-23.4pt,2.7pt) scale(0.902589932659547,0.902589932659547) ;">
<table id="A3.T6.2.1" class="ltx_tabular ltx_align_middle">
<tr id="A3.T6.2.1.1" class="ltx_tr">
<td id="A3.T6.2.1.1.1" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T6.2.1.1.1.1" class="ltx_text ltx_font_bold">Parameter</span></td>
<td id="A3.T6.2.1.1.2" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T6.2.1.1.2.1" class="ltx_text ltx_font_bold">Description</span></td>
<td id="A3.T6.2.1.1.3" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T6.2.1.1.3.1" class="ltx_text ltx_font_bold">Search</span></td>
<td id="A3.T6.2.1.1.4" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T6.2.1.1.4.1" class="ltx_text ltx_font_bold">Selected</span></td>
</tr>
<tr id="A3.T6.2.1.2" class="ltx_tr">
<td id="A3.T6.2.1.2.1" class="ltx_td ltx_align_left ltx_border_t">LR</td>
<td id="A3.T6.2.1.2.2" class="ltx_td ltx_align_left ltx_border_t">The learning rate</td>
<td id="A3.T6.2.1.2.3" class="ltx_td ltx_align_left ltx_border_t">[5e-6, 1e-6, … 5e-3, 1e-3]</td>
<td id="A3.T6.2.1.2.4" class="ltx_td ltx_align_left ltx_border_t">5e-5</td>
</tr>
<tr id="A3.T6.2.1.3" class="ltx_tr">
<td id="A3.T6.2.1.3.1" class="ltx_td ltx_align_left ltx_border_bb">Training Samples</td>
<td id="A3.T6.2.1.3.2" class="ltx_td ltx_align_left ltx_border_bb">Train size balanced across Cyber/Bio</td>
<td id="A3.T6.2.1.3.3" class="ltx_td ltx_align_left ltx_border_bb">[10, 25, 50, 100, … 500, 1000, 2000]</td>
<td id="A3.T6.2.1.3.4" class="ltx_td ltx_align_left ltx_border_bb">50</td>
</tr>
</table>
</span></div>
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_table"><span id="A3.T6.4.1.1" class="ltx_text" style="font-size:90%;">Table 6</span>: </span><span id="A3.T6.5.2" class="ltx_text ltx_font_bold" style="font-size:90%;">GA Hyperparameter Search.<span id="A3.T6.5.2.1" class="ltx_text ltx_font_medium"> Details for the grid search over possible GA settings. Hyperparameters were selected using grid search.</span></span></figcaption>
</figure>
</section>
<section id="A3.SS4.SSS3" class="ltx_subsubsection">
<h4 class="ltx_title ltx_title_subsubsection">
<span class="ltx_tag ltx_tag_subsubsection">C.4.3 </span>Gradient Difference</h4>

<div id="A3.SS4.SSS3.p1" class="ltx_para">
<p id="A3.SS4.SSS3.p1.1" class="ltx_p">A significant challenge when leveraging GA is balancing reducing performance on the unlearning target while retaining overall model utility. Since GA reduces the likelihood of the correct token, it is fragile to catastrophic forgetting, where the model incorrectly generalizes to forgetting how to model natural language.</p>
</div>
<div id="A3.SS4.SSS3.p2" class="ltx_para">
<p id="A3.SS4.SSS3.p2.1" class="ltx_p">GD <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib39" title="" class="ltx_ref">39</a>]</cite> aims to address this shortcoming by adding a retain loss term to the loss function. This term aims to maximize the likelihood of the correct token on an unrelated retain dataset. We use Wikitetx <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib100" title="" class="ltx_ref">100</a>]</cite> as the retain set. As with GA, we only train the final 16 layers of the model.</p>
</div>
<div id="A3.SS4.SSS3.p3" class="ltx_para">
<p id="A3.SS4.SSS3.p3.1" class="ltx_p">Table <a href="#A3.T7" title="Table 7 ‣ C.4.3 Gradient Difference ‣ C.4 Machine Unlearning Details ‣ Appendix C Implementation Details ‣ Acknowledgements ‣ 5 Conclusion ‣ 4.4 General Trends in Composability ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">7</span></a> reports our hyperparameter search. Unlike <cite class="ltx_cite ltx_citemacro_cite">[<a href="#bib.bib39" title="" class="ltx_ref">39</a>]</cite>, which found that GD provided only a marginal improvement over GA, we found that GD can perform significantly better if we upweight the retain loss term. Upweighting is necessary since it is easy for the model to significantly minimize the correct token likelihood, which causes it to drown out the retain loss term. Assigning a weight of 400 to the retain term solved this issue. This hyperparameter combination can make GD competitive with RMU, though still less composable (Section <a href="#S4" title="4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">4</span></a>).</p>
</div>
<figure id="A3.T7" class="ltx_table">
<div id="A3.T7.2" class="ltx_inline-block ltx_align_center ltx_transformed_outer" style="width:433.6pt;height:67.3pt;vertical-align:-1.9pt;"><span class="ltx_transformed_inner" style="transform:translate(-24.8pt,3.7pt) scale(0.897401221340423,0.897401221340423) ;">
<table id="A3.T7.2.1" class="ltx_tabular ltx_align_middle">
<tr id="A3.T7.2.1.1" class="ltx_tr">
<td id="A3.T7.2.1.1.1" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T7.2.1.1.1.1" class="ltx_text ltx_font_bold">Parameter</span></td>
<td id="A3.T7.2.1.1.2" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T7.2.1.1.2.1" class="ltx_text ltx_font_bold">Description</span></td>
<td id="A3.T7.2.1.1.3" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T7.2.1.1.3.1" class="ltx_text ltx_font_bold">Search</span></td>
<td id="A3.T7.2.1.1.4" class="ltx_td ltx_align_left ltx_border_tt"><span id="A3.T7.2.1.1.4.1" class="ltx_text ltx_font_bold">Selected</span></td>
</tr>
<tr id="A3.T7.2.1.2" class="ltx_tr">
<td id="A3.T7.2.1.2.1" class="ltx_td ltx_align_left ltx_border_t">Alpha</td>
<td id="A3.T7.2.1.2.2" class="ltx_td ltx_align_left ltx_border_t">The weight for the retain loss</td>
<td id="A3.T7.2.1.2.3" class="ltx_td ltx_align_left ltx_border_t">[1, 2, 4, 6, 8, 10, 20, 40, 60, 80, 100]</td>
<td id="A3.T7.2.1.2.4" class="ltx_td ltx_align_left ltx_border_t">40</td>
</tr>
<tr id="A3.T7.2.1.3" class="ltx_tr">
<td id="A3.T7.2.1.3.1" class="ltx_td ltx_align_left">LR</td>
<td id="A3.T7.2.1.3.2" class="ltx_td ltx_align_left">The learning rate</td>
<td id="A3.T7.2.1.3.3" class="ltx_td ltx_align_left">[5e-6, 1e-6, … 5e-3, 1e-3]</td>
<td id="A3.T7.2.1.3.4" class="ltx_td ltx_align_left">5e-5</td>
</tr>
<tr id="A3.T7.2.1.4" class="ltx_tr">
<td id="A3.T7.2.1.4.1" class="ltx_td ltx_align_left ltx_border_bb">Training Samples</td>
<td id="A3.T7.2.1.4.2" class="ltx_td ltx_align_left ltx_border_bb">Train size balanced across Cyber/Bio</td>
<td id="A3.T7.2.1.4.3" class="ltx_td ltx_align_left ltx_border_bb">[10, 25, 50, 100, … 500, 1000, 2000]</td>
<td id="A3.T7.2.1.4.4" class="ltx_td ltx_align_left ltx_border_bb">50</td>
</tr>
</table>
</span></div>
<figcaption class="ltx_caption ltx_centering"><span class="ltx_tag ltx_tag_table"><span id="A3.T7.4.1.1" class="ltx_text" style="font-size:90%;">Table 7</span>: </span><span id="A3.T7.5.2" class="ltx_text ltx_font_bold" style="font-size:90%;">GD Hyperparameter Search.<span id="A3.T7.5.2.1" class="ltx_text ltx_font_medium"> Details for the grid search over possible GD settings. Hyperparameters were selected using grid search.</span></span></figcaption>
</figure>
</section>
</section>
</section>
<section id="A4" class="ltx_appendix">
<h2 class="ltx_title ltx_title_appendix">
<span class="ltx_tag ltx_tag_appendix">Appendix D </span>Extended Results</h2>

<div id="A4.p1" class="ltx_para">
<p id="A4.p1.1" class="ltx_p">We report the task performance for all compositions below and the baseline results, where only a single intervention is applied (Table <a href="#A4" title="Appendix D Extended Results ‣ Acknowledgements ‣ 5 Conclusion ‣ 4.4 General Trends in Composability ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">D</span></a>). Perplexity on Wikitext <cite class="ltx_cite ltx_citemacro_citep">[<a href="#bib.bib100" title="" class="ltx_ref">100</a>]</cite> is also included as another overall utility metric. Average Bits (Appendix <a href="#A3.SS3.SSS1" title="C.3.1 Compression Metrics ‣ C.3 Model Compression Details ‣ Appendix C Implementation Details ‣ Acknowledgements ‣ 5 Conclusion ‣ 4.4 General Trends in Composability ‣ 4.3 Composing Knowledge Editing with Machine Unlearning ‣ 4.2 Composing Model Compression with Machine Unlearning ‣ 4.1 Composing Model Compression with Knowledge Editing ‣ 4 Results ‣ Composable Interventions for Language Models" class="ltx_ref"><span class="ltx_text ltx_ref_tag">C.3.1</span></a>) measures the degree to which the composition compresses the model.</p>
</div>
<figure id="A4.322" class="ltx_table">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="A4.28.28" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:443.7pt;vertical-align:-1.6pt;"><span class="ltx_transformed_inner" style="transform:translate(-74.6pt,76.0pt) scale(0.744091467228088,0.744091467228088) ;">
<table id="A4.28.28.28" class="ltx_tabular ltx_align_middle">
<tr id="A4.28.28.28.29" class="ltx_tr">
<td id="A4.28.28.28.29.1" class="ltx_td ltx_border_tt"></td>
<td id="A4.28.28.28.29.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="3">Editing</td>
<td id="A4.28.28.28.29.3" class="ltx_td ltx_align_center ltx_border_tt">Compression</td>
<td id="A4.28.28.28.29.4" class="ltx_td ltx_align_center ltx_border_tt">Unlearning</td>
<td id="A4.28.28.28.29.5" class="ltx_td ltx_align_center ltx_border_tt" colspan="2">Utility</td>
</tr>
<tr id="A4.28.28.28.30" class="ltx_tr">
<td id="A4.28.28.28.30.1" class="ltx_td"></td>
<td id="A4.28.28.28.30.2" class="ltx_td ltx_align_center ltx_border_t">Edit Success</td>
<td id="A4.28.28.28.30.3" class="ltx_td ltx_align_center ltx_border_t">Generalization</td>
<td id="A4.28.28.28.30.4" class="ltx_td ltx_align_center ltx_border_t">Locality</td>
<td id="A4.28.28.28.30.5" class="ltx_td ltx_align_center ltx_border_t">Avg. Bits</td>
<td id="A4.28.28.28.30.6" class="ltx_td ltx_align_center ltx_border_t">Avg. WMDP</td>
<td id="A4.28.28.28.30.7" class="ltx_td ltx_align_center ltx_border_t">MMLU</td>
<td id="A4.28.28.28.30.8" class="ltx_td ltx_align_center ltx_border_t">WikiText PPL</td>
</tr>
<tr id="A4.28.28.28.31" class="ltx_tr">
<td id="A4.28.28.28.31.1" class="ltx_td ltx_align_left ltx_border_t">None</td>
<td id="A4.28.28.28.31.2" class="ltx_td ltx_align_center ltx_border_t">0.02</td>
<td id="A4.28.28.28.31.3" class="ltx_td ltx_align_center ltx_border_t">0.02</td>
<td id="A4.28.28.28.31.4" class="ltx_td ltx_align_center ltx_border_t">0.04</td>
<td id="A4.28.28.28.31.5" class="ltx_td ltx_align_center ltx_border_t">16.00</td>
<td id="A4.28.28.28.31.6" class="ltx_td ltx_align_center ltx_border_t">0.58</td>
<td id="A4.28.28.28.31.7" class="ltx_td ltx_align_center ltx_border_t">0.62</td>
<td id="A4.28.28.28.31.8" class="ltx_td ltx_align_center ltx_border_t">5.54</td>
</tr>
<tr id="A4.1.1.1.1" class="ltx_tr">
<td id="A4.1.1.1.1.1" class="ltx_td ltx_align_left">
<span id="A4.1.1.1.1.1.1" class="ltx_ERROR undefined">\cdashline</span>1-9

FT<math id="A4.1.1.1.1.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.1.1.1.1.1.m1.1a"><mo stretchy="false" id="A4.1.1.1.1.1.m1.1.1" xref="A4.1.1.1.1.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.1.1.1.1.1.m1.1b"><ci id="A4.1.1.1.1.1.m1.1.1.cmml" xref="A4.1.1.1.1.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.1.1.1.1.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.1.1.1.1.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.1.1.1.1.3" class="ltx_td ltx_align_center">0.82</td>
<td id="A4.1.1.1.1.4" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.1.1.1.1.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.1.1.1.1.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.1.1.1.1.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.1.1.1.1.8" class="ltx_td ltx_align_center">5.57</td>
</tr>
<tr id="A4.2.2.2.2" class="ltx_tr">
<td id="A4.2.2.2.2.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.2.2.2.2.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.2.2.2.2.1.m1.1a"><mo stretchy="false" id="A4.2.2.2.2.1.m1.1.1" xref="A4.2.2.2.2.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.2.2.2.2.1.m1.1b"><ci id="A4.2.2.2.2.1.m1.1.1.cmml" xref="A4.2.2.2.2.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.2.2.2.2.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.2.2.2.2.2" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.2.2.2.2.3" class="ltx_td ltx_align_center">0.85</td>
<td id="A4.2.2.2.2.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.2.2.2.2.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.2.2.2.2.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.2.2.2.2.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.2.2.2.2.8" class="ltx_td ltx_align_center">5.57</td>
</tr>
<tr id="A4.3.3.3.3" class="ltx_tr">
<td id="A4.3.3.3.3.1" class="ltx_td ltx_align_left">LoRA<math id="A4.3.3.3.3.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.3.3.3.3.1.m1.1a"><mo stretchy="false" id="A4.3.3.3.3.1.m1.1.1" xref="A4.3.3.3.3.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.3.3.3.3.1.m1.1b"><ci id="A4.3.3.3.3.1.m1.1.1.cmml" xref="A4.3.3.3.3.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.3.3.3.3.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.3.3.3.3.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.3.3.3.3.3" class="ltx_td ltx_align_center">0.71</td>
<td id="A4.3.3.3.3.4" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.3.3.3.3.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.3.3.3.3.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.3.3.3.3.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.3.3.3.3.8" class="ltx_td ltx_align_center">19.25</td>
</tr>
<tr id="A4.4.4.4.4" class="ltx_tr">
<td id="A4.4.4.4.4.1" class="ltx_td ltx_align_left">
<span id="A4.4.4.4.4.1.1" class="ltx_ERROR undefined">\cdashline</span>1-9

SparseGPT (0.25) <math id="A4.4.4.4.4.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.4.4.4.4.1.m1.1a"><mo stretchy="false" id="A4.4.4.4.4.1.m1.1.1" xref="A4.4.4.4.4.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.4.4.4.4.1.m1.1b"><ci id="A4.4.4.4.4.1.m1.1.1.cmml" xref="A4.4.4.4.4.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.4.4.4.4.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.4.4.4.4.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.4.4.4.4.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.4.4.4.4.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.4.4.4.4.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.4.4.4.4.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.4.4.4.4.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.4.4.4.4.8" class="ltx_td ltx_align_center">5.87</td>
</tr>
<tr id="A4.5.5.5.5" class="ltx_tr">
<td id="A4.5.5.5.5.1" class="ltx_td ltx_align_left">SparseGPT (0.35) <math id="A4.5.5.5.5.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.5.5.5.5.1.m1.1a"><mo stretchy="false" id="A4.5.5.5.5.1.m1.1.1" xref="A4.5.5.5.5.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.5.5.5.5.1.m1.1b"><ci id="A4.5.5.5.5.1.m1.1.1.cmml" xref="A4.5.5.5.5.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.5.5.5.5.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.5.5.5.5.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.5.5.5.5.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.5.5.5.5.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.5.5.5.5.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.5.5.5.5.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.5.5.5.5.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.5.5.5.5.8" class="ltx_td ltx_align_center">6.34</td>
</tr>
<tr id="A4.6.6.6.6" class="ltx_tr">
<td id="A4.6.6.6.6.1" class="ltx_td ltx_align_left">SparseGPT (0.45) <math id="A4.6.6.6.6.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.6.6.6.6.1.m1.1a"><mo stretchy="false" id="A4.6.6.6.6.1.m1.1.1" xref="A4.6.6.6.6.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.6.6.6.6.1.m1.1b"><ci id="A4.6.6.6.6.1.m1.1.1.cmml" xref="A4.6.6.6.6.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.6.6.6.6.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.6.6.6.6.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.6.6.6.6.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.6.6.6.6.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.6.6.6.6.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.6.6.6.6.6" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.6.6.6.6.7" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.6.6.6.6.8" class="ltx_td ltx_align_center">7.43</td>
</tr>
<tr id="A4.7.7.7.7" class="ltx_tr">
<td id="A4.7.7.7.7.1" class="ltx_td ltx_align_left">SparseGPT (0.55) <math id="A4.7.7.7.7.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.7.7.7.7.1.m1.1a"><mo stretchy="false" id="A4.7.7.7.7.1.m1.1.1" xref="A4.7.7.7.7.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.7.7.7.7.1.m1.1b"><ci id="A4.7.7.7.7.1.m1.1.1.cmml" xref="A4.7.7.7.7.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.7.7.7.7.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.7.7.7.7.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.7.7.7.7.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.7.7.7.7.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.7.7.7.7.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.7.7.7.7.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.7.7.7.7.7" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.7.7.7.7.8" class="ltx_td ltx_align_center">10.43</td>
</tr>
<tr id="A4.8.8.8.8" class="ltx_tr">
<td id="A4.8.8.8.8.1" class="ltx_td ltx_align_left">SparseGPT (0.65) <math id="A4.8.8.8.8.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.8.8.8.8.1.m1.1a"><mo stretchy="false" id="A4.8.8.8.8.1.m1.1.1" xref="A4.8.8.8.8.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.8.8.8.8.1.m1.1b"><ci id="A4.8.8.8.8.1.m1.1.1.cmml" xref="A4.8.8.8.8.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.8.8.8.8.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.8.8.8.8.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.8.8.8.8.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.8.8.8.8.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.8.8.8.8.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.8.8.8.8.6" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.8.8.8.8.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.8.8.8.8.8" class="ltx_td ltx_align_center">20.83</td>
</tr>
<tr id="A4.9.9.9.9" class="ltx_tr">
<td id="A4.9.9.9.9.1" class="ltx_td ltx_align_left">SparseGPT (0.75) <math id="A4.9.9.9.9.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.9.9.9.9.1.m1.1a"><mo stretchy="false" id="A4.9.9.9.9.1.m1.1.1" xref="A4.9.9.9.9.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.9.9.9.9.1.m1.1b"><ci id="A4.9.9.9.9.1.m1.1.1.cmml" xref="A4.9.9.9.9.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.9.9.9.9.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.9.9.9.9.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.9.9.9.9.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.9.9.9.9.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.9.9.9.9.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.9.9.9.9.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.9.9.9.9.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.9.9.9.9.8" class="ltx_td ltx_align_center">81.84</td>
</tr>
<tr id="A4.10.10.10.10" class="ltx_tr">
<td id="A4.10.10.10.10.1" class="ltx_td ltx_align_left">Wanda (0.25) <math id="A4.10.10.10.10.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.10.10.10.10.1.m1.1a"><mo stretchy="false" id="A4.10.10.10.10.1.m1.1.1" xref="A4.10.10.10.10.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.10.10.10.10.1.m1.1b"><ci id="A4.10.10.10.10.1.m1.1.1.cmml" xref="A4.10.10.10.10.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.10.10.10.10.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.10.10.10.10.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.10.10.10.10.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.10.10.10.10.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.10.10.10.10.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.10.10.10.10.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.10.10.10.10.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.10.10.10.10.8" class="ltx_td ltx_align_center">5.84</td>
</tr>
<tr id="A4.11.11.11.11" class="ltx_tr">
<td id="A4.11.11.11.11.1" class="ltx_td ltx_align_left">Wanda (0.35) <math id="A4.11.11.11.11.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.11.11.11.11.1.m1.1a"><mo stretchy="false" id="A4.11.11.11.11.1.m1.1.1" xref="A4.11.11.11.11.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.11.11.11.11.1.m1.1b"><ci id="A4.11.11.11.11.1.m1.1.1.cmml" xref="A4.11.11.11.11.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.11.11.11.11.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.11.11.11.11.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.11.11.11.11.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.11.11.11.11.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.11.11.11.11.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.11.11.11.11.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.11.11.11.11.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.11.11.11.11.8" class="ltx_td ltx_align_center">6.31</td>
</tr>
<tr id="A4.12.12.12.12" class="ltx_tr">
<td id="A4.12.12.12.12.1" class="ltx_td ltx_align_left">Wanda (0.45) <math id="A4.12.12.12.12.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.12.12.12.12.1.m1.1a"><mo stretchy="false" id="A4.12.12.12.12.1.m1.1.1" xref="A4.12.12.12.12.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.12.12.12.12.1.m1.1b"><ci id="A4.12.12.12.12.1.m1.1.1.cmml" xref="A4.12.12.12.12.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.12.12.12.12.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.12.12.12.12.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.12.12.12.12.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.12.12.12.12.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.12.12.12.12.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.12.12.12.12.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.12.12.12.12.7" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.12.12.12.12.8" class="ltx_td ltx_align_center">7.52</td>
</tr>
<tr id="A4.13.13.13.13" class="ltx_tr">
<td id="A4.13.13.13.13.1" class="ltx_td ltx_align_left">Wanda (0.55) <math id="A4.13.13.13.13.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.13.13.13.13.1.m1.1a"><mo stretchy="false" id="A4.13.13.13.13.1.m1.1.1" xref="A4.13.13.13.13.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.13.13.13.13.1.m1.1b"><ci id="A4.13.13.13.13.1.m1.1.1.cmml" xref="A4.13.13.13.13.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.13.13.13.13.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.13.13.13.13.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.13.13.13.13.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.13.13.13.13.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.13.13.13.13.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.13.13.13.13.6" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.13.13.13.13.7" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.13.13.13.13.8" class="ltx_td ltx_align_center">12.5</td>
</tr>
<tr id="A4.14.14.14.14" class="ltx_tr">
<td id="A4.14.14.14.14.1" class="ltx_td ltx_align_left">Wanda (0.65) <math id="A4.14.14.14.14.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.14.14.14.14.1.m1.1a"><mo stretchy="false" id="A4.14.14.14.14.1.m1.1.1" xref="A4.14.14.14.14.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.14.14.14.14.1.m1.1b"><ci id="A4.14.14.14.14.1.m1.1.1.cmml" xref="A4.14.14.14.14.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.14.14.14.14.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.14.14.14.14.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.14.14.14.14.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.14.14.14.14.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.14.14.14.14.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.14.14.14.14.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.14.14.14.14.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.14.14.14.14.8" class="ltx_td ltx_align_center">47.18</td>
</tr>
<tr id="A4.15.15.15.15" class="ltx_tr">
<td id="A4.15.15.15.15.1" class="ltx_td ltx_align_left">Wanda (0.75) <math id="A4.15.15.15.15.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.15.15.15.15.1.m1.1a"><mo stretchy="false" id="A4.15.15.15.15.1.m1.1.1" xref="A4.15.15.15.15.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.15.15.15.15.1.m1.1b"><ci id="A4.15.15.15.15.1.m1.1.1.cmml" xref="A4.15.15.15.15.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.15.15.15.15.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.15.15.15.15.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.15.15.15.15.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.15.15.15.15.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.15.15.15.15.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.15.15.15.15.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.15.15.15.15.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.15.15.15.15.8" class="ltx_td ltx_align_center">257.46</td>
</tr>
<tr id="A4.16.16.16.16" class="ltx_tr">
<td id="A4.16.16.16.16.1" class="ltx_td ltx_align_left">GPTQ (2-Bit) <math id="A4.16.16.16.16.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.16.16.16.16.1.m1.1a"><mo stretchy="false" id="A4.16.16.16.16.1.m1.1.1" xref="A4.16.16.16.16.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.16.16.16.16.1.m1.1b"><ci id="A4.16.16.16.16.1.m1.1.1.cmml" xref="A4.16.16.16.16.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.16.16.16.16.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.16.16.16.16.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.16.16.16.16.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.16.16.16.16.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.16.16.16.16.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.16.16.16.16.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.16.16.16.16.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.16.16.16.16.8" class="ltx_td ltx_align_center">3681.23</td>
</tr>
<tr id="A4.17.17.17.17" class="ltx_tr">
<td id="A4.17.17.17.17.1" class="ltx_td ltx_align_left">GPTQ (3-Bit) <math id="A4.17.17.17.17.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.17.17.17.17.1.m1.1a"><mo stretchy="false" id="A4.17.17.17.17.1.m1.1.1" xref="A4.17.17.17.17.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.17.17.17.17.1.m1.1b"><ci id="A4.17.17.17.17.1.m1.1.1.cmml" xref="A4.17.17.17.17.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.17.17.17.17.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.17.17.17.17.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.17.17.17.17.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.17.17.17.17.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.17.17.17.17.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.17.17.17.17.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.17.17.17.17.7" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.17.17.17.17.8" class="ltx_td ltx_align_center">8.6</td>
</tr>
<tr id="A4.18.18.18.18" class="ltx_tr">
<td id="A4.18.18.18.18.1" class="ltx_td ltx_align_left">GPTQ (4-Bit) <math id="A4.18.18.18.18.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.18.18.18.18.1.m1.1a"><mo stretchy="false" id="A4.18.18.18.18.1.m1.1.1" xref="A4.18.18.18.18.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.18.18.18.18.1.m1.1b"><ci id="A4.18.18.18.18.1.m1.1.1.cmml" xref="A4.18.18.18.18.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.18.18.18.18.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.18.18.18.18.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.18.18.18.18.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.18.18.18.18.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.18.18.18.18.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.18.18.18.18.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.18.18.18.18.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.18.18.18.18.8" class="ltx_td ltx_align_center">9.97</td>
</tr>
<tr id="A4.19.19.19.19" class="ltx_tr">
<td id="A4.19.19.19.19.1" class="ltx_td ltx_align_left">GPTQ (8-Bit) <math id="A4.19.19.19.19.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.19.19.19.19.1.m1.1a"><mo stretchy="false" id="A4.19.19.19.19.1.m1.1.1" xref="A4.19.19.19.19.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.19.19.19.19.1.m1.1b"><ci id="A4.19.19.19.19.1.m1.1.1.cmml" xref="A4.19.19.19.19.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.19.19.19.19.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.19.19.19.19.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.19.19.19.19.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.19.19.19.19.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.19.19.19.19.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.19.19.19.19.6" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.19.19.19.19.7" class="ltx_td ltx_align_center">0.62</td>
<td id="A4.19.19.19.19.8" class="ltx_td ltx_align_center">5.54</td>
</tr>
<tr id="A4.20.20.20.20" class="ltx_tr">
<td id="A4.20.20.20.20.1" class="ltx_td ltx_align_left">AWQ (2-Bit) <math id="A4.20.20.20.20.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.20.20.20.20.1.m1.1a"><mo stretchy="false" id="A4.20.20.20.20.1.m1.1.1" xref="A4.20.20.20.20.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.20.20.20.20.1.m1.1b"><ci id="A4.20.20.20.20.1.m1.1.1.cmml" xref="A4.20.20.20.20.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.20.20.20.20.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.20.20.20.20.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.20.20.20.20.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.20.20.20.20.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.20.20.20.20.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.20.20.20.20.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.20.20.20.20.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.20.20.20.20.8" class="ltx_td ltx_align_center">1748954.75</td>
</tr>
<tr id="A4.21.21.21.21" class="ltx_tr">
<td id="A4.21.21.21.21.1" class="ltx_td ltx_align_left">AWQ (3-Bit) <math id="A4.21.21.21.21.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.21.21.21.21.1.m1.1a"><mo stretchy="false" id="A4.21.21.21.21.1.m1.1.1" xref="A4.21.21.21.21.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.21.21.21.21.1.m1.1b"><ci id="A4.21.21.21.21.1.m1.1.1.cmml" xref="A4.21.21.21.21.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.21.21.21.21.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.21.21.21.21.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.21.21.21.21.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.21.21.21.21.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.21.21.21.21.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.21.21.21.21.6" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.21.21.21.21.7" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.21.21.21.21.8" class="ltx_td ltx_align_center">7.47</td>
</tr>
<tr id="A4.22.22.22.22" class="ltx_tr">
<td id="A4.22.22.22.22.1" class="ltx_td ltx_align_left">AWQ (4-Bit) <math id="A4.22.22.22.22.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.22.22.22.22.1.m1.1a"><mo stretchy="false" id="A4.22.22.22.22.1.m1.1.1" xref="A4.22.22.22.22.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.22.22.22.22.1.m1.1b"><ci id="A4.22.22.22.22.1.m1.1.1.cmml" xref="A4.22.22.22.22.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.22.22.22.22.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.22.22.22.22.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.22.22.22.22.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.22.22.22.22.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.22.22.22.22.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.22.22.22.22.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.22.22.22.22.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.22.22.22.22.8" class="ltx_td ltx_align_center">5.91</td>
</tr>
<tr id="A4.23.23.23.23" class="ltx_tr">
<td id="A4.23.23.23.23.1" class="ltx_td ltx_align_left">AWQ (5-Bit) <math id="A4.23.23.23.23.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.23.23.23.23.1.m1.1a"><mo stretchy="false" id="A4.23.23.23.23.1.m1.1.1" xref="A4.23.23.23.23.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.23.23.23.23.1.m1.1b"><ci id="A4.23.23.23.23.1.m1.1.1.cmml" xref="A4.23.23.23.23.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.23.23.23.23.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.23.23.23.23.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.23.23.23.23.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.23.23.23.23.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.23.23.23.23.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.23.23.23.23.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.23.23.23.23.7" class="ltx_td ltx_align_center">0.62</td>
<td id="A4.23.23.23.23.8" class="ltx_td ltx_align_center">5.62</td>
</tr>
<tr id="A4.24.24.24.24" class="ltx_tr">
<td id="A4.24.24.24.24.1" class="ltx_td ltx_align_left">AWQ (6-Bit) <math id="A4.24.24.24.24.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.24.24.24.24.1.m1.1a"><mo stretchy="false" id="A4.24.24.24.24.1.m1.1.1" xref="A4.24.24.24.24.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.24.24.24.24.1.m1.1b"><ci id="A4.24.24.24.24.1.m1.1.1.cmml" xref="A4.24.24.24.24.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.24.24.24.24.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.24.24.24.24.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.24.24.24.24.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.24.24.24.24.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.24.24.24.24.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.24.24.24.24.6" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.24.24.24.24.7" class="ltx_td ltx_align_center">0.62</td>
<td id="A4.24.24.24.24.8" class="ltx_td ltx_align_center">5.56</td>
</tr>
<tr id="A4.25.25.25.25" class="ltx_tr">
<td id="A4.25.25.25.25.1" class="ltx_td ltx_align_left">AWQ (8-Bit) <math id="A4.25.25.25.25.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.25.25.25.25.1.m1.1a"><mo stretchy="false" id="A4.25.25.25.25.1.m1.1.1" xref="A4.25.25.25.25.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.25.25.25.25.1.m1.1b"><ci id="A4.25.25.25.25.1.m1.1.1.cmml" xref="A4.25.25.25.25.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.25.25.25.25.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.25.25.25.25.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.25.25.25.25.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.25.25.25.25.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.25.25.25.25.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.25.25.25.25.6" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.25.25.25.25.7" class="ltx_td ltx_align_center">0.62</td>
<td id="A4.25.25.25.25.8" class="ltx_td ltx_align_center">5.54</td>
</tr>
<tr id="A4.26.26.26.26" class="ltx_tr">
<td id="A4.26.26.26.26.1" class="ltx_td ltx_align_left">
<span id="A4.26.26.26.26.1.1" class="ltx_ERROR undefined">\cdashline</span>1-9

GA<math id="A4.26.26.26.26.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.26.26.26.26.1.m1.1a"><mo stretchy="false" id="A4.26.26.26.26.1.m1.1.1" xref="A4.26.26.26.26.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.26.26.26.26.1.m1.1b"><ci id="A4.26.26.26.26.1.m1.1.1.cmml" xref="A4.26.26.26.26.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.26.26.26.26.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.26.26.26.26.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.26.26.26.26.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.26.26.26.26.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.26.26.26.26.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.26.26.26.26.6" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.26.26.26.26.7" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.26.26.26.26.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.27.27.27.27" class="ltx_tr">
<td id="A4.27.27.27.27.1" class="ltx_td ltx_align_left">GD<math id="A4.27.27.27.27.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.27.27.27.27.1.m1.1a"><mo stretchy="false" id="A4.27.27.27.27.1.m1.1.1" xref="A4.27.27.27.27.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.27.27.27.27.1.m1.1b"><ci id="A4.27.27.27.27.1.m1.1.1.cmml" xref="A4.27.27.27.27.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.27.27.27.27.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.27.27.27.27.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.27.27.27.27.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.27.27.27.27.4" class="ltx_td ltx_align_center">0.09</td>
<td id="A4.27.27.27.27.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.27.27.27.27.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.27.27.27.27.7" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.27.27.27.27.8" class="ltx_td ltx_align_center">4.48</td>
</tr>
<tr id="A4.28.28.28.28" class="ltx_tr">
<td id="A4.28.28.28.28.1" class="ltx_td ltx_align_left ltx_border_bb">RMU<math id="A4.28.28.28.28.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.28.28.28.28.1.m1.1a"><mo stretchy="false" id="A4.28.28.28.28.1.m1.1.1" xref="A4.28.28.28.28.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.28.28.28.28.1.m1.1b"><ci id="A4.28.28.28.28.1.m1.1.1.cmml" xref="A4.28.28.28.28.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.28.28.28.28.1.m1.1c">\rightarrow</annotation></semantics></math>None</td>
<td id="A4.28.28.28.28.2" class="ltx_td ltx_align_center ltx_border_bb">0.02</td>
<td id="A4.28.28.28.28.3" class="ltx_td ltx_align_center ltx_border_bb">0.02</td>
<td id="A4.28.28.28.28.4" class="ltx_td ltx_align_center ltx_border_bb">0.03</td>
<td id="A4.28.28.28.28.5" class="ltx_td ltx_align_center ltx_border_bb">16.0</td>
<td id="A4.28.28.28.28.6" class="ltx_td ltx_align_center ltx_border_bb">0.29</td>
<td id="A4.28.28.28.28.7" class="ltx_td ltx_align_center ltx_border_bb">0.57</td>
<td id="A4.28.28.28.28.8" class="ltx_td ltx_align_center ltx_border_bb">5.56</td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="A4.322.323.1.1" class="ltx_text" style="font-size:90%;">Table 8</span>: </span><span id="A4.322.324.2" class="ltx_text" style="font-size:90%;">Baseline intervention Results without composition.</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="A4.322.322" class="ltx_table ltx_figure_panel">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="A4.94.94.66" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:925.1pt;vertical-align:-1.6pt;"><span class="ltx_transformed_inner" style="transform:translate(-79.6pt,169.6pt) scale(0.731366876083771,0.731366876083771) ;">
<table id="A4.94.94.66.66" class="ltx_tabular ltx_align_middle">
<tr id="A4.94.94.66.66.67" class="ltx_tr">
<td id="A4.94.94.66.66.67.1" class="ltx_td ltx_border_tt"></td>
<td id="A4.94.94.66.66.67.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="3">Editing</td>
<td id="A4.94.94.66.66.67.3" class="ltx_td ltx_align_center ltx_border_tt">Compression</td>
<td id="A4.94.94.66.66.67.4" class="ltx_td ltx_align_center ltx_border_tt">Unlearning</td>
<td id="A4.94.94.66.66.67.5" class="ltx_td ltx_align_center ltx_border_tt" colspan="2">Utility</td>
</tr>
<tr id="A4.94.94.66.66.68" class="ltx_tr">
<td id="A4.94.94.66.66.68.1" class="ltx_td"></td>
<td id="A4.94.94.66.66.68.2" class="ltx_td ltx_align_center ltx_border_t">Edit Success</td>
<td id="A4.94.94.66.66.68.3" class="ltx_td ltx_align_center ltx_border_t">Generalization</td>
<td id="A4.94.94.66.66.68.4" class="ltx_td ltx_align_center ltx_border_t">Locality</td>
<td id="A4.94.94.66.66.68.5" class="ltx_td ltx_align_center ltx_border_t">Avg. Bits</td>
<td id="A4.94.94.66.66.68.6" class="ltx_td ltx_align_center ltx_border_t">Avg. WMDP</td>
<td id="A4.94.94.66.66.68.7" class="ltx_td ltx_align_center ltx_border_t">MMLU</td>
<td id="A4.94.94.66.66.68.8" class="ltx_td ltx_align_center ltx_border_t">WikiText PPL</td>
</tr>
<tr id="A4.29.29.1.1.1" class="ltx_tr">
<td id="A4.29.29.1.1.1.1" class="ltx_td ltx_align_left ltx_border_t">FT<math id="A4.29.29.1.1.1.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.29.29.1.1.1.1.m1.1a"><mo stretchy="false" id="A4.29.29.1.1.1.1.m1.1.1" xref="A4.29.29.1.1.1.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.29.29.1.1.1.1.m1.1b"><ci id="A4.29.29.1.1.1.1.m1.1.1.cmml" xref="A4.29.29.1.1.1.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.29.29.1.1.1.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.25)</td>
<td id="A4.29.29.1.1.1.2" class="ltx_td ltx_align_center ltx_border_t">0.96</td>
<td id="A4.29.29.1.1.1.3" class="ltx_td ltx_align_center ltx_border_t">0.76</td>
<td id="A4.29.29.1.1.1.4" class="ltx_td ltx_align_center ltx_border_t">0.11</td>
<td id="A4.29.29.1.1.1.5" class="ltx_td ltx_align_center ltx_border_t">12.25</td>
<td id="A4.29.29.1.1.1.6" class="ltx_td ltx_align_center ltx_border_t">0.55</td>
<td id="A4.29.29.1.1.1.7" class="ltx_td ltx_align_center ltx_border_t">0.6</td>
<td id="A4.29.29.1.1.1.8" class="ltx_td ltx_align_center ltx_border_t">5.9</td>
</tr>
<tr id="A4.30.30.2.2.2" class="ltx_tr">
<td id="A4.30.30.2.2.2.1" class="ltx_td ltx_align_left">FT<math id="A4.30.30.2.2.2.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.30.30.2.2.2.1.m1.1a"><mo stretchy="false" id="A4.30.30.2.2.2.1.m1.1.1" xref="A4.30.30.2.2.2.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.30.30.2.2.2.1.m1.1b"><ci id="A4.30.30.2.2.2.1.m1.1.1.cmml" xref="A4.30.30.2.2.2.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.30.30.2.2.2.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.35)</td>
<td id="A4.30.30.2.2.2.2" class="ltx_td ltx_align_center">0.84</td>
<td id="A4.30.30.2.2.2.3" class="ltx_td ltx_align_center">0.66</td>
<td id="A4.30.30.2.2.2.4" class="ltx_td ltx_align_center">0.09</td>
<td id="A4.30.30.2.2.2.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.30.30.2.2.2.6" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.30.30.2.2.2.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.30.30.2.2.2.8" class="ltx_td ltx_align_center">6.38</td>
</tr>
<tr id="A4.31.31.3.3.3" class="ltx_tr">
<td id="A4.31.31.3.3.3.1" class="ltx_td ltx_align_left">FT<math id="A4.31.31.3.3.3.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.31.31.3.3.3.1.m1.1a"><mo stretchy="false" id="A4.31.31.3.3.3.1.m1.1.1" xref="A4.31.31.3.3.3.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.31.31.3.3.3.1.m1.1b"><ci id="A4.31.31.3.3.3.1.m1.1.1.cmml" xref="A4.31.31.3.3.3.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.31.31.3.3.3.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.45)</td>
<td id="A4.31.31.3.3.3.2" class="ltx_td ltx_align_center">0.64</td>
<td id="A4.31.31.3.3.3.3" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.31.31.3.3.3.4" class="ltx_td ltx_align_center">0.07</td>
<td id="A4.31.31.3.3.3.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.31.31.3.3.3.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.31.31.3.3.3.7" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.31.31.3.3.3.8" class="ltx_td ltx_align_center">7.48</td>
</tr>
<tr id="A4.32.32.4.4.4" class="ltx_tr">
<td id="A4.32.32.4.4.4.1" class="ltx_td ltx_align_left">FT<math id="A4.32.32.4.4.4.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.32.32.4.4.4.1.m1.1a"><mo stretchy="false" id="A4.32.32.4.4.4.1.m1.1.1" xref="A4.32.32.4.4.4.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.32.32.4.4.4.1.m1.1b"><ci id="A4.32.32.4.4.4.1.m1.1.1.cmml" xref="A4.32.32.4.4.4.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.32.32.4.4.4.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.55)</td>
<td id="A4.32.32.4.4.4.2" class="ltx_td ltx_align_center">0.33</td>
<td id="A4.32.32.4.4.4.3" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.32.32.4.4.4.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.32.32.4.4.4.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.32.32.4.4.4.6" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.32.32.4.4.4.7" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.32.32.4.4.4.8" class="ltx_td ltx_align_center">10.35</td>
</tr>
<tr id="A4.33.33.5.5.5" class="ltx_tr">
<td id="A4.33.33.5.5.5.1" class="ltx_td ltx_align_left">FT<math id="A4.33.33.5.5.5.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.33.33.5.5.5.1.m1.1a"><mo stretchy="false" id="A4.33.33.5.5.5.1.m1.1.1" xref="A4.33.33.5.5.5.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.33.33.5.5.5.1.m1.1b"><ci id="A4.33.33.5.5.5.1.m1.1.1.cmml" xref="A4.33.33.5.5.5.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.33.33.5.5.5.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.65)</td>
<td id="A4.33.33.5.5.5.2" class="ltx_td ltx_align_center">0.12</td>
<td id="A4.33.33.5.5.5.3" class="ltx_td ltx_align_center">0.1</td>
<td id="A4.33.33.5.5.5.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.33.33.5.5.5.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.33.33.5.5.5.6" class="ltx_td ltx_align_center">0.33</td>
<td id="A4.33.33.5.5.5.7" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.33.33.5.5.5.8" class="ltx_td ltx_align_center">21.54</td>
</tr>
<tr id="A4.34.34.6.6.6" class="ltx_tr">
<td id="A4.34.34.6.6.6.1" class="ltx_td ltx_align_left">FT<math id="A4.34.34.6.6.6.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.34.34.6.6.6.1.m1.1a"><mo stretchy="false" id="A4.34.34.6.6.6.1.m1.1.1" xref="A4.34.34.6.6.6.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.34.34.6.6.6.1.m1.1b"><ci id="A4.34.34.6.6.6.1.m1.1.1.cmml" xref="A4.34.34.6.6.6.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.34.34.6.6.6.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.75)</td>
<td id="A4.34.34.6.6.6.2" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.34.34.6.6.6.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.34.34.6.6.6.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.34.34.6.6.6.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.34.34.6.6.6.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.34.34.6.6.6.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.34.34.6.6.6.8" class="ltx_td ltx_align_center">88.73</td>
</tr>
<tr id="A4.35.35.7.7.7" class="ltx_tr">
<td id="A4.35.35.7.7.7.1" class="ltx_td ltx_align_left">FT<math id="A4.35.35.7.7.7.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.35.35.7.7.7.1.m1.1a"><mo stretchy="false" id="A4.35.35.7.7.7.1.m1.1.1" xref="A4.35.35.7.7.7.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.35.35.7.7.7.1.m1.1b"><ci id="A4.35.35.7.7.7.1.m1.1.1.cmml" xref="A4.35.35.7.7.7.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.35.35.7.7.7.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.25)</td>
<td id="A4.35.35.7.7.7.2" class="ltx_td ltx_align_center">0.96</td>
<td id="A4.35.35.7.7.7.3" class="ltx_td ltx_align_center">0.76</td>
<td id="A4.35.35.7.7.7.4" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.35.35.7.7.7.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.35.35.7.7.7.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.35.35.7.7.7.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.35.35.7.7.7.8" class="ltx_td ltx_align_center">5.88</td>
</tr>
<tr id="A4.36.36.8.8.8" class="ltx_tr">
<td id="A4.36.36.8.8.8.1" class="ltx_td ltx_align_left">FT<math id="A4.36.36.8.8.8.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.36.36.8.8.8.1.m1.1a"><mo stretchy="false" id="A4.36.36.8.8.8.1.m1.1.1" xref="A4.36.36.8.8.8.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.36.36.8.8.8.1.m1.1b"><ci id="A4.36.36.8.8.8.1.m1.1.1.cmml" xref="A4.36.36.8.8.8.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.36.36.8.8.8.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.35)</td>
<td id="A4.36.36.8.8.8.2" class="ltx_td ltx_align_center">0.88</td>
<td id="A4.36.36.8.8.8.3" class="ltx_td ltx_align_center">0.68</td>
<td id="A4.36.36.8.8.8.4" class="ltx_td ltx_align_center">0.09</td>
<td id="A4.36.36.8.8.8.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.36.36.8.8.8.6" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.36.36.8.8.8.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.36.36.8.8.8.8" class="ltx_td ltx_align_center">6.35</td>
</tr>
<tr id="A4.37.37.9.9.9" class="ltx_tr">
<td id="A4.37.37.9.9.9.1" class="ltx_td ltx_align_left">FT<math id="A4.37.37.9.9.9.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.37.37.9.9.9.1.m1.1a"><mo stretchy="false" id="A4.37.37.9.9.9.1.m1.1.1" xref="A4.37.37.9.9.9.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.37.37.9.9.9.1.m1.1b"><ci id="A4.37.37.9.9.9.1.m1.1.1.cmml" xref="A4.37.37.9.9.9.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.37.37.9.9.9.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.45)</td>
<td id="A4.37.37.9.9.9.2" class="ltx_td ltx_align_center">0.65</td>
<td id="A4.37.37.9.9.9.3" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.37.37.9.9.9.4" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.37.37.9.9.9.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.37.37.9.9.9.6" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.37.37.9.9.9.7" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.37.37.9.9.9.8" class="ltx_td ltx_align_center">7.57</td>
</tr>
<tr id="A4.38.38.10.10.10" class="ltx_tr">
<td id="A4.38.38.10.10.10.1" class="ltx_td ltx_align_left">FT<math id="A4.38.38.10.10.10.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.38.38.10.10.10.1.m1.1a"><mo stretchy="false" id="A4.38.38.10.10.10.1.m1.1.1" xref="A4.38.38.10.10.10.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.38.38.10.10.10.1.m1.1b"><ci id="A4.38.38.10.10.10.1.m1.1.1.cmml" xref="A4.38.38.10.10.10.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.38.38.10.10.10.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.55)</td>
<td id="A4.38.38.10.10.10.2" class="ltx_td ltx_align_center">0.31</td>
<td id="A4.38.38.10.10.10.3" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.38.38.10.10.10.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.38.38.10.10.10.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.38.38.10.10.10.6" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.38.38.10.10.10.7" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.38.38.10.10.10.8" class="ltx_td ltx_align_center">12.48</td>
</tr>
<tr id="A4.39.39.11.11.11" class="ltx_tr">
<td id="A4.39.39.11.11.11.1" class="ltx_td ltx_align_left">FT<math id="A4.39.39.11.11.11.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.39.39.11.11.11.1.m1.1a"><mo stretchy="false" id="A4.39.39.11.11.11.1.m1.1.1" xref="A4.39.39.11.11.11.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.39.39.11.11.11.1.m1.1b"><ci id="A4.39.39.11.11.11.1.m1.1.1.cmml" xref="A4.39.39.11.11.11.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.39.39.11.11.11.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.65)</td>
<td id="A4.39.39.11.11.11.2" class="ltx_td ltx_align_center">0.08</td>
<td id="A4.39.39.11.11.11.3" class="ltx_td ltx_align_center">0.07</td>
<td id="A4.39.39.11.11.11.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.39.39.11.11.11.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.39.39.11.11.11.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.39.39.11.11.11.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.39.39.11.11.11.8" class="ltx_td ltx_align_center">46.37</td>
</tr>
<tr id="A4.40.40.12.12.12" class="ltx_tr">
<td id="A4.40.40.12.12.12.1" class="ltx_td ltx_align_left">FT<math id="A4.40.40.12.12.12.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.40.40.12.12.12.1.m1.1a"><mo stretchy="false" id="A4.40.40.12.12.12.1.m1.1.1" xref="A4.40.40.12.12.12.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.40.40.12.12.12.1.m1.1b"><ci id="A4.40.40.12.12.12.1.m1.1.1.cmml" xref="A4.40.40.12.12.12.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.40.40.12.12.12.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.75)</td>
<td id="A4.40.40.12.12.12.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.40.40.12.12.12.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.40.40.12.12.12.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.40.40.12.12.12.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.40.40.12.12.12.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.40.40.12.12.12.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.40.40.12.12.12.8" class="ltx_td ltx_align_center">344.62</td>
</tr>
<tr id="A4.41.41.13.13.13" class="ltx_tr">
<td id="A4.41.41.13.13.13.1" class="ltx_td ltx_align_left">FT<math id="A4.41.41.13.13.13.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.41.41.13.13.13.1.m1.1a"><mo stretchy="false" id="A4.41.41.13.13.13.1.m1.1.1" xref="A4.41.41.13.13.13.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.41.41.13.13.13.1.m1.1b"><ci id="A4.41.41.13.13.13.1.m1.1.1.cmml" xref="A4.41.41.13.13.13.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.41.41.13.13.13.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (2-Bit)</td>
<td id="A4.41.41.13.13.13.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.41.41.13.13.13.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.41.41.13.13.13.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.41.41.13.13.13.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.41.41.13.13.13.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.41.41.13.13.13.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.41.41.13.13.13.8" class="ltx_td ltx_align_center">1995.71</td>
</tr>
<tr id="A4.42.42.14.14.14" class="ltx_tr">
<td id="A4.42.42.14.14.14.1" class="ltx_td ltx_align_left">FT<math id="A4.42.42.14.14.14.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.42.42.14.14.14.1.m1.1a"><mo stretchy="false" id="A4.42.42.14.14.14.1.m1.1.1" xref="A4.42.42.14.14.14.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.42.42.14.14.14.1.m1.1b"><ci id="A4.42.42.14.14.14.1.m1.1.1.cmml" xref="A4.42.42.14.14.14.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.42.42.14.14.14.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (3-Bit)</td>
<td id="A4.42.42.14.14.14.2" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.42.42.14.14.14.3" class="ltx_td ltx_align_center">0.2</td>
<td id="A4.42.42.14.14.14.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.42.42.14.14.14.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.42.42.14.14.14.6" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.42.42.14.14.14.7" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.42.42.14.14.14.8" class="ltx_td ltx_align_center">9.03</td>
</tr>
<tr id="A4.43.43.15.15.15" class="ltx_tr">
<td id="A4.43.43.15.15.15.1" class="ltx_td ltx_align_left">FT<math id="A4.43.43.15.15.15.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.43.43.15.15.15.1.m1.1a"><mo stretchy="false" id="A4.43.43.15.15.15.1.m1.1.1" xref="A4.43.43.15.15.15.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.43.43.15.15.15.1.m1.1b"><ci id="A4.43.43.15.15.15.1.m1.1.1.cmml" xref="A4.43.43.15.15.15.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.43.43.15.15.15.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (4-Bit)</td>
<td id="A4.43.43.15.15.15.2" class="ltx_td ltx_align_center">0.79</td>
<td id="A4.43.43.15.15.15.3" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.43.43.15.15.15.4" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.43.43.15.15.15.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.43.43.15.15.15.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.43.43.15.15.15.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.43.43.15.15.15.8" class="ltx_td ltx_align_center">12.93</td>
</tr>
<tr id="A4.44.44.16.16.16" class="ltx_tr">
<td id="A4.44.44.16.16.16.1" class="ltx_td ltx_align_left">FT<math id="A4.44.44.16.16.16.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.44.44.16.16.16.1.m1.1a"><mo stretchy="false" id="A4.44.44.16.16.16.1.m1.1.1" xref="A4.44.44.16.16.16.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.44.44.16.16.16.1.m1.1b"><ci id="A4.44.44.16.16.16.1.m1.1.1.cmml" xref="A4.44.44.16.16.16.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.44.44.16.16.16.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (8-Bit)</td>
<td id="A4.44.44.16.16.16.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.44.44.16.16.16.3" class="ltx_td ltx_align_center">0.83</td>
<td id="A4.44.44.16.16.16.4" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.44.44.16.16.16.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.44.44.16.16.16.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.44.44.16.16.16.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.44.44.16.16.16.8" class="ltx_td ltx_align_center">5.57</td>
</tr>
<tr id="A4.45.45.17.17.17" class="ltx_tr">
<td id="A4.45.45.17.17.17.1" class="ltx_td ltx_align_left">FT<math id="A4.45.45.17.17.17.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.45.45.17.17.17.1.m1.1a"><mo stretchy="false" id="A4.45.45.17.17.17.1.m1.1.1" xref="A4.45.45.17.17.17.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.45.45.17.17.17.1.m1.1b"><ci id="A4.45.45.17.17.17.1.m1.1.1.cmml" xref="A4.45.45.17.17.17.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.45.45.17.17.17.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (2-Bit)</td>
<td id="A4.45.45.17.17.17.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.45.45.17.17.17.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.45.45.17.17.17.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.45.45.17.17.17.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.45.45.17.17.17.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.45.45.17.17.17.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.45.45.17.17.17.8" class="ltx_td ltx_align_center">1739216.25</td>
</tr>
<tr id="A4.46.46.18.18.18" class="ltx_tr">
<td id="A4.46.46.18.18.18.1" class="ltx_td ltx_align_left">FT<math id="A4.46.46.18.18.18.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.46.46.18.18.18.1.m1.1a"><mo stretchy="false" id="A4.46.46.18.18.18.1.m1.1.1" xref="A4.46.46.18.18.18.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.46.46.18.18.18.1.m1.1b"><ci id="A4.46.46.18.18.18.1.m1.1.1.cmml" xref="A4.46.46.18.18.18.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.46.46.18.18.18.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (3-Bit)</td>
<td id="A4.46.46.18.18.18.2" class="ltx_td ltx_align_center">0.7</td>
<td id="A4.46.46.18.18.18.3" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.46.46.18.18.18.4" class="ltx_td ltx_align_center">0.07</td>
<td id="A4.46.46.18.18.18.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.46.46.18.18.18.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.46.46.18.18.18.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.46.46.18.18.18.8" class="ltx_td ltx_align_center">7.56</td>
</tr>
<tr id="A4.47.47.19.19.19" class="ltx_tr">
<td id="A4.47.47.19.19.19.1" class="ltx_td ltx_align_left">FT<math id="A4.47.47.19.19.19.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.47.47.19.19.19.1.m1.1a"><mo stretchy="false" id="A4.47.47.19.19.19.1.m1.1.1" xref="A4.47.47.19.19.19.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.47.47.19.19.19.1.m1.1b"><ci id="A4.47.47.19.19.19.1.m1.1.1.cmml" xref="A4.47.47.19.19.19.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.47.47.19.19.19.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (4-Bit)</td>
<td id="A4.47.47.19.19.19.2" class="ltx_td ltx_align_center">0.95</td>
<td id="A4.47.47.19.19.19.3" class="ltx_td ltx_align_center">0.79</td>
<td id="A4.47.47.19.19.19.4" class="ltx_td ltx_align_center">0.1</td>
<td id="A4.47.47.19.19.19.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.47.47.19.19.19.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.47.47.19.19.19.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.47.47.19.19.19.8" class="ltx_td ltx_align_center">5.95</td>
</tr>
<tr id="A4.48.48.20.20.20" class="ltx_tr">
<td id="A4.48.48.20.20.20.1" class="ltx_td ltx_align_left">FT<math id="A4.48.48.20.20.20.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.48.48.20.20.20.1.m1.1a"><mo stretchy="false" id="A4.48.48.20.20.20.1.m1.1.1" xref="A4.48.48.20.20.20.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.48.48.20.20.20.1.m1.1b"><ci id="A4.48.48.20.20.20.1.m1.1.1.cmml" xref="A4.48.48.20.20.20.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.48.48.20.20.20.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (5-Bit)</td>
<td id="A4.48.48.20.20.20.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.48.48.20.20.20.3" class="ltx_td ltx_align_center">0.84</td>
<td id="A4.48.48.20.20.20.4" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.48.48.20.20.20.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.48.48.20.20.20.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.48.48.20.20.20.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.48.48.20.20.20.8" class="ltx_td ltx_align_center">5.66</td>
</tr>
<tr id="A4.49.49.21.21.21" class="ltx_tr">
<td id="A4.49.49.21.21.21.1" class="ltx_td ltx_align_left">FT<math id="A4.49.49.21.21.21.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.49.49.21.21.21.1.m1.1a"><mo stretchy="false" id="A4.49.49.21.21.21.1.m1.1.1" xref="A4.49.49.21.21.21.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.49.49.21.21.21.1.m1.1b"><ci id="A4.49.49.21.21.21.1.m1.1.1.cmml" xref="A4.49.49.21.21.21.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.49.49.21.21.21.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (6-Bit)</td>
<td id="A4.49.49.21.21.21.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.49.49.21.21.21.3" class="ltx_td ltx_align_center">0.84</td>
<td id="A4.49.49.21.21.21.4" class="ltx_td ltx_align_center">0.1</td>
<td id="A4.49.49.21.21.21.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.49.49.21.21.21.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.49.49.21.21.21.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.49.49.21.21.21.8" class="ltx_td ltx_align_center">5.59</td>
</tr>
<tr id="A4.50.50.22.22.22" class="ltx_tr">
<td id="A4.50.50.22.22.22.1" class="ltx_td ltx_align_left">FT<math id="A4.50.50.22.22.22.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.50.50.22.22.22.1.m1.1a"><mo stretchy="false" id="A4.50.50.22.22.22.1.m1.1.1" xref="A4.50.50.22.22.22.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.50.50.22.22.22.1.m1.1b"><ci id="A4.50.50.22.22.22.1.m1.1.1.cmml" xref="A4.50.50.22.22.22.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.50.50.22.22.22.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (8-Bit)</td>
<td id="A4.50.50.22.22.22.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.50.50.22.22.22.3" class="ltx_td ltx_align_center">0.83</td>
<td id="A4.50.50.22.22.22.4" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.50.50.22.22.22.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.50.50.22.22.22.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.50.50.22.22.22.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.50.50.22.22.22.8" class="ltx_td ltx_align_center">5.57</td>
</tr>
<tr id="A4.51.51.23.23.23" class="ltx_tr">
<td id="A4.51.51.23.23.23.1" class="ltx_td ltx_align_left">
<span id="A4.51.51.23.23.23.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

MEMIT<math id="A4.51.51.23.23.23.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.51.51.23.23.23.1.m1.1a"><mo stretchy="false" id="A4.51.51.23.23.23.1.m1.1.1" xref="A4.51.51.23.23.23.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.51.51.23.23.23.1.m1.1b"><ci id="A4.51.51.23.23.23.1.m1.1.1.cmml" xref="A4.51.51.23.23.23.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.51.51.23.23.23.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.25)</td>
<td id="A4.51.51.23.23.23.2" class="ltx_td ltx_align_center">0.87</td>
<td id="A4.51.51.23.23.23.3" class="ltx_td ltx_align_center">0.83</td>
<td id="A4.51.51.23.23.23.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.51.51.23.23.23.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.51.51.23.23.23.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.51.51.23.23.23.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.51.51.23.23.23.8" class="ltx_td ltx_align_center">5.89</td>
</tr>
<tr id="A4.52.52.24.24.24" class="ltx_tr">
<td id="A4.52.52.24.24.24.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.52.52.24.24.24.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.52.52.24.24.24.1.m1.1a"><mo stretchy="false" id="A4.52.52.24.24.24.1.m1.1.1" xref="A4.52.52.24.24.24.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.52.52.24.24.24.1.m1.1b"><ci id="A4.52.52.24.24.24.1.m1.1.1.cmml" xref="A4.52.52.24.24.24.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.52.52.24.24.24.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.35)</td>
<td id="A4.52.52.24.24.24.2" class="ltx_td ltx_align_center">0.81</td>
<td id="A4.52.52.24.24.24.3" class="ltx_td ltx_align_center">0.74</td>
<td id="A4.52.52.24.24.24.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.52.52.24.24.24.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.52.52.24.24.24.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.52.52.24.24.24.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.52.52.24.24.24.8" class="ltx_td ltx_align_center">6.4</td>
</tr>
<tr id="A4.53.53.25.25.25" class="ltx_tr">
<td id="A4.53.53.25.25.25.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.53.53.25.25.25.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.53.53.25.25.25.1.m1.1a"><mo stretchy="false" id="A4.53.53.25.25.25.1.m1.1.1" xref="A4.53.53.25.25.25.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.53.53.25.25.25.1.m1.1b"><ci id="A4.53.53.25.25.25.1.m1.1.1.cmml" xref="A4.53.53.25.25.25.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.53.53.25.25.25.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.45)</td>
<td id="A4.53.53.25.25.25.2" class="ltx_td ltx_align_center">0.62</td>
<td id="A4.53.53.25.25.25.3" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.53.53.25.25.25.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.53.53.25.25.25.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.53.53.25.25.25.6" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.53.53.25.25.25.7" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.53.53.25.25.25.8" class="ltx_td ltx_align_center">7.43</td>
</tr>
<tr id="A4.54.54.26.26.26" class="ltx_tr">
<td id="A4.54.54.26.26.26.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.54.54.26.26.26.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.54.54.26.26.26.1.m1.1a"><mo stretchy="false" id="A4.54.54.26.26.26.1.m1.1.1" xref="A4.54.54.26.26.26.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.54.54.26.26.26.1.m1.1b"><ci id="A4.54.54.26.26.26.1.m1.1.1.cmml" xref="A4.54.54.26.26.26.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.54.54.26.26.26.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.55)</td>
<td id="A4.54.54.26.26.26.2" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.54.54.26.26.26.3" class="ltx_td ltx_align_center">0.2</td>
<td id="A4.54.54.26.26.26.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.54.54.26.26.26.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.54.54.26.26.26.6" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.54.54.26.26.26.7" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.54.54.26.26.26.8" class="ltx_td ltx_align_center">10.37</td>
</tr>
<tr id="A4.55.55.27.27.27" class="ltx_tr">
<td id="A4.55.55.27.27.27.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.55.55.27.27.27.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.55.55.27.27.27.1.m1.1a"><mo stretchy="false" id="A4.55.55.27.27.27.1.m1.1.1" xref="A4.55.55.27.27.27.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.55.55.27.27.27.1.m1.1b"><ci id="A4.55.55.27.27.27.1.m1.1.1.cmml" xref="A4.55.55.27.27.27.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.55.55.27.27.27.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.65)</td>
<td id="A4.55.55.27.27.27.2" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.55.55.27.27.27.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.55.55.27.27.27.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.55.55.27.27.27.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.55.55.27.27.27.6" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.55.55.27.27.27.7" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.55.55.27.27.27.8" class="ltx_td ltx_align_center">21.02</td>
</tr>
<tr id="A4.56.56.28.28.28" class="ltx_tr">
<td id="A4.56.56.28.28.28.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.56.56.28.28.28.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.56.56.28.28.28.1.m1.1a"><mo stretchy="false" id="A4.56.56.28.28.28.1.m1.1.1" xref="A4.56.56.28.28.28.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.56.56.28.28.28.1.m1.1b"><ci id="A4.56.56.28.28.28.1.m1.1.1.cmml" xref="A4.56.56.28.28.28.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.56.56.28.28.28.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.75)</td>
<td id="A4.56.56.28.28.28.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.56.56.28.28.28.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.56.56.28.28.28.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.56.56.28.28.28.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.56.56.28.28.28.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.56.56.28.28.28.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.56.56.28.28.28.8" class="ltx_td ltx_align_center">71.62</td>
</tr>
<tr id="A4.57.57.29.29.29" class="ltx_tr">
<td id="A4.57.57.29.29.29.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.57.57.29.29.29.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.57.57.29.29.29.1.m1.1a"><mo stretchy="false" id="A4.57.57.29.29.29.1.m1.1.1" xref="A4.57.57.29.29.29.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.57.57.29.29.29.1.m1.1b"><ci id="A4.57.57.29.29.29.1.m1.1.1.cmml" xref="A4.57.57.29.29.29.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.57.57.29.29.29.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.25)</td>
<td id="A4.57.57.29.29.29.2" class="ltx_td ltx_align_center">0.7</td>
<td id="A4.57.57.29.29.29.3" class="ltx_td ltx_align_center">0.69</td>
<td id="A4.57.57.29.29.29.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.57.57.29.29.29.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.57.57.29.29.29.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.57.57.29.29.29.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.57.57.29.29.29.8" class="ltx_td ltx_align_center">5.87</td>
</tr>
<tr id="A4.58.58.30.30.30" class="ltx_tr">
<td id="A4.58.58.30.30.30.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.58.58.30.30.30.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.58.58.30.30.30.1.m1.1a"><mo stretchy="false" id="A4.58.58.30.30.30.1.m1.1.1" xref="A4.58.58.30.30.30.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.58.58.30.30.30.1.m1.1b"><ci id="A4.58.58.30.30.30.1.m1.1.1.cmml" xref="A4.58.58.30.30.30.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.58.58.30.30.30.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.35)</td>
<td id="A4.58.58.30.30.30.2" class="ltx_td ltx_align_center">0.85</td>
<td id="A4.58.58.30.30.30.3" class="ltx_td ltx_align_center">0.78</td>
<td id="A4.58.58.30.30.30.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.58.58.30.30.30.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.58.58.30.30.30.6" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.58.58.30.30.30.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.58.58.30.30.30.8" class="ltx_td ltx_align_center">6.34</td>
</tr>
<tr id="A4.59.59.31.31.31" class="ltx_tr">
<td id="A4.59.59.31.31.31.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.59.59.31.31.31.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.59.59.31.31.31.1.m1.1a"><mo stretchy="false" id="A4.59.59.31.31.31.1.m1.1.1" xref="A4.59.59.31.31.31.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.59.59.31.31.31.1.m1.1b"><ci id="A4.59.59.31.31.31.1.m1.1.1.cmml" xref="A4.59.59.31.31.31.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.59.59.31.31.31.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.45)</td>
<td id="A4.59.59.31.31.31.2" class="ltx_td ltx_align_center">0.75</td>
<td id="A4.59.59.31.31.31.3" class="ltx_td ltx_align_center">0.67</td>
<td id="A4.59.59.31.31.31.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.59.59.31.31.31.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.59.59.31.31.31.6" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.59.59.31.31.31.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.59.59.31.31.31.8" class="ltx_td ltx_align_center">7.56</td>
</tr>
<tr id="A4.60.60.32.32.32" class="ltx_tr">
<td id="A4.60.60.32.32.32.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.60.60.32.32.32.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.60.60.32.32.32.1.m1.1a"><mo stretchy="false" id="A4.60.60.32.32.32.1.m1.1.1" xref="A4.60.60.32.32.32.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.60.60.32.32.32.1.m1.1b"><ci id="A4.60.60.32.32.32.1.m1.1.1.cmml" xref="A4.60.60.32.32.32.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.60.60.32.32.32.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.55)</td>
<td id="A4.60.60.32.32.32.2" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.60.60.32.32.32.3" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.60.60.32.32.32.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.60.60.32.32.32.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.60.60.32.32.32.6" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.60.60.32.32.32.7" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.60.60.32.32.32.8" class="ltx_td ltx_align_center">12.33</td>
</tr>
<tr id="A4.61.61.33.33.33" class="ltx_tr">
<td id="A4.61.61.33.33.33.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.61.61.33.33.33.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.61.61.33.33.33.1.m1.1a"><mo stretchy="false" id="A4.61.61.33.33.33.1.m1.1.1" xref="A4.61.61.33.33.33.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.61.61.33.33.33.1.m1.1b"><ci id="A4.61.61.33.33.33.1.m1.1.1.cmml" xref="A4.61.61.33.33.33.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.61.61.33.33.33.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.65)</td>
<td id="A4.61.61.33.33.33.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.61.61.33.33.33.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.61.61.33.33.33.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.61.61.33.33.33.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.61.61.33.33.33.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.61.61.33.33.33.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.61.61.33.33.33.8" class="ltx_td ltx_align_center">46.98</td>
</tr>
<tr id="A4.62.62.34.34.34" class="ltx_tr">
<td id="A4.62.62.34.34.34.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.62.62.34.34.34.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.62.62.34.34.34.1.m1.1a"><mo stretchy="false" id="A4.62.62.34.34.34.1.m1.1.1" xref="A4.62.62.34.34.34.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.62.62.34.34.34.1.m1.1b"><ci id="A4.62.62.34.34.34.1.m1.1.1.cmml" xref="A4.62.62.34.34.34.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.62.62.34.34.34.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.75)</td>
<td id="A4.62.62.34.34.34.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.62.62.34.34.34.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.62.62.34.34.34.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.62.62.34.34.34.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.62.62.34.34.34.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.62.62.34.34.34.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.62.62.34.34.34.8" class="ltx_td ltx_align_center">290.27</td>
</tr>
<tr id="A4.63.63.35.35.35" class="ltx_tr">
<td id="A4.63.63.35.35.35.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.63.63.35.35.35.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.63.63.35.35.35.1.m1.1a"><mo stretchy="false" id="A4.63.63.35.35.35.1.m1.1.1" xref="A4.63.63.35.35.35.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.63.63.35.35.35.1.m1.1b"><ci id="A4.63.63.35.35.35.1.m1.1.1.cmml" xref="A4.63.63.35.35.35.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.63.63.35.35.35.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (2-Bit)</td>
<td id="A4.63.63.35.35.35.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.63.63.35.35.35.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.63.63.35.35.35.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.63.63.35.35.35.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.63.63.35.35.35.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.63.63.35.35.35.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.63.63.35.35.35.8" class="ltx_td ltx_align_center">1663.86</td>
</tr>
<tr id="A4.64.64.36.36.36" class="ltx_tr">
<td id="A4.64.64.36.36.36.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.64.64.36.36.36.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.64.64.36.36.36.1.m1.1a"><mo stretchy="false" id="A4.64.64.36.36.36.1.m1.1.1" xref="A4.64.64.36.36.36.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.64.64.36.36.36.1.m1.1b"><ci id="A4.64.64.36.36.36.1.m1.1.1.cmml" xref="A4.64.64.36.36.36.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.64.64.36.36.36.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (3-Bit)</td>
<td id="A4.64.64.36.36.36.2" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.64.64.36.36.36.3" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.64.64.36.36.36.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.64.64.36.36.36.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.64.64.36.36.36.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.64.64.36.36.36.7" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.64.64.36.36.36.8" class="ltx_td ltx_align_center">8.89</td>
</tr>
<tr id="A4.65.65.37.37.37" class="ltx_tr">
<td id="A4.65.65.37.37.37.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.65.65.37.37.37.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.65.65.37.37.37.1.m1.1a"><mo stretchy="false" id="A4.65.65.37.37.37.1.m1.1.1" xref="A4.65.65.37.37.37.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.65.65.37.37.37.1.m1.1b"><ci id="A4.65.65.37.37.37.1.m1.1.1.cmml" xref="A4.65.65.37.37.37.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.65.65.37.37.37.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (4-Bit)</td>
<td id="A4.65.65.37.37.37.2" class="ltx_td ltx_align_center">0.84</td>
<td id="A4.65.65.37.37.37.3" class="ltx_td ltx_align_center">0.8</td>
<td id="A4.65.65.37.37.37.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.65.65.37.37.37.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.65.65.37.37.37.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.65.65.37.37.37.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.65.65.37.37.37.8" class="ltx_td ltx_align_center">13.17</td>
</tr>
<tr id="A4.66.66.38.38.38" class="ltx_tr">
<td id="A4.66.66.38.38.38.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.66.66.38.38.38.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.66.66.38.38.38.1.m1.1a"><mo stretchy="false" id="A4.66.66.38.38.38.1.m1.1.1" xref="A4.66.66.38.38.38.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.66.66.38.38.38.1.m1.1b"><ci id="A4.66.66.38.38.38.1.m1.1.1.cmml" xref="A4.66.66.38.38.38.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.66.66.38.38.38.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (8-Bit)</td>
<td id="A4.66.66.38.38.38.2" class="ltx_td ltx_align_center">0.94</td>
<td id="A4.66.66.38.38.38.3" class="ltx_td ltx_align_center">0.9</td>
<td id="A4.66.66.38.38.38.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.66.66.38.38.38.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.66.66.38.38.38.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.66.66.38.38.38.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.66.66.38.38.38.8" class="ltx_td ltx_align_center">5.56</td>
</tr>
<tr id="A4.67.67.39.39.39" class="ltx_tr">
<td id="A4.67.67.39.39.39.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.67.67.39.39.39.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.67.67.39.39.39.1.m1.1a"><mo stretchy="false" id="A4.67.67.39.39.39.1.m1.1.1" xref="A4.67.67.39.39.39.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.67.67.39.39.39.1.m1.1b"><ci id="A4.67.67.39.39.39.1.m1.1.1.cmml" xref="A4.67.67.39.39.39.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.67.67.39.39.39.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (2-Bit)</td>
<td id="A4.67.67.39.39.39.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.67.67.39.39.39.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.67.67.39.39.39.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.67.67.39.39.39.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.67.67.39.39.39.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.67.67.39.39.39.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.67.67.39.39.39.8" class="ltx_td ltx_align_center">1738085.5</td>
</tr>
<tr id="A4.68.68.40.40.40" class="ltx_tr">
<td id="A4.68.68.40.40.40.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.68.68.40.40.40.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.68.68.40.40.40.1.m1.1a"><mo stretchy="false" id="A4.68.68.40.40.40.1.m1.1.1" xref="A4.68.68.40.40.40.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.68.68.40.40.40.1.m1.1b"><ci id="A4.68.68.40.40.40.1.m1.1.1.cmml" xref="A4.68.68.40.40.40.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.68.68.40.40.40.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (3-Bit)</td>
<td id="A4.68.68.40.40.40.2" class="ltx_td ltx_align_center">0.83</td>
<td id="A4.68.68.40.40.40.3" class="ltx_td ltx_align_center">0.79</td>
<td id="A4.68.68.40.40.40.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.68.68.40.40.40.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.68.68.40.40.40.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.68.68.40.40.40.7" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.68.68.40.40.40.8" class="ltx_td ltx_align_center">7.54</td>
</tr>
<tr id="A4.69.69.41.41.41" class="ltx_tr">
<td id="A4.69.69.41.41.41.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.69.69.41.41.41.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.69.69.41.41.41.1.m1.1a"><mo stretchy="false" id="A4.69.69.41.41.41.1.m1.1.1" xref="A4.69.69.41.41.41.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.69.69.41.41.41.1.m1.1b"><ci id="A4.69.69.41.41.41.1.m1.1.1.cmml" xref="A4.69.69.41.41.41.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.69.69.41.41.41.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (4-Bit)</td>
<td id="A4.69.69.41.41.41.2" class="ltx_td ltx_align_center">0.93</td>
<td id="A4.69.69.41.41.41.3" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.69.69.41.41.41.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.69.69.41.41.41.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.69.69.41.41.41.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.69.69.41.41.41.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.69.69.41.41.41.8" class="ltx_td ltx_align_center">5.94</td>
</tr>
<tr id="A4.70.70.42.42.42" class="ltx_tr">
<td id="A4.70.70.42.42.42.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.70.70.42.42.42.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.70.70.42.42.42.1.m1.1a"><mo stretchy="false" id="A4.70.70.42.42.42.1.m1.1.1" xref="A4.70.70.42.42.42.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.70.70.42.42.42.1.m1.1b"><ci id="A4.70.70.42.42.42.1.m1.1.1.cmml" xref="A4.70.70.42.42.42.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.70.70.42.42.42.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (5-Bit)</td>
<td id="A4.70.70.42.42.42.2" class="ltx_td ltx_align_center">0.92</td>
<td id="A4.70.70.42.42.42.3" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.70.70.42.42.42.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.70.70.42.42.42.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.70.70.42.42.42.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.70.70.42.42.42.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.70.70.42.42.42.8" class="ltx_td ltx_align_center">5.65</td>
</tr>
<tr id="A4.71.71.43.43.43" class="ltx_tr">
<td id="A4.71.71.43.43.43.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.71.71.43.43.43.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.71.71.43.43.43.1.m1.1a"><mo stretchy="false" id="A4.71.71.43.43.43.1.m1.1.1" xref="A4.71.71.43.43.43.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.71.71.43.43.43.1.m1.1b"><ci id="A4.71.71.43.43.43.1.m1.1.1.cmml" xref="A4.71.71.43.43.43.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.71.71.43.43.43.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (6-Bit)</td>
<td id="A4.71.71.43.43.43.2" class="ltx_td ltx_align_center">0.93</td>
<td id="A4.71.71.43.43.43.3" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.71.71.43.43.43.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.71.71.43.43.43.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.71.71.43.43.43.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.71.71.43.43.43.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.71.71.43.43.43.8" class="ltx_td ltx_align_center">5.59</td>
</tr>
<tr id="A4.72.72.44.44.44" class="ltx_tr">
<td id="A4.72.72.44.44.44.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.72.72.44.44.44.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.72.72.44.44.44.1.m1.1a"><mo stretchy="false" id="A4.72.72.44.44.44.1.m1.1.1" xref="A4.72.72.44.44.44.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.72.72.44.44.44.1.m1.1b"><ci id="A4.72.72.44.44.44.1.m1.1.1.cmml" xref="A4.72.72.44.44.44.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.72.72.44.44.44.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (8-Bit)</td>
<td id="A4.72.72.44.44.44.2" class="ltx_td ltx_align_center">0.94</td>
<td id="A4.72.72.44.44.44.3" class="ltx_td ltx_align_center">0.9</td>
<td id="A4.72.72.44.44.44.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.72.72.44.44.44.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.72.72.44.44.44.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.72.72.44.44.44.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.72.72.44.44.44.8" class="ltx_td ltx_align_center">5.56</td>
</tr>
<tr id="A4.73.73.45.45.45" class="ltx_tr">
<td id="A4.73.73.45.45.45.1" class="ltx_td ltx_align_left">
<span id="A4.73.73.45.45.45.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

LoRA<math id="A4.73.73.45.45.45.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.73.73.45.45.45.1.m1.1a"><mo stretchy="false" id="A4.73.73.45.45.45.1.m1.1.1" xref="A4.73.73.45.45.45.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.73.73.45.45.45.1.m1.1b"><ci id="A4.73.73.45.45.45.1.m1.1.1.cmml" xref="A4.73.73.45.45.45.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.73.73.45.45.45.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.25)</td>
<td id="A4.73.73.45.45.45.2" class="ltx_td ltx_align_center">0.86</td>
<td id="A4.73.73.45.45.45.3" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.73.73.45.45.45.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.73.73.45.45.45.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.73.73.45.45.45.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.73.73.45.45.45.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.73.73.45.45.45.8" class="ltx_td ltx_align_center">13.81</td>
</tr>
<tr id="A4.74.74.46.46.46" class="ltx_tr">
<td id="A4.74.74.46.46.46.1" class="ltx_td ltx_align_left">LoRA<math id="A4.74.74.46.46.46.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.74.74.46.46.46.1.m1.1a"><mo stretchy="false" id="A4.74.74.46.46.46.1.m1.1.1" xref="A4.74.74.46.46.46.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.74.74.46.46.46.1.m1.1b"><ci id="A4.74.74.46.46.46.1.m1.1.1.cmml" xref="A4.74.74.46.46.46.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.74.74.46.46.46.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.35)</td>
<td id="A4.74.74.46.46.46.2" class="ltx_td ltx_align_center">0.62</td>
<td id="A4.74.74.46.46.46.3" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.74.74.46.46.46.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.74.74.46.46.46.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.74.74.46.46.46.6" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.74.74.46.46.46.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.74.74.46.46.46.8" class="ltx_td ltx_align_center">10.6</td>
</tr>
<tr id="A4.75.75.47.47.47" class="ltx_tr">
<td id="A4.75.75.47.47.47.1" class="ltx_td ltx_align_left">LoRA<math id="A4.75.75.47.47.47.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.75.75.47.47.47.1.m1.1a"><mo stretchy="false" id="A4.75.75.47.47.47.1.m1.1.1" xref="A4.75.75.47.47.47.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.75.75.47.47.47.1.m1.1b"><ci id="A4.75.75.47.47.47.1.m1.1.1.cmml" xref="A4.75.75.47.47.47.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.75.75.47.47.47.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.45)</td>
<td id="A4.75.75.47.47.47.2" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.75.75.47.47.47.3" class="ltx_td ltx_align_center">0.22</td>
<td id="A4.75.75.47.47.47.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.75.75.47.47.47.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.75.75.47.47.47.6" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.75.75.47.47.47.7" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.75.75.47.47.47.8" class="ltx_td ltx_align_center">16.48</td>
</tr>
<tr id="A4.76.76.48.48.48" class="ltx_tr">
<td id="A4.76.76.48.48.48.1" class="ltx_td ltx_align_left">LoRA<math id="A4.76.76.48.48.48.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.76.76.48.48.48.1.m1.1a"><mo stretchy="false" id="A4.76.76.48.48.48.1.m1.1.1" xref="A4.76.76.48.48.48.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.76.76.48.48.48.1.m1.1b"><ci id="A4.76.76.48.48.48.1.m1.1.1.cmml" xref="A4.76.76.48.48.48.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.76.76.48.48.48.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.55)</td>
<td id="A4.76.76.48.48.48.2" class="ltx_td ltx_align_center">0.17</td>
<td id="A4.76.76.48.48.48.3" class="ltx_td ltx_align_center">0.14</td>
<td id="A4.76.76.48.48.48.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.76.76.48.48.48.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.76.76.48.48.48.6" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.76.76.48.48.48.7" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.76.76.48.48.48.8" class="ltx_td ltx_align_center">23.46</td>
</tr>
<tr id="A4.77.77.49.49.49" class="ltx_tr">
<td id="A4.77.77.49.49.49.1" class="ltx_td ltx_align_left">LoRA<math id="A4.77.77.49.49.49.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.77.77.49.49.49.1.m1.1a"><mo stretchy="false" id="A4.77.77.49.49.49.1.m1.1.1" xref="A4.77.77.49.49.49.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.77.77.49.49.49.1.m1.1b"><ci id="A4.77.77.49.49.49.1.m1.1.1.cmml" xref="A4.77.77.49.49.49.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.77.77.49.49.49.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.65)</td>
<td id="A4.77.77.49.49.49.2" class="ltx_td ltx_align_center">0.07</td>
<td id="A4.77.77.49.49.49.3" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.77.77.49.49.49.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.77.77.49.49.49.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.77.77.49.49.49.6" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.77.77.49.49.49.7" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.77.77.49.49.49.8" class="ltx_td ltx_align_center">53.89</td>
</tr>
<tr id="A4.78.78.50.50.50" class="ltx_tr">
<td id="A4.78.78.50.50.50.1" class="ltx_td ltx_align_left">LoRA<math id="A4.78.78.50.50.50.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.78.78.50.50.50.1.m1.1a"><mo stretchy="false" id="A4.78.78.50.50.50.1.m1.1.1" xref="A4.78.78.50.50.50.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.78.78.50.50.50.1.m1.1b"><ci id="A4.78.78.50.50.50.1.m1.1.1.cmml" xref="A4.78.78.50.50.50.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.78.78.50.50.50.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.75)</td>
<td id="A4.78.78.50.50.50.2" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.78.78.50.50.50.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.78.78.50.50.50.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.78.78.50.50.50.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.78.78.50.50.50.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.78.78.50.50.50.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.78.78.50.50.50.8" class="ltx_td ltx_align_center">186.65</td>
</tr>
<tr id="A4.79.79.51.51.51" class="ltx_tr">
<td id="A4.79.79.51.51.51.1" class="ltx_td ltx_align_left">LoRA<math id="A4.79.79.51.51.51.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.79.79.51.51.51.1.m1.1a"><mo stretchy="false" id="A4.79.79.51.51.51.1.m1.1.1" xref="A4.79.79.51.51.51.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.79.79.51.51.51.1.m1.1b"><ci id="A4.79.79.51.51.51.1.m1.1.1.cmml" xref="A4.79.79.51.51.51.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.79.79.51.51.51.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.25)</td>
<td id="A4.79.79.51.51.51.2" class="ltx_td ltx_align_center">0.87</td>
<td id="A4.79.79.51.51.51.3" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.79.79.51.51.51.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.79.79.51.51.51.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.79.79.51.51.51.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.79.79.51.51.51.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.79.79.51.51.51.8" class="ltx_td ltx_align_center">9.23</td>
</tr>
<tr id="A4.80.80.52.52.52" class="ltx_tr">
<td id="A4.80.80.52.52.52.1" class="ltx_td ltx_align_left">LoRA<math id="A4.80.80.52.52.52.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.80.80.52.52.52.1.m1.1a"><mo stretchy="false" id="A4.80.80.52.52.52.1.m1.1.1" xref="A4.80.80.52.52.52.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.80.80.52.52.52.1.m1.1b"><ci id="A4.80.80.52.52.52.1.m1.1.1.cmml" xref="A4.80.80.52.52.52.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.80.80.52.52.52.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.35)</td>
<td id="A4.80.80.52.52.52.2" class="ltx_td ltx_align_center">0.64</td>
<td id="A4.80.80.52.52.52.3" class="ltx_td ltx_align_center">0.4</td>
<td id="A4.80.80.52.52.52.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.80.80.52.52.52.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.80.80.52.52.52.6" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.80.80.52.52.52.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.80.80.52.52.52.8" class="ltx_td ltx_align_center">11.01</td>
</tr>
<tr id="A4.81.81.53.53.53" class="ltx_tr">
<td id="A4.81.81.53.53.53.1" class="ltx_td ltx_align_left">LoRA<math id="A4.81.81.53.53.53.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.81.81.53.53.53.1.m1.1a"><mo stretchy="false" id="A4.81.81.53.53.53.1.m1.1.1" xref="A4.81.81.53.53.53.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.81.81.53.53.53.1.m1.1b"><ci id="A4.81.81.53.53.53.1.m1.1.1.cmml" xref="A4.81.81.53.53.53.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.81.81.53.53.53.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.45)</td>
<td id="A4.81.81.53.53.53.2" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.81.81.53.53.53.3" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.81.81.53.53.53.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.81.81.53.53.53.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.81.81.53.53.53.6" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.81.81.53.53.53.7" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.81.81.53.53.53.8" class="ltx_td ltx_align_center">12.6</td>
</tr>
<tr id="A4.82.82.54.54.54" class="ltx_tr">
<td id="A4.82.82.54.54.54.1" class="ltx_td ltx_align_left">LoRA<math id="A4.82.82.54.54.54.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.82.82.54.54.54.1.m1.1a"><mo stretchy="false" id="A4.82.82.54.54.54.1.m1.1.1" xref="A4.82.82.54.54.54.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.82.82.54.54.54.1.m1.1b"><ci id="A4.82.82.54.54.54.1.m1.1.1.cmml" xref="A4.82.82.54.54.54.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.82.82.54.54.54.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.55)</td>
<td id="A4.82.82.54.54.54.2" class="ltx_td ltx_align_center">0.18</td>
<td id="A4.82.82.54.54.54.3" class="ltx_td ltx_align_center">0.14</td>
<td id="A4.82.82.54.54.54.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.82.82.54.54.54.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.82.82.54.54.54.6" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.82.82.54.54.54.7" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.82.82.54.54.54.8" class="ltx_td ltx_align_center">33.59</td>
</tr>
<tr id="A4.83.83.55.55.55" class="ltx_tr">
<td id="A4.83.83.55.55.55.1" class="ltx_td ltx_align_left">LoRA<math id="A4.83.83.55.55.55.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.83.83.55.55.55.1.m1.1a"><mo stretchy="false" id="A4.83.83.55.55.55.1.m1.1.1" xref="A4.83.83.55.55.55.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.83.83.55.55.55.1.m1.1b"><ci id="A4.83.83.55.55.55.1.m1.1.1.cmml" xref="A4.83.83.55.55.55.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.83.83.55.55.55.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.65)</td>
<td id="A4.83.83.55.55.55.2" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.83.83.55.55.55.3" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.83.83.55.55.55.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.83.83.55.55.55.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.83.83.55.55.55.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.83.83.55.55.55.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.83.83.55.55.55.8" class="ltx_td ltx_align_center">194.26</td>
</tr>
<tr id="A4.84.84.56.56.56" class="ltx_tr">
<td id="A4.84.84.56.56.56.1" class="ltx_td ltx_align_left">LoRA<math id="A4.84.84.56.56.56.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.84.84.56.56.56.1.m1.1a"><mo stretchy="false" id="A4.84.84.56.56.56.1.m1.1.1" xref="A4.84.84.56.56.56.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.84.84.56.56.56.1.m1.1b"><ci id="A4.84.84.56.56.56.1.m1.1.1.cmml" xref="A4.84.84.56.56.56.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.84.84.56.56.56.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.75)</td>
<td id="A4.84.84.56.56.56.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.84.84.56.56.56.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.84.84.56.56.56.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.84.84.56.56.56.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.84.84.56.56.56.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.84.84.56.56.56.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.84.84.56.56.56.8" class="ltx_td ltx_align_center">914.43</td>
</tr>
<tr id="A4.85.85.57.57.57" class="ltx_tr">
<td id="A4.85.85.57.57.57.1" class="ltx_td ltx_align_left">LoRA<math id="A4.85.85.57.57.57.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.85.85.57.57.57.1.m1.1a"><mo stretchy="false" id="A4.85.85.57.57.57.1.m1.1.1" xref="A4.85.85.57.57.57.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.85.85.57.57.57.1.m1.1b"><ci id="A4.85.85.57.57.57.1.m1.1.1.cmml" xref="A4.85.85.57.57.57.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.85.85.57.57.57.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (2-Bit)</td>
<td id="A4.85.85.57.57.57.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.85.85.57.57.57.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.85.85.57.57.57.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.85.85.57.57.57.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.85.85.57.57.57.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.85.85.57.57.57.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.85.85.57.57.57.8" class="ltx_td ltx_align_center">10465.63</td>
</tr>
<tr id="A4.86.86.58.58.58" class="ltx_tr">
<td id="A4.86.86.58.58.58.1" class="ltx_td ltx_align_left">LoRA<math id="A4.86.86.58.58.58.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.86.86.58.58.58.1.m1.1a"><mo stretchy="false" id="A4.86.86.58.58.58.1.m1.1.1" xref="A4.86.86.58.58.58.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.86.86.58.58.58.1.m1.1b"><ci id="A4.86.86.58.58.58.1.m1.1.1.cmml" xref="A4.86.86.58.58.58.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.86.86.58.58.58.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (3-Bit)</td>
<td id="A4.86.86.58.58.58.2" class="ltx_td ltx_align_center">0.08</td>
<td id="A4.86.86.58.58.58.3" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.86.86.58.58.58.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.86.86.58.58.58.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.86.86.58.58.58.6" class="ltx_td ltx_align_center">0.4</td>
<td id="A4.86.86.58.58.58.7" class="ltx_td ltx_align_center">0.42</td>
<td id="A4.86.86.58.58.58.8" class="ltx_td ltx_align_center">40.02</td>
</tr>
<tr id="A4.87.87.59.59.59" class="ltx_tr">
<td id="A4.87.87.59.59.59.1" class="ltx_td ltx_align_left">LoRA<math id="A4.87.87.59.59.59.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.87.87.59.59.59.1.m1.1a"><mo stretchy="false" id="A4.87.87.59.59.59.1.m1.1.1" xref="A4.87.87.59.59.59.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.87.87.59.59.59.1.m1.1b"><ci id="A4.87.87.59.59.59.1.m1.1.1.cmml" xref="A4.87.87.59.59.59.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.87.87.59.59.59.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (4-Bit)</td>
<td id="A4.87.87.59.59.59.2" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.87.87.59.59.59.3" class="ltx_td ltx_align_center">0.41</td>
<td id="A4.87.87.59.59.59.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.87.87.59.59.59.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.87.87.59.59.59.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.87.87.59.59.59.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.87.87.59.59.59.8" class="ltx_td ltx_align_center">63.45</td>
</tr>
<tr id="A4.88.88.60.60.60" class="ltx_tr">
<td id="A4.88.88.60.60.60.1" class="ltx_td ltx_align_left">LoRA<math id="A4.88.88.60.60.60.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.88.88.60.60.60.1.m1.1a"><mo stretchy="false" id="A4.88.88.60.60.60.1.m1.1.1" xref="A4.88.88.60.60.60.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.88.88.60.60.60.1.m1.1b"><ci id="A4.88.88.60.60.60.1.m1.1.1.cmml" xref="A4.88.88.60.60.60.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.88.88.60.60.60.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (8-Bit)</td>
<td id="A4.88.88.60.60.60.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.88.88.60.60.60.3" class="ltx_td ltx_align_center">0.71</td>
<td id="A4.88.88.60.60.60.4" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.88.88.60.60.60.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.88.88.60.60.60.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.88.88.60.60.60.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.88.88.60.60.60.8" class="ltx_td ltx_align_center">18.86</td>
</tr>
<tr id="A4.89.89.61.61.61" class="ltx_tr">
<td id="A4.89.89.61.61.61.1" class="ltx_td ltx_align_left">LoRA<math id="A4.89.89.61.61.61.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.89.89.61.61.61.1.m1.1a"><mo stretchy="false" id="A4.89.89.61.61.61.1.m1.1.1" xref="A4.89.89.61.61.61.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.89.89.61.61.61.1.m1.1b"><ci id="A4.89.89.61.61.61.1.m1.1.1.cmml" xref="A4.89.89.61.61.61.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.89.89.61.61.61.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (2-Bit)</td>
<td id="A4.89.89.61.61.61.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.89.89.61.61.61.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.89.89.61.61.61.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.89.89.61.61.61.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.89.89.61.61.61.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.89.89.61.61.61.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.89.89.61.61.61.8" class="ltx_td ltx_align_center">1764088.0</td>
</tr>
<tr id="A4.90.90.62.62.62" class="ltx_tr">
<td id="A4.90.90.62.62.62.1" class="ltx_td ltx_align_left">LoRA<math id="A4.90.90.62.62.62.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.90.90.62.62.62.1.m1.1a"><mo stretchy="false" id="A4.90.90.62.62.62.1.m1.1.1" xref="A4.90.90.62.62.62.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.90.90.62.62.62.1.m1.1b"><ci id="A4.90.90.62.62.62.1.m1.1.1.cmml" xref="A4.90.90.62.62.62.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.90.90.62.62.62.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (3-Bit)</td>
<td id="A4.90.90.62.62.62.2" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.90.90.62.62.62.3" class="ltx_td ltx_align_center">0.31</td>
<td id="A4.90.90.62.62.62.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.90.90.62.62.62.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.90.90.62.62.62.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.90.90.62.62.62.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.90.90.62.62.62.8" class="ltx_td ltx_align_center">24.29</td>
</tr>
<tr id="A4.91.91.63.63.63" class="ltx_tr">
<td id="A4.91.91.63.63.63.1" class="ltx_td ltx_align_left">LoRA<math id="A4.91.91.63.63.63.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.91.91.63.63.63.1.m1.1a"><mo stretchy="false" id="A4.91.91.63.63.63.1.m1.1.1" xref="A4.91.91.63.63.63.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.91.91.63.63.63.1.m1.1b"><ci id="A4.91.91.63.63.63.1.m1.1.1.cmml" xref="A4.91.91.63.63.63.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.91.91.63.63.63.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (4-Bit)</td>
<td id="A4.91.91.63.63.63.2" class="ltx_td ltx_align_center">0.91</td>
<td id="A4.91.91.63.63.63.3" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.91.91.63.63.63.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.91.91.63.63.63.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.91.91.63.63.63.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.91.91.63.63.63.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.91.91.63.63.63.8" class="ltx_td ltx_align_center">16.37</td>
</tr>
<tr id="A4.92.92.64.64.64" class="ltx_tr">
<td id="A4.92.92.64.64.64.1" class="ltx_td ltx_align_left">LoRA<math id="A4.92.92.64.64.64.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.92.92.64.64.64.1.m1.1a"><mo stretchy="false" id="A4.92.92.64.64.64.1.m1.1.1" xref="A4.92.92.64.64.64.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.92.92.64.64.64.1.m1.1b"><ci id="A4.92.92.64.64.64.1.m1.1.1.cmml" xref="A4.92.92.64.64.64.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.92.92.64.64.64.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (5-Bit)</td>
<td id="A4.92.92.64.64.64.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.92.92.64.64.64.3" class="ltx_td ltx_align_center">0.7</td>
<td id="A4.92.92.64.64.64.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.92.92.64.64.64.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.92.92.64.64.64.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.92.92.64.64.64.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.92.92.64.64.64.8" class="ltx_td ltx_align_center">16.31</td>
</tr>
<tr id="A4.93.93.65.65.65" class="ltx_tr">
<td id="A4.93.93.65.65.65.1" class="ltx_td ltx_align_left">LoRA<math id="A4.93.93.65.65.65.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.93.93.65.65.65.1.m1.1a"><mo stretchy="false" id="A4.93.93.65.65.65.1.m1.1.1" xref="A4.93.93.65.65.65.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.93.93.65.65.65.1.m1.1b"><ci id="A4.93.93.65.65.65.1.m1.1.1.cmml" xref="A4.93.93.65.65.65.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.93.93.65.65.65.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (6-Bit)</td>
<td id="A4.93.93.65.65.65.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.93.93.65.65.65.3" class="ltx_td ltx_align_center">0.71</td>
<td id="A4.93.93.65.65.65.4" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.93.93.65.65.65.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.93.93.65.65.65.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.93.93.65.65.65.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.93.93.65.65.65.8" class="ltx_td ltx_align_center">15.34</td>
</tr>
<tr id="A4.94.94.66.66.66" class="ltx_tr">
<td id="A4.94.94.66.66.66.1" class="ltx_td ltx_align_left ltx_border_bb">LoRA<math id="A4.94.94.66.66.66.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.94.94.66.66.66.1.m1.1a"><mo stretchy="false" id="A4.94.94.66.66.66.1.m1.1.1" xref="A4.94.94.66.66.66.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.94.94.66.66.66.1.m1.1b"><ci id="A4.94.94.66.66.66.1.m1.1.1.cmml" xref="A4.94.94.66.66.66.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.94.94.66.66.66.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (8-Bit)</td>
<td id="A4.94.94.66.66.66.2" class="ltx_td ltx_align_center ltx_border_bb">1.0</td>
<td id="A4.94.94.66.66.66.3" class="ltx_td ltx_align_center ltx_border_bb">0.72</td>
<td id="A4.94.94.66.66.66.4" class="ltx_td ltx_align_center ltx_border_bb">0.06</td>
<td id="A4.94.94.66.66.66.5" class="ltx_td ltx_align_center ltx_border_bb">8.25</td>
<td id="A4.94.94.66.66.66.6" class="ltx_td ltx_align_center ltx_border_bb">0.57</td>
<td id="A4.94.94.66.66.66.7" class="ltx_td ltx_align_center ltx_border_bb">0.61</td>
<td id="A4.94.94.66.66.66.8" class="ltx_td ltx_align_center ltx_border_bb">15.25</td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="A4.322.322.295.2.1" class="ltx_text" style="font-size:90%;">Table 9</span>: </span><span id="A4.96.96.68.1" class="ltx_text" style="font-size:90%;">Detailed results for editing <math id="A4.96.96.68.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.96.96.68.1.m1.1b"><mo stretchy="false" id="A4.96.96.68.1.m1.1.1" xref="A4.96.96.68.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.96.96.68.1.m1.1c"><ci id="A4.96.96.68.1.m1.1.1.cmml" xref="A4.96.96.68.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.96.96.68.1.m1.1d">\rightarrow</annotation></semantics></math> compression.</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="A4.322.322.294" class="ltx_table ltx_figure_panel">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="A4.162.162.134.66" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:933.6pt;vertical-align:-1.6pt;"><span class="ltx_transformed_inner" style="transform:translate(-79.6pt,171.2pt) scale(0.731366876083771,0.731366876083771) ;">
<table id="A4.162.162.134.66.66" class="ltx_tabular ltx_align_middle">
<tr id="A4.162.162.134.66.66.67" class="ltx_tr">
<td id="A4.162.162.134.66.66.67.1" class="ltx_td ltx_border_tt"></td>
<td id="A4.162.162.134.66.66.67.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="3">Editing</td>
<td id="A4.162.162.134.66.66.67.3" class="ltx_td ltx_align_center ltx_border_tt">Compression</td>
<td id="A4.162.162.134.66.66.67.4" class="ltx_td ltx_align_center ltx_border_tt">Unlearning</td>
<td id="A4.162.162.134.66.66.67.5" class="ltx_td ltx_align_center ltx_border_tt" colspan="2">Utility</td>
</tr>
<tr id="A4.162.162.134.66.66.68" class="ltx_tr">
<td id="A4.162.162.134.66.66.68.1" class="ltx_td"></td>
<td id="A4.162.162.134.66.66.68.2" class="ltx_td ltx_align_center ltx_border_t">Edit Success</td>
<td id="A4.162.162.134.66.66.68.3" class="ltx_td ltx_align_center ltx_border_t">Generalization</td>
<td id="A4.162.162.134.66.66.68.4" class="ltx_td ltx_align_center ltx_border_t">Locality</td>
<td id="A4.162.162.134.66.66.68.5" class="ltx_td ltx_align_center ltx_border_t">Avg. Bits</td>
<td id="A4.162.162.134.66.66.68.6" class="ltx_td ltx_align_center ltx_border_t">Avg. WMDP</td>
<td id="A4.162.162.134.66.66.68.7" class="ltx_td ltx_align_center ltx_border_t">MMLU</td>
<td id="A4.162.162.134.66.66.68.8" class="ltx_td ltx_align_center ltx_border_t">WikiText PPL</td>
</tr>
<tr id="A4.97.97.69.1.1.1" class="ltx_tr">
<td id="A4.97.97.69.1.1.1.1" class="ltx_td ltx_align_left ltx_border_t">SparseGPT (0.25) <math id="A4.97.97.69.1.1.1.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.97.97.69.1.1.1.1.m1.1a"><mo stretchy="false" id="A4.97.97.69.1.1.1.1.m1.1.1" xref="A4.97.97.69.1.1.1.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.97.97.69.1.1.1.1.m1.1b"><ci id="A4.97.97.69.1.1.1.1.m1.1.1.cmml" xref="A4.97.97.69.1.1.1.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.97.97.69.1.1.1.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.97.97.69.1.1.1.2" class="ltx_td ltx_align_center ltx_border_t">0.99</td>
<td id="A4.97.97.69.1.1.1.3" class="ltx_td ltx_align_center ltx_border_t">0.77</td>
<td id="A4.97.97.69.1.1.1.4" class="ltx_td ltx_align_center ltx_border_t">0.08</td>
<td id="A4.97.97.69.1.1.1.5" class="ltx_td ltx_align_center ltx_border_t">12.25</td>
<td id="A4.97.97.69.1.1.1.6" class="ltx_td ltx_align_center ltx_border_t">0.55</td>
<td id="A4.97.97.69.1.1.1.7" class="ltx_td ltx_align_center ltx_border_t">0.6</td>
<td id="A4.97.97.69.1.1.1.8" class="ltx_td ltx_align_center ltx_border_t">5.91</td>
</tr>
<tr id="A4.98.98.70.2.2.2" class="ltx_tr">
<td id="A4.98.98.70.2.2.2.1" class="ltx_td ltx_align_left">SparseGPT (0.35) <math id="A4.98.98.70.2.2.2.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.98.98.70.2.2.2.1.m1.1a"><mo stretchy="false" id="A4.98.98.70.2.2.2.1.m1.1.1" xref="A4.98.98.70.2.2.2.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.98.98.70.2.2.2.1.m1.1b"><ci id="A4.98.98.70.2.2.2.1.m1.1.1.cmml" xref="A4.98.98.70.2.2.2.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.98.98.70.2.2.2.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.98.98.70.2.2.2.2" class="ltx_td ltx_align_center">0.98</td>
<td id="A4.98.98.70.2.2.2.3" class="ltx_td ltx_align_center">0.75</td>
<td id="A4.98.98.70.2.2.2.4" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.98.98.70.2.2.2.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.98.98.70.2.2.2.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.98.98.70.2.2.2.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.98.98.70.2.2.2.8" class="ltx_td ltx_align_center">6.37</td>
</tr>
<tr id="A4.99.99.71.3.3.3" class="ltx_tr">
<td id="A4.99.99.71.3.3.3.1" class="ltx_td ltx_align_left">SparseGPT (0.45) <math id="A4.99.99.71.3.3.3.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.99.99.71.3.3.3.1.m1.1a"><mo stretchy="false" id="A4.99.99.71.3.3.3.1.m1.1.1" xref="A4.99.99.71.3.3.3.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.99.99.71.3.3.3.1.m1.1b"><ci id="A4.99.99.71.3.3.3.1.m1.1.1.cmml" xref="A4.99.99.71.3.3.3.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.99.99.71.3.3.3.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.99.99.71.3.3.3.2" class="ltx_td ltx_align_center">0.96</td>
<td id="A4.99.99.71.3.3.3.3" class="ltx_td ltx_align_center">0.71</td>
<td id="A4.99.99.71.3.3.3.4" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.99.99.71.3.3.3.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.99.99.71.3.3.3.6" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.99.99.71.3.3.3.7" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.99.99.71.3.3.3.8" class="ltx_td ltx_align_center">7.49</td>
</tr>
<tr id="A4.100.100.72.4.4.4" class="ltx_tr">
<td id="A4.100.100.72.4.4.4.1" class="ltx_td ltx_align_left">SparseGPT (0.55) <math id="A4.100.100.72.4.4.4.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.100.100.72.4.4.4.1.m1.1a"><mo stretchy="false" id="A4.100.100.72.4.4.4.1.m1.1.1" xref="A4.100.100.72.4.4.4.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.100.100.72.4.4.4.1.m1.1b"><ci id="A4.100.100.72.4.4.4.1.m1.1.1.cmml" xref="A4.100.100.72.4.4.4.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.100.100.72.4.4.4.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.100.100.72.4.4.4.2" class="ltx_td ltx_align_center">0.95</td>
<td id="A4.100.100.72.4.4.4.3" class="ltx_td ltx_align_center">0.66</td>
<td id="A4.100.100.72.4.4.4.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.100.100.72.4.4.4.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.100.100.72.4.4.4.6" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.100.100.72.4.4.4.7" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.100.100.72.4.4.4.8" class="ltx_td ltx_align_center">10.19</td>
</tr>
<tr id="A4.101.101.73.5.5.5" class="ltx_tr">
<td id="A4.101.101.73.5.5.5.1" class="ltx_td ltx_align_left">SparseGPT (0.65) <math id="A4.101.101.73.5.5.5.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.101.101.73.5.5.5.1.m1.1a"><mo stretchy="false" id="A4.101.101.73.5.5.5.1.m1.1.1" xref="A4.101.101.73.5.5.5.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.101.101.73.5.5.5.1.m1.1b"><ci id="A4.101.101.73.5.5.5.1.m1.1.1.cmml" xref="A4.101.101.73.5.5.5.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.101.101.73.5.5.5.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.101.101.73.5.5.5.2" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.101.101.73.5.5.5.3" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.101.101.73.5.5.5.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.101.101.73.5.5.5.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.101.101.73.5.5.5.6" class="ltx_td ltx_align_center">0.33</td>
<td id="A4.101.101.73.5.5.5.7" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.101.101.73.5.5.5.8" class="ltx_td ltx_align_center">17.67</td>
</tr>
<tr id="A4.102.102.74.6.6.6" class="ltx_tr">
<td id="A4.102.102.74.6.6.6.1" class="ltx_td ltx_align_left">SparseGPT (0.75) <math id="A4.102.102.74.6.6.6.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.102.102.74.6.6.6.1.m1.1a"><mo stretchy="false" id="A4.102.102.74.6.6.6.1.m1.1.1" xref="A4.102.102.74.6.6.6.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.102.102.74.6.6.6.1.m1.1b"><ci id="A4.102.102.74.6.6.6.1.m1.1.1.cmml" xref="A4.102.102.74.6.6.6.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.102.102.74.6.6.6.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.102.102.74.6.6.6.2" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.102.102.74.6.6.6.3" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.102.102.74.6.6.6.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.102.102.74.6.6.6.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.102.102.74.6.6.6.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.102.102.74.6.6.6.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.102.102.74.6.6.6.8" class="ltx_td ltx_align_center">98.18</td>
</tr>
<tr id="A4.103.103.75.7.7.7" class="ltx_tr">
<td id="A4.103.103.75.7.7.7.1" class="ltx_td ltx_align_left">SparseGPT (0.25) <math id="A4.103.103.75.7.7.7.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.103.103.75.7.7.7.1.m1.1a"><mo stretchy="false" id="A4.103.103.75.7.7.7.1.m1.1.1" xref="A4.103.103.75.7.7.7.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.103.103.75.7.7.7.1.m1.1b"><ci id="A4.103.103.75.7.7.7.1.m1.1.1.cmml" xref="A4.103.103.75.7.7.7.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.103.103.75.7.7.7.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.103.103.75.7.7.7.2" class="ltx_td ltx_align_center">0.91</td>
<td id="A4.103.103.75.7.7.7.3" class="ltx_td ltx_align_center">0.86</td>
<td id="A4.103.103.75.7.7.7.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.103.103.75.7.7.7.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.103.103.75.7.7.7.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.103.103.75.7.7.7.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.103.103.75.7.7.7.8" class="ltx_td ltx_align_center">5.89</td>
</tr>
<tr id="A4.104.104.76.8.8.8" class="ltx_tr">
<td id="A4.104.104.76.8.8.8.1" class="ltx_td ltx_align_left">SparseGPT (0.35) <math id="A4.104.104.76.8.8.8.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.104.104.76.8.8.8.1.m1.1a"><mo stretchy="false" id="A4.104.104.76.8.8.8.1.m1.1.1" xref="A4.104.104.76.8.8.8.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.104.104.76.8.8.8.1.m1.1b"><ci id="A4.104.104.76.8.8.8.1.m1.1.1.cmml" xref="A4.104.104.76.8.8.8.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.104.104.76.8.8.8.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.104.104.76.8.8.8.2" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.104.104.76.8.8.8.3" class="ltx_td ltx_align_center">0.8</td>
<td id="A4.104.104.76.8.8.8.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.104.104.76.8.8.8.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.104.104.76.8.8.8.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.104.104.76.8.8.8.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.104.104.76.8.8.8.8" class="ltx_td ltx_align_center">6.38</td>
</tr>
<tr id="A4.105.105.77.9.9.9" class="ltx_tr">
<td id="A4.105.105.77.9.9.9.1" class="ltx_td ltx_align_left">SparseGPT (0.45) <math id="A4.105.105.77.9.9.9.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.105.105.77.9.9.9.1.m1.1a"><mo stretchy="false" id="A4.105.105.77.9.9.9.1.m1.1.1" xref="A4.105.105.77.9.9.9.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.105.105.77.9.9.9.1.m1.1b"><ci id="A4.105.105.77.9.9.9.1.m1.1.1.cmml" xref="A4.105.105.77.9.9.9.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.105.105.77.9.9.9.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.105.105.77.9.9.9.2" class="ltx_td ltx_align_center">0.62</td>
<td id="A4.105.105.77.9.9.9.3" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.105.105.77.9.9.9.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.105.105.77.9.9.9.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.105.105.77.9.9.9.6" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.105.105.77.9.9.9.7" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.105.105.77.9.9.9.8" class="ltx_td ltx_align_center">7.49</td>
</tr>
<tr id="A4.106.106.78.10.10.10" class="ltx_tr">
<td id="A4.106.106.78.10.10.10.1" class="ltx_td ltx_align_left">SparseGPT (0.55) <math id="A4.106.106.78.10.10.10.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.106.106.78.10.10.10.1.m1.1a"><mo stretchy="false" id="A4.106.106.78.10.10.10.1.m1.1.1" xref="A4.106.106.78.10.10.10.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.106.106.78.10.10.10.1.m1.1b"><ci id="A4.106.106.78.10.10.10.1.m1.1.1.cmml" xref="A4.106.106.78.10.10.10.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.106.106.78.10.10.10.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.106.106.78.10.10.10.2" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.106.106.78.10.10.10.3" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.106.106.78.10.10.10.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.106.106.78.10.10.10.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.106.106.78.10.10.10.6" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.106.106.78.10.10.10.7" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.106.106.78.10.10.10.8" class="ltx_td ltx_align_center">10.62</td>
</tr>
<tr id="A4.107.107.79.11.11.11" class="ltx_tr">
<td id="A4.107.107.79.11.11.11.1" class="ltx_td ltx_align_left">SparseGPT (0.65) <math id="A4.107.107.79.11.11.11.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.107.107.79.11.11.11.1.m1.1a"><mo stretchy="false" id="A4.107.107.79.11.11.11.1.m1.1.1" xref="A4.107.107.79.11.11.11.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.107.107.79.11.11.11.1.m1.1b"><ci id="A4.107.107.79.11.11.11.1.m1.1.1.cmml" xref="A4.107.107.79.11.11.11.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.107.107.79.11.11.11.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.107.107.79.11.11.11.2" class="ltx_td ltx_align_center">0.18</td>
<td id="A4.107.107.79.11.11.11.3" class="ltx_td ltx_align_center">0.13</td>
<td id="A4.107.107.79.11.11.11.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.107.107.79.11.11.11.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.107.107.79.11.11.11.6" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.107.107.79.11.11.11.7" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.107.107.79.11.11.11.8" class="ltx_td ltx_align_center">22.83</td>
</tr>
<tr id="A4.108.108.80.12.12.12" class="ltx_tr">
<td id="A4.108.108.80.12.12.12.1" class="ltx_td ltx_align_left">SparseGPT (0.75) <math id="A4.108.108.80.12.12.12.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.108.108.80.12.12.12.1.m1.1a"><mo stretchy="false" id="A4.108.108.80.12.12.12.1.m1.1.1" xref="A4.108.108.80.12.12.12.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.108.108.80.12.12.12.1.m1.1b"><ci id="A4.108.108.80.12.12.12.1.m1.1.1.cmml" xref="A4.108.108.80.12.12.12.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.108.108.80.12.12.12.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.108.108.80.12.12.12.2" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.108.108.80.12.12.12.3" class="ltx_td ltx_align_center">0.08</td>
<td id="A4.108.108.80.12.12.12.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.108.108.80.12.12.12.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.108.108.80.12.12.12.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.108.108.80.12.12.12.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.108.108.80.12.12.12.8" class="ltx_td ltx_align_center">394.59</td>
</tr>
<tr id="A4.109.109.81.13.13.13" class="ltx_tr">
<td id="A4.109.109.81.13.13.13.1" class="ltx_td ltx_align_left">SparseGPT (0.25) <math id="A4.109.109.81.13.13.13.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.109.109.81.13.13.13.1.m1.1a"><mo stretchy="false" id="A4.109.109.81.13.13.13.1.m1.1.1" xref="A4.109.109.81.13.13.13.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.109.109.81.13.13.13.1.m1.1b"><ci id="A4.109.109.81.13.13.13.1.m1.1.1.cmml" xref="A4.109.109.81.13.13.13.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.109.109.81.13.13.13.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.109.109.81.13.13.13.2" class="ltx_td ltx_align_center">0.85</td>
<td id="A4.109.109.81.13.13.13.3" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.109.109.81.13.13.13.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.109.109.81.13.13.13.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.109.109.81.13.13.13.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.109.109.81.13.13.13.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.109.109.81.13.13.13.8" class="ltx_td ltx_align_center">10.69</td>
</tr>
<tr id="A4.110.110.82.14.14.14" class="ltx_tr">
<td id="A4.110.110.82.14.14.14.1" class="ltx_td ltx_align_left">SparseGPT (0.35) <math id="A4.110.110.82.14.14.14.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.110.110.82.14.14.14.1.m1.1a"><mo stretchy="false" id="A4.110.110.82.14.14.14.1.m1.1.1" xref="A4.110.110.82.14.14.14.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.110.110.82.14.14.14.1.m1.1b"><ci id="A4.110.110.82.14.14.14.1.m1.1.1.cmml" xref="A4.110.110.82.14.14.14.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.110.110.82.14.14.14.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.110.110.82.14.14.14.2" class="ltx_td ltx_align_center">0.76</td>
<td id="A4.110.110.82.14.14.14.3" class="ltx_td ltx_align_center">0.42</td>
<td id="A4.110.110.82.14.14.14.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.110.110.82.14.14.14.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.110.110.82.14.14.14.6" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.110.110.82.14.14.14.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.110.110.82.14.14.14.8" class="ltx_td ltx_align_center">9.54</td>
</tr>
<tr id="A4.111.111.83.15.15.15" class="ltx_tr">
<td id="A4.111.111.83.15.15.15.1" class="ltx_td ltx_align_left">SparseGPT (0.45) <math id="A4.111.111.83.15.15.15.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.111.111.83.15.15.15.1.m1.1a"><mo stretchy="false" id="A4.111.111.83.15.15.15.1.m1.1.1" xref="A4.111.111.83.15.15.15.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.111.111.83.15.15.15.1.m1.1b"><ci id="A4.111.111.83.15.15.15.1.m1.1.1.cmml" xref="A4.111.111.83.15.15.15.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.111.111.83.15.15.15.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.111.111.83.15.15.15.2" class="ltx_td ltx_align_center">0.64</td>
<td id="A4.111.111.83.15.15.15.3" class="ltx_td ltx_align_center">0.33</td>
<td id="A4.111.111.83.15.15.15.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.111.111.83.15.15.15.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.111.111.83.15.15.15.6" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.111.111.83.15.15.15.7" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.111.111.83.15.15.15.8" class="ltx_td ltx_align_center">10.52</td>
</tr>
<tr id="A4.112.112.84.16.16.16" class="ltx_tr">
<td id="A4.112.112.84.16.16.16.1" class="ltx_td ltx_align_left">SparseGPT (0.55) <math id="A4.112.112.84.16.16.16.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.112.112.84.16.16.16.1.m1.1a"><mo stretchy="false" id="A4.112.112.84.16.16.16.1.m1.1.1" xref="A4.112.112.84.16.16.16.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.112.112.84.16.16.16.1.m1.1b"><ci id="A4.112.112.84.16.16.16.1.m1.1.1.cmml" xref="A4.112.112.84.16.16.16.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.112.112.84.16.16.16.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.112.112.84.16.16.16.2" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.112.112.84.16.16.16.3" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.112.112.84.16.16.16.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.112.112.84.16.16.16.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.112.112.84.16.16.16.6" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.112.112.84.16.16.16.7" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.112.112.84.16.16.16.8" class="ltx_td ltx_align_center">11.5</td>
</tr>
<tr id="A4.113.113.85.17.17.17" class="ltx_tr">
<td id="A4.113.113.85.17.17.17.1" class="ltx_td ltx_align_left">SparseGPT (0.65) <math id="A4.113.113.85.17.17.17.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.113.113.85.17.17.17.1.m1.1a"><mo stretchy="false" id="A4.113.113.85.17.17.17.1.m1.1.1" xref="A4.113.113.85.17.17.17.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.113.113.85.17.17.17.1.m1.1b"><ci id="A4.113.113.85.17.17.17.1.m1.1.1.cmml" xref="A4.113.113.85.17.17.17.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.113.113.85.17.17.17.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.113.113.85.17.17.17.2" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.113.113.85.17.17.17.3" class="ltx_td ltx_align_center">0.17</td>
<td id="A4.113.113.85.17.17.17.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.113.113.85.17.17.17.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.113.113.85.17.17.17.6" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.113.113.85.17.17.17.7" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.113.113.85.17.17.17.8" class="ltx_td ltx_align_center">23.13</td>
</tr>
<tr id="A4.114.114.86.18.18.18" class="ltx_tr">
<td id="A4.114.114.86.18.18.18.1" class="ltx_td ltx_align_left">SparseGPT (0.75) <math id="A4.114.114.86.18.18.18.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.114.114.86.18.18.18.1.m1.1a"><mo stretchy="false" id="A4.114.114.86.18.18.18.1.m1.1.1" xref="A4.114.114.86.18.18.18.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.114.114.86.18.18.18.1.m1.1b"><ci id="A4.114.114.86.18.18.18.1.m1.1.1.cmml" xref="A4.114.114.86.18.18.18.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.114.114.86.18.18.18.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.114.114.86.18.18.18.2" class="ltx_td ltx_align_center">0.08</td>
<td id="A4.114.114.86.18.18.18.3" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.114.114.86.18.18.18.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.114.114.86.18.18.18.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.114.114.86.18.18.18.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.114.114.86.18.18.18.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.114.114.86.18.18.18.8" class="ltx_td ltx_align_center">89.47</td>
</tr>
<tr id="A4.115.115.87.19.19.19" class="ltx_tr">
<td id="A4.115.115.87.19.19.19.1" class="ltx_td ltx_align_left">
<span id="A4.115.115.87.19.19.19.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

Wanda (0.25) <math id="A4.115.115.87.19.19.19.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.115.115.87.19.19.19.1.m1.1a"><mo stretchy="false" id="A4.115.115.87.19.19.19.1.m1.1.1" xref="A4.115.115.87.19.19.19.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.115.115.87.19.19.19.1.m1.1b"><ci id="A4.115.115.87.19.19.19.1.m1.1.1.cmml" xref="A4.115.115.87.19.19.19.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.115.115.87.19.19.19.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.115.115.87.19.19.19.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.115.115.87.19.19.19.3" class="ltx_td ltx_align_center">0.8</td>
<td id="A4.115.115.87.19.19.19.4" class="ltx_td ltx_align_center">0.07</td>
<td id="A4.115.115.87.19.19.19.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.115.115.87.19.19.19.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.115.115.87.19.19.19.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.115.115.87.19.19.19.8" class="ltx_td ltx_align_center">5.86</td>
</tr>
<tr id="A4.116.116.88.20.20.20" class="ltx_tr">
<td id="A4.116.116.88.20.20.20.1" class="ltx_td ltx_align_left">Wanda (0.35) <math id="A4.116.116.88.20.20.20.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.116.116.88.20.20.20.1.m1.1a"><mo stretchy="false" id="A4.116.116.88.20.20.20.1.m1.1.1" xref="A4.116.116.88.20.20.20.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.116.116.88.20.20.20.1.m1.1b"><ci id="A4.116.116.88.20.20.20.1.m1.1.1.cmml" xref="A4.116.116.88.20.20.20.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.116.116.88.20.20.20.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.116.116.88.20.20.20.2" class="ltx_td ltx_align_center">0.97</td>
<td id="A4.116.116.88.20.20.20.3" class="ltx_td ltx_align_center">0.76</td>
<td id="A4.116.116.88.20.20.20.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.116.116.88.20.20.20.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.116.116.88.20.20.20.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.116.116.88.20.20.20.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.116.116.88.20.20.20.8" class="ltx_td ltx_align_center">6.32</td>
</tr>
<tr id="A4.117.117.89.21.21.21" class="ltx_tr">
<td id="A4.117.117.89.21.21.21.1" class="ltx_td ltx_align_left">Wanda (0.45) <math id="A4.117.117.89.21.21.21.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.117.117.89.21.21.21.1.m1.1a"><mo stretchy="false" id="A4.117.117.89.21.21.21.1.m1.1.1" xref="A4.117.117.89.21.21.21.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.117.117.89.21.21.21.1.m1.1b"><ci id="A4.117.117.89.21.21.21.1.m1.1.1.cmml" xref="A4.117.117.89.21.21.21.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.117.117.89.21.21.21.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.117.117.89.21.21.21.2" class="ltx_td ltx_align_center">0.94</td>
<td id="A4.117.117.89.21.21.21.3" class="ltx_td ltx_align_center">0.69</td>
<td id="A4.117.117.89.21.21.21.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.117.117.89.21.21.21.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.117.117.89.21.21.21.6" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.117.117.89.21.21.21.7" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.117.117.89.21.21.21.8" class="ltx_td ltx_align_center">7.52</td>
</tr>
<tr id="A4.118.118.90.22.22.22" class="ltx_tr">
<td id="A4.118.118.90.22.22.22.1" class="ltx_td ltx_align_left">Wanda (0.55) <math id="A4.118.118.90.22.22.22.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.118.118.90.22.22.22.1.m1.1a"><mo stretchy="false" id="A4.118.118.90.22.22.22.1.m1.1.1" xref="A4.118.118.90.22.22.22.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.118.118.90.22.22.22.1.m1.1b"><ci id="A4.118.118.90.22.22.22.1.m1.1.1.cmml" xref="A4.118.118.90.22.22.22.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.118.118.90.22.22.22.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.118.118.90.22.22.22.2" class="ltx_td ltx_align_center">0.8</td>
<td id="A4.118.118.90.22.22.22.3" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.118.118.90.22.22.22.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.118.118.90.22.22.22.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.118.118.90.22.22.22.6" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.118.118.90.22.22.22.7" class="ltx_td ltx_align_center">0.38</td>
<td id="A4.118.118.90.22.22.22.8" class="ltx_td ltx_align_center">11.98</td>
</tr>
<tr id="A4.119.119.91.23.23.23" class="ltx_tr">
<td id="A4.119.119.91.23.23.23.1" class="ltx_td ltx_align_left">Wanda (0.65) <math id="A4.119.119.91.23.23.23.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.119.119.91.23.23.23.1.m1.1a"><mo stretchy="false" id="A4.119.119.91.23.23.23.1.m1.1.1" xref="A4.119.119.91.23.23.23.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.119.119.91.23.23.23.1.m1.1b"><ci id="A4.119.119.91.23.23.23.1.m1.1.1.cmml" xref="A4.119.119.91.23.23.23.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.119.119.91.23.23.23.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.119.119.91.23.23.23.2" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.119.119.91.23.23.23.3" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.119.119.91.23.23.23.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.119.119.91.23.23.23.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.119.119.91.23.23.23.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.119.119.91.23.23.23.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.119.119.91.23.23.23.8" class="ltx_td ltx_align_center">41.41</td>
</tr>
<tr id="A4.120.120.92.24.24.24" class="ltx_tr">
<td id="A4.120.120.92.24.24.24.1" class="ltx_td ltx_align_left">Wanda (0.75) <math id="A4.120.120.92.24.24.24.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.120.120.92.24.24.24.1.m1.1a"><mo stretchy="false" id="A4.120.120.92.24.24.24.1.m1.1.1" xref="A4.120.120.92.24.24.24.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.120.120.92.24.24.24.1.m1.1b"><ci id="A4.120.120.92.24.24.24.1.m1.1.1.cmml" xref="A4.120.120.92.24.24.24.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.120.120.92.24.24.24.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.120.120.92.24.24.24.2" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.120.120.92.24.24.24.3" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.120.120.92.24.24.24.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.120.120.92.24.24.24.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.120.120.92.24.24.24.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.120.120.92.24.24.24.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.120.120.92.24.24.24.8" class="ltx_td ltx_align_center">249.29</td>
</tr>
<tr id="A4.121.121.93.25.25.25" class="ltx_tr">
<td id="A4.121.121.93.25.25.25.1" class="ltx_td ltx_align_left">Wanda (0.25) <math id="A4.121.121.93.25.25.25.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.121.121.93.25.25.25.1.m1.1a"><mo stretchy="false" id="A4.121.121.93.25.25.25.1.m1.1.1" xref="A4.121.121.93.25.25.25.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.121.121.93.25.25.25.1.m1.1b"><ci id="A4.121.121.93.25.25.25.1.m1.1.1.cmml" xref="A4.121.121.93.25.25.25.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.121.121.93.25.25.25.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.121.121.93.25.25.25.2" class="ltx_td ltx_align_center">0.95</td>
<td id="A4.121.121.93.25.25.25.3" class="ltx_td ltx_align_center">0.9</td>
<td id="A4.121.121.93.25.25.25.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.121.121.93.25.25.25.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.121.121.93.25.25.25.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.121.121.93.25.25.25.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.121.121.93.25.25.25.8" class="ltx_td ltx_align_center">5.87</td>
</tr>
<tr id="A4.122.122.94.26.26.26" class="ltx_tr">
<td id="A4.122.122.94.26.26.26.1" class="ltx_td ltx_align_left">Wanda (0.35) <math id="A4.122.122.94.26.26.26.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.122.122.94.26.26.26.1.m1.1a"><mo stretchy="false" id="A4.122.122.94.26.26.26.1.m1.1.1" xref="A4.122.122.94.26.26.26.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.122.122.94.26.26.26.1.m1.1b"><ci id="A4.122.122.94.26.26.26.1.m1.1.1.cmml" xref="A4.122.122.94.26.26.26.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.122.122.94.26.26.26.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.122.122.94.26.26.26.2" class="ltx_td ltx_align_center">0.86</td>
<td id="A4.122.122.94.26.26.26.3" class="ltx_td ltx_align_center">0.76</td>
<td id="A4.122.122.94.26.26.26.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.122.122.94.26.26.26.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.122.122.94.26.26.26.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.122.122.94.26.26.26.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.122.122.94.26.26.26.8" class="ltx_td ltx_align_center">6.34</td>
</tr>
<tr id="A4.123.123.95.27.27.27" class="ltx_tr">
<td id="A4.123.123.95.27.27.27.1" class="ltx_td ltx_align_left">Wanda (0.45) <math id="A4.123.123.95.27.27.27.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.123.123.95.27.27.27.1.m1.1a"><mo stretchy="false" id="A4.123.123.95.27.27.27.1.m1.1.1" xref="A4.123.123.95.27.27.27.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.123.123.95.27.27.27.1.m1.1b"><ci id="A4.123.123.95.27.27.27.1.m1.1.1.cmml" xref="A4.123.123.95.27.27.27.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.123.123.95.27.27.27.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.123.123.95.27.27.27.2" class="ltx_td ltx_align_center">0.72</td>
<td id="A4.123.123.95.27.27.27.3" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.123.123.95.27.27.27.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.123.123.95.27.27.27.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.123.123.95.27.27.27.6" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.123.123.95.27.27.27.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.123.123.95.27.27.27.8" class="ltx_td ltx_align_center">7.54</td>
</tr>
<tr id="A4.124.124.96.28.28.28" class="ltx_tr">
<td id="A4.124.124.96.28.28.28.1" class="ltx_td ltx_align_left">Wanda (0.55) <math id="A4.124.124.96.28.28.28.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.124.124.96.28.28.28.1.m1.1a"><mo stretchy="false" id="A4.124.124.96.28.28.28.1.m1.1.1" xref="A4.124.124.96.28.28.28.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.124.124.96.28.28.28.1.m1.1b"><ci id="A4.124.124.96.28.28.28.1.m1.1.1.cmml" xref="A4.124.124.96.28.28.28.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.124.124.96.28.28.28.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.124.124.96.28.28.28.2" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.124.124.96.28.28.28.3" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.124.124.96.28.28.28.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.124.124.96.28.28.28.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.124.124.96.28.28.28.6" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.124.124.96.28.28.28.7" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.124.124.96.28.28.28.8" class="ltx_td ltx_align_center">12.07</td>
</tr>
<tr id="A4.125.125.97.29.29.29" class="ltx_tr">
<td id="A4.125.125.97.29.29.29.1" class="ltx_td ltx_align_left">Wanda (0.65) <math id="A4.125.125.97.29.29.29.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.125.125.97.29.29.29.1.m1.1a"><mo stretchy="false" id="A4.125.125.97.29.29.29.1.m1.1.1" xref="A4.125.125.97.29.29.29.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.125.125.97.29.29.29.1.m1.1b"><ci id="A4.125.125.97.29.29.29.1.m1.1.1.cmml" xref="A4.125.125.97.29.29.29.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.125.125.97.29.29.29.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.125.125.97.29.29.29.2" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.125.125.97.29.29.29.3" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.125.125.97.29.29.29.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.125.125.97.29.29.29.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.125.125.97.29.29.29.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.125.125.97.29.29.29.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.125.125.97.29.29.29.8" class="ltx_td ltx_align_center">40.49</td>
</tr>
<tr id="A4.126.126.98.30.30.30" class="ltx_tr">
<td id="A4.126.126.98.30.30.30.1" class="ltx_td ltx_align_left">Wanda (0.75) <math id="A4.126.126.98.30.30.30.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.126.126.98.30.30.30.1.m1.1a"><mo stretchy="false" id="A4.126.126.98.30.30.30.1.m1.1.1" xref="A4.126.126.98.30.30.30.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.126.126.98.30.30.30.1.m1.1b"><ci id="A4.126.126.98.30.30.30.1.m1.1.1.cmml" xref="A4.126.126.98.30.30.30.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.126.126.98.30.30.30.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.126.126.98.30.30.30.2" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.126.126.98.30.30.30.3" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.126.126.98.30.30.30.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.126.126.98.30.30.30.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.126.126.98.30.30.30.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.126.126.98.30.30.30.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.126.126.98.30.30.30.8" class="ltx_td ltx_align_center">280.04</td>
</tr>
<tr id="A4.127.127.99.31.31.31" class="ltx_tr">
<td id="A4.127.127.99.31.31.31.1" class="ltx_td ltx_align_left">Wanda (0.25) <math id="A4.127.127.99.31.31.31.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.127.127.99.31.31.31.1.m1.1a"><mo stretchy="false" id="A4.127.127.99.31.31.31.1.m1.1.1" xref="A4.127.127.99.31.31.31.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.127.127.99.31.31.31.1.m1.1b"><ci id="A4.127.127.99.31.31.31.1.m1.1.1.cmml" xref="A4.127.127.99.31.31.31.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.127.127.99.31.31.31.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.127.127.99.31.31.31.2" class="ltx_td ltx_align_center">0.95</td>
<td id="A4.127.127.99.31.31.31.3" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.127.127.99.31.31.31.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.127.127.99.31.31.31.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.127.127.99.31.31.31.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.127.127.99.31.31.31.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.127.127.99.31.31.31.8" class="ltx_td ltx_align_center">10.65</td>
</tr>
<tr id="A4.128.128.100.32.32.32" class="ltx_tr">
<td id="A4.128.128.100.32.32.32.1" class="ltx_td ltx_align_left">Wanda (0.35) <math id="A4.128.128.100.32.32.32.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.128.128.100.32.32.32.1.m1.1a"><mo stretchy="false" id="A4.128.128.100.32.32.32.1.m1.1.1" xref="A4.128.128.100.32.32.32.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.128.128.100.32.32.32.1.m1.1b"><ci id="A4.128.128.100.32.32.32.1.m1.1.1.cmml" xref="A4.128.128.100.32.32.32.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.128.128.100.32.32.32.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.128.128.100.32.32.32.2" class="ltx_td ltx_align_center">0.86</td>
<td id="A4.128.128.100.32.32.32.3" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.128.128.100.32.32.32.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.128.128.100.32.32.32.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.128.128.100.32.32.32.6" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.128.128.100.32.32.32.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.128.128.100.32.32.32.8" class="ltx_td ltx_align_center">7.98</td>
</tr>
<tr id="A4.129.129.101.33.33.33" class="ltx_tr">
<td id="A4.129.129.101.33.33.33.1" class="ltx_td ltx_align_left">Wanda (0.45) <math id="A4.129.129.101.33.33.33.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.129.129.101.33.33.33.1.m1.1a"><mo stretchy="false" id="A4.129.129.101.33.33.33.1.m1.1.1" xref="A4.129.129.101.33.33.33.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.129.129.101.33.33.33.1.m1.1b"><ci id="A4.129.129.101.33.33.33.1.m1.1.1.cmml" xref="A4.129.129.101.33.33.33.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.129.129.101.33.33.33.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.129.129.101.33.33.33.2" class="ltx_td ltx_align_center">0.74</td>
<td id="A4.129.129.101.33.33.33.3" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.129.129.101.33.33.33.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.129.129.101.33.33.33.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.129.129.101.33.33.33.6" class="ltx_td ltx_align_center">0.47</td>
<td id="A4.129.129.101.33.33.33.7" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.129.129.101.33.33.33.8" class="ltx_td ltx_align_center">9.18</td>
</tr>
<tr id="A4.130.130.102.34.34.34" class="ltx_tr">
<td id="A4.130.130.102.34.34.34.1" class="ltx_td ltx_align_left">Wanda (0.55) <math id="A4.130.130.102.34.34.34.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.130.130.102.34.34.34.1.m1.1a"><mo stretchy="false" id="A4.130.130.102.34.34.34.1.m1.1.1" xref="A4.130.130.102.34.34.34.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.130.130.102.34.34.34.1.m1.1b"><ci id="A4.130.130.102.34.34.34.1.m1.1.1.cmml" xref="A4.130.130.102.34.34.34.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.130.130.102.34.34.34.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.130.130.102.34.34.34.2" class="ltx_td ltx_align_center">0.68</td>
<td id="A4.130.130.102.34.34.34.3" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.130.130.102.34.34.34.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.130.130.102.34.34.34.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.130.130.102.34.34.34.6" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.130.130.102.34.34.34.7" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.130.130.102.34.34.34.8" class="ltx_td ltx_align_center">15.01</td>
</tr>
<tr id="A4.131.131.103.35.35.35" class="ltx_tr">
<td id="A4.131.131.103.35.35.35.1" class="ltx_td ltx_align_left">Wanda (0.65) <math id="A4.131.131.103.35.35.35.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.131.131.103.35.35.35.1.m1.1a"><mo stretchy="false" id="A4.131.131.103.35.35.35.1.m1.1.1" xref="A4.131.131.103.35.35.35.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.131.131.103.35.35.35.1.m1.1b"><ci id="A4.131.131.103.35.35.35.1.m1.1.1.cmml" xref="A4.131.131.103.35.35.35.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.131.131.103.35.35.35.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.131.131.103.35.35.35.2" class="ltx_td ltx_align_center">0.34</td>
<td id="A4.131.131.103.35.35.35.3" class="ltx_td ltx_align_center">0.19</td>
<td id="A4.131.131.103.35.35.35.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.131.131.103.35.35.35.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.131.131.103.35.35.35.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.131.131.103.35.35.35.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.131.131.103.35.35.35.8" class="ltx_td ltx_align_center">57.63</td>
</tr>
<tr id="A4.132.132.104.36.36.36" class="ltx_tr">
<td id="A4.132.132.104.36.36.36.1" class="ltx_td ltx_align_left">Wanda (0.75) <math id="A4.132.132.104.36.36.36.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.132.132.104.36.36.36.1.m1.1a"><mo stretchy="false" id="A4.132.132.104.36.36.36.1.m1.1.1" xref="A4.132.132.104.36.36.36.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.132.132.104.36.36.36.1.m1.1b"><ci id="A4.132.132.104.36.36.36.1.m1.1.1.cmml" xref="A4.132.132.104.36.36.36.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.132.132.104.36.36.36.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.132.132.104.36.36.36.2" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.132.132.104.36.36.36.3" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.132.132.104.36.36.36.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.132.132.104.36.36.36.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.132.132.104.36.36.36.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.132.132.104.36.36.36.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.132.132.104.36.36.36.8" class="ltx_td ltx_align_center">318.06</td>
</tr>
<tr id="A4.133.133.105.37.37.37" class="ltx_tr">
<td id="A4.133.133.105.37.37.37.1" class="ltx_td ltx_align_left">
<span id="A4.133.133.105.37.37.37.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

GPTQ (2-Bit) <math id="A4.133.133.105.37.37.37.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.133.133.105.37.37.37.1.m1.1a"><mo stretchy="false" id="A4.133.133.105.37.37.37.1.m1.1.1" xref="A4.133.133.105.37.37.37.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.133.133.105.37.37.37.1.m1.1b"><ci id="A4.133.133.105.37.37.37.1.m1.1.1.cmml" xref="A4.133.133.105.37.37.37.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.133.133.105.37.37.37.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.133.133.105.37.37.37.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.133.133.105.37.37.37.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.133.133.105.37.37.37.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.133.133.105.37.37.37.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.133.133.105.37.37.37.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.133.133.105.37.37.37.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.133.133.105.37.37.37.8" class="ltx_td ltx_align_center">970816.25</td>
</tr>
<tr id="A4.134.134.106.38.38.38" class="ltx_tr">
<td id="A4.134.134.106.38.38.38.1" class="ltx_td ltx_align_left">GPTQ (3-Bit) <math id="A4.134.134.106.38.38.38.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.134.134.106.38.38.38.1.m1.1a"><mo stretchy="false" id="A4.134.134.106.38.38.38.1.m1.1.1" xref="A4.134.134.106.38.38.38.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.134.134.106.38.38.38.1.m1.1b"><ci id="A4.134.134.106.38.38.38.1.m1.1.1.cmml" xref="A4.134.134.106.38.38.38.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.134.134.106.38.38.38.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.134.134.106.38.38.38.2" class="ltx_td ltx_align_center">0.1</td>
<td id="A4.134.134.106.38.38.38.3" class="ltx_td ltx_align_center">0.08</td>
<td id="A4.134.134.106.38.38.38.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.134.134.106.38.38.38.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.134.134.106.38.38.38.6" class="ltx_td ltx_align_center">0.41</td>
<td id="A4.134.134.106.38.38.38.7" class="ltx_td ltx_align_center">0.42</td>
<td id="A4.134.134.106.38.38.38.8" class="ltx_td ltx_align_center">9.96</td>
</tr>
<tr id="A4.135.135.107.39.39.39" class="ltx_tr">
<td id="A4.135.135.107.39.39.39.1" class="ltx_td ltx_align_left">GPTQ (4-Bit) <math id="A4.135.135.107.39.39.39.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.135.135.107.39.39.39.1.m1.1a"><mo stretchy="false" id="A4.135.135.107.39.39.39.1.m1.1.1" xref="A4.135.135.107.39.39.39.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.135.135.107.39.39.39.1.m1.1b"><ci id="A4.135.135.107.39.39.39.1.m1.1.1.cmml" xref="A4.135.135.107.39.39.39.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.135.135.107.39.39.39.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.135.135.107.39.39.39.2" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.135.135.107.39.39.39.3" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.135.135.107.39.39.39.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.135.135.107.39.39.39.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.135.135.107.39.39.39.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.135.135.107.39.39.39.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.135.135.107.39.39.39.8" class="ltx_td ltx_align_center">6.4</td>
</tr>
<tr id="A4.136.136.108.40.40.40" class="ltx_tr">
<td id="A4.136.136.108.40.40.40.1" class="ltx_td ltx_align_left">GPTQ (8-Bit) <math id="A4.136.136.108.40.40.40.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.136.136.108.40.40.40.1.m1.1a"><mo stretchy="false" id="A4.136.136.108.40.40.40.1.m1.1.1" xref="A4.136.136.108.40.40.40.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.136.136.108.40.40.40.1.m1.1b"><ci id="A4.136.136.108.40.40.40.1.m1.1.1.cmml" xref="A4.136.136.108.40.40.40.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.136.136.108.40.40.40.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.136.136.108.40.40.40.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.136.136.108.40.40.40.3" class="ltx_td ltx_align_center">0.83</td>
<td id="A4.136.136.108.40.40.40.4" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.136.136.108.40.40.40.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.136.136.108.40.40.40.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.136.136.108.40.40.40.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.136.136.108.40.40.40.8" class="ltx_td ltx_align_center">5.57</td>
</tr>
<tr id="A4.137.137.109.41.41.41" class="ltx_tr">
<td id="A4.137.137.109.41.41.41.1" class="ltx_td ltx_align_left">GPTQ (2-Bit) <math id="A4.137.137.109.41.41.41.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.137.137.109.41.41.41.1.m1.1a"><mo stretchy="false" id="A4.137.137.109.41.41.41.1.m1.1.1" xref="A4.137.137.109.41.41.41.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.137.137.109.41.41.41.1.m1.1b"><ci id="A4.137.137.109.41.41.41.1.m1.1.1.cmml" xref="A4.137.137.109.41.41.41.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.137.137.109.41.41.41.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.137.137.109.41.41.41.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.137.137.109.41.41.41.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.137.137.109.41.41.41.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.137.137.109.41.41.41.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.137.137.109.41.41.41.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.137.137.109.41.41.41.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.137.137.109.41.41.41.8" class="ltx_td ltx_align_center">739248.62</td>
</tr>
<tr id="A4.138.138.110.42.42.42" class="ltx_tr">
<td id="A4.138.138.110.42.42.42.1" class="ltx_td ltx_align_left">GPTQ (3-Bit) <math id="A4.138.138.110.42.42.42.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.138.138.110.42.42.42.1.m1.1a"><mo stretchy="false" id="A4.138.138.110.42.42.42.1.m1.1.1" xref="A4.138.138.110.42.42.42.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.138.138.110.42.42.42.1.m1.1b"><ci id="A4.138.138.110.42.42.42.1.m1.1.1.cmml" xref="A4.138.138.110.42.42.42.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.138.138.110.42.42.42.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.138.138.110.42.42.42.2" class="ltx_td ltx_align_center">0.12</td>
<td id="A4.138.138.110.42.42.42.3" class="ltx_td ltx_align_center">0.1</td>
<td id="A4.138.138.110.42.42.42.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.138.138.110.42.42.42.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.138.138.110.42.42.42.6" class="ltx_td ltx_align_center">0.41</td>
<td id="A4.138.138.110.42.42.42.7" class="ltx_td ltx_align_center">0.42</td>
<td id="A4.138.138.110.42.42.42.8" class="ltx_td ltx_align_center">9.81</td>
</tr>
<tr id="A4.139.139.111.43.43.43" class="ltx_tr">
<td id="A4.139.139.111.43.43.43.1" class="ltx_td ltx_align_left">GPTQ (4-Bit) <math id="A4.139.139.111.43.43.43.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.139.139.111.43.43.43.1.m1.1a"><mo stretchy="false" id="A4.139.139.111.43.43.43.1.m1.1.1" xref="A4.139.139.111.43.43.43.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.139.139.111.43.43.43.1.m1.1b"><ci id="A4.139.139.111.43.43.43.1.m1.1.1.cmml" xref="A4.139.139.111.43.43.43.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.139.139.111.43.43.43.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.139.139.111.43.43.43.2" class="ltx_td ltx_align_center">0.81</td>
<td id="A4.139.139.111.43.43.43.3" class="ltx_td ltx_align_center">0.69</td>
<td id="A4.139.139.111.43.43.43.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.139.139.111.43.43.43.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.139.139.111.43.43.43.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.139.139.111.43.43.43.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.139.139.111.43.43.43.8" class="ltx_td ltx_align_center">6.38</td>
</tr>
<tr id="A4.140.140.112.44.44.44" class="ltx_tr">
<td id="A4.140.140.112.44.44.44.1" class="ltx_td ltx_align_left">GPTQ (8-Bit) <math id="A4.140.140.112.44.44.44.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.140.140.112.44.44.44.1.m1.1a"><mo stretchy="false" id="A4.140.140.112.44.44.44.1.m1.1.1" xref="A4.140.140.112.44.44.44.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.140.140.112.44.44.44.1.m1.1b"><ci id="A4.140.140.112.44.44.44.1.m1.1.1.cmml" xref="A4.140.140.112.44.44.44.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.140.140.112.44.44.44.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.140.140.112.44.44.44.2" class="ltx_td ltx_align_center">0.95</td>
<td id="A4.140.140.112.44.44.44.3" class="ltx_td ltx_align_center">0.91</td>
<td id="A4.140.140.112.44.44.44.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.140.140.112.44.44.44.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.140.140.112.44.44.44.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.140.140.112.44.44.44.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.140.140.112.44.44.44.8" class="ltx_td ltx_align_center">5.57</td>
</tr>
<tr id="A4.141.141.113.45.45.45" class="ltx_tr">
<td id="A4.141.141.113.45.45.45.1" class="ltx_td ltx_align_left">GPTQ (2-Bit) <math id="A4.141.141.113.45.45.45.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.141.141.113.45.45.45.1.m1.1a"><mo stretchy="false" id="A4.141.141.113.45.45.45.1.m1.1.1" xref="A4.141.141.113.45.45.45.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.141.141.113.45.45.45.1.m1.1b"><ci id="A4.141.141.113.45.45.45.1.m1.1.1.cmml" xref="A4.141.141.113.45.45.45.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.141.141.113.45.45.45.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.141.141.113.45.45.45.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.141.141.113.45.45.45.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.141.141.113.45.45.45.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.141.141.113.45.45.45.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.141.141.113.45.45.45.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.141.141.113.45.45.45.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.141.141.113.45.45.45.8" class="ltx_td ltx_align_center">695958.19</td>
</tr>
<tr id="A4.142.142.114.46.46.46" class="ltx_tr">
<td id="A4.142.142.114.46.46.46.1" class="ltx_td ltx_align_left">GPTQ (3-Bit) <math id="A4.142.142.114.46.46.46.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.142.142.114.46.46.46.1.m1.1a"><mo stretchy="false" id="A4.142.142.114.46.46.46.1.m1.1.1" xref="A4.142.142.114.46.46.46.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.142.142.114.46.46.46.1.m1.1b"><ci id="A4.142.142.114.46.46.46.1.m1.1.1.cmml" xref="A4.142.142.114.46.46.46.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.142.142.114.46.46.46.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.142.142.114.46.46.46.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.142.142.114.46.46.46.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.142.142.114.46.46.46.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.142.142.114.46.46.46.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.142.142.114.46.46.46.6" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.142.142.114.46.46.46.7" class="ltx_td ltx_align_center">0.38</td>
<td id="A4.142.142.114.46.46.46.8" class="ltx_td ltx_align_center">11.14</td>
</tr>
<tr id="A4.143.143.115.47.47.47" class="ltx_tr">
<td id="A4.143.143.115.47.47.47.1" class="ltx_td ltx_align_left">GPTQ (4-Bit) <math id="A4.143.143.115.47.47.47.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.143.143.115.47.47.47.1.m1.1a"><mo stretchy="false" id="A4.143.143.115.47.47.47.1.m1.1.1" xref="A4.143.143.115.47.47.47.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.143.143.115.47.47.47.1.m1.1b"><ci id="A4.143.143.115.47.47.47.1.m1.1.1.cmml" xref="A4.143.143.115.47.47.47.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.143.143.115.47.47.47.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.143.143.115.47.47.47.2" class="ltx_td ltx_align_center">0.21</td>
<td id="A4.143.143.115.47.47.47.3" class="ltx_td ltx_align_center">0.14</td>
<td id="A4.143.143.115.47.47.47.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.143.143.115.47.47.47.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.143.143.115.47.47.47.6" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.143.143.115.47.47.47.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.143.143.115.47.47.47.8" class="ltx_td ltx_align_center">9.54</td>
</tr>
<tr id="A4.144.144.116.48.48.48" class="ltx_tr">
<td id="A4.144.144.116.48.48.48.1" class="ltx_td ltx_align_left">GPTQ (8-Bit) <math id="A4.144.144.116.48.48.48.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.144.144.116.48.48.48.1.m1.1a"><mo stretchy="false" id="A4.144.144.116.48.48.48.1.m1.1.1" xref="A4.144.144.116.48.48.48.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.144.144.116.48.48.48.1.m1.1b"><ci id="A4.144.144.116.48.48.48.1.m1.1.1.cmml" xref="A4.144.144.116.48.48.48.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.144.144.116.48.48.48.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.144.144.116.48.48.48.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.144.144.116.48.48.48.3" class="ltx_td ltx_align_center">0.72</td>
<td id="A4.144.144.116.48.48.48.4" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.144.144.116.48.48.48.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.144.144.116.48.48.48.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.144.144.116.48.48.48.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.144.144.116.48.48.48.8" class="ltx_td ltx_align_center">19.99</td>
</tr>
<tr id="A4.145.145.117.49.49.49" class="ltx_tr">
<td id="A4.145.145.117.49.49.49.1" class="ltx_td ltx_align_left">
<span id="A4.145.145.117.49.49.49.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

AWQ (2-Bit) <math id="A4.145.145.117.49.49.49.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.145.145.117.49.49.49.1.m1.1a"><mo stretchy="false" id="A4.145.145.117.49.49.49.1.m1.1.1" xref="A4.145.145.117.49.49.49.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.145.145.117.49.49.49.1.m1.1b"><ci id="A4.145.145.117.49.49.49.1.m1.1.1.cmml" xref="A4.145.145.117.49.49.49.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.145.145.117.49.49.49.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.145.145.117.49.49.49.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.145.145.117.49.49.49.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.145.145.117.49.49.49.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.145.145.117.49.49.49.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.145.145.117.49.49.49.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.145.145.117.49.49.49.7" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.145.145.117.49.49.49.8" class="ltx_td ltx_align_center">33638.44</td>
</tr>
<tr id="A4.146.146.118.50.50.50" class="ltx_tr">
<td id="A4.146.146.118.50.50.50.1" class="ltx_td ltx_align_left">AWQ (3-Bit) <math id="A4.146.146.118.50.50.50.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.146.146.118.50.50.50.1.m1.1a"><mo stretchy="false" id="A4.146.146.118.50.50.50.1.m1.1.1" xref="A4.146.146.118.50.50.50.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.146.146.118.50.50.50.1.m1.1b"><ci id="A4.146.146.118.50.50.50.1.m1.1.1.cmml" xref="A4.146.146.118.50.50.50.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.146.146.118.50.50.50.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.146.146.118.50.50.50.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.146.146.118.50.50.50.3" class="ltx_td ltx_align_center">0.82</td>
<td id="A4.146.146.118.50.50.50.4" class="ltx_td ltx_align_center">0.08</td>
<td id="A4.146.146.118.50.50.50.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.146.146.118.50.50.50.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.146.146.118.50.50.50.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.146.146.118.50.50.50.8" class="ltx_td ltx_align_center">7.57</td>
</tr>
<tr id="A4.147.147.119.51.51.51" class="ltx_tr">
<td id="A4.147.147.119.51.51.51.1" class="ltx_td ltx_align_left">AWQ (4-Bit) <math id="A4.147.147.119.51.51.51.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.147.147.119.51.51.51.1.m1.1a"><mo stretchy="false" id="A4.147.147.119.51.51.51.1.m1.1.1" xref="A4.147.147.119.51.51.51.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.147.147.119.51.51.51.1.m1.1b"><ci id="A4.147.147.119.51.51.51.1.m1.1.1.cmml" xref="A4.147.147.119.51.51.51.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.147.147.119.51.51.51.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.147.147.119.51.51.51.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.147.147.119.51.51.51.3" class="ltx_td ltx_align_center">0.84</td>
<td id="A4.147.147.119.51.51.51.4" class="ltx_td ltx_align_center">0.15</td>
<td id="A4.147.147.119.51.51.51.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.147.147.119.51.51.51.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.147.147.119.51.51.51.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.147.147.119.51.51.51.8" class="ltx_td ltx_align_center">5.98</td>
</tr>
<tr id="A4.148.148.120.52.52.52" class="ltx_tr">
<td id="A4.148.148.120.52.52.52.1" class="ltx_td ltx_align_left">AWQ (5-Bit) <math id="A4.148.148.120.52.52.52.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.148.148.120.52.52.52.1.m1.1a"><mo stretchy="false" id="A4.148.148.120.52.52.52.1.m1.1.1" xref="A4.148.148.120.52.52.52.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.148.148.120.52.52.52.1.m1.1b"><ci id="A4.148.148.120.52.52.52.1.m1.1.1.cmml" xref="A4.148.148.120.52.52.52.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.148.148.120.52.52.52.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.148.148.120.52.52.52.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.148.148.120.52.52.52.3" class="ltx_td ltx_align_center">0.85</td>
<td id="A4.148.148.120.52.52.52.4" class="ltx_td ltx_align_center">0.12</td>
<td id="A4.148.148.120.52.52.52.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.148.148.120.52.52.52.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.148.148.120.52.52.52.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.148.148.120.52.52.52.8" class="ltx_td ltx_align_center">5.68</td>
</tr>
<tr id="A4.149.149.121.53.53.53" class="ltx_tr">
<td id="A4.149.149.121.53.53.53.1" class="ltx_td ltx_align_left">AWQ (6-Bit) <math id="A4.149.149.121.53.53.53.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.149.149.121.53.53.53.1.m1.1a"><mo stretchy="false" id="A4.149.149.121.53.53.53.1.m1.1.1" xref="A4.149.149.121.53.53.53.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.149.149.121.53.53.53.1.m1.1b"><ci id="A4.149.149.121.53.53.53.1.m1.1.1.cmml" xref="A4.149.149.121.53.53.53.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.149.149.121.53.53.53.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.149.149.121.53.53.53.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.149.149.121.53.53.53.3" class="ltx_td ltx_align_center">0.88</td>
<td id="A4.149.149.121.53.53.53.4" class="ltx_td ltx_align_center">0.13</td>
<td id="A4.149.149.121.53.53.53.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.149.149.121.53.53.53.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.149.149.121.53.53.53.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.149.149.121.53.53.53.8" class="ltx_td ltx_align_center">5.61</td>
</tr>
<tr id="A4.150.150.122.54.54.54" class="ltx_tr">
<td id="A4.150.150.122.54.54.54.1" class="ltx_td ltx_align_left">AWQ (8-Bit) <math id="A4.150.150.122.54.54.54.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.150.150.122.54.54.54.1.m1.1a"><mo stretchy="false" id="A4.150.150.122.54.54.54.1.m1.1.1" xref="A4.150.150.122.54.54.54.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.150.150.122.54.54.54.1.m1.1b"><ci id="A4.150.150.122.54.54.54.1.m1.1.1.cmml" xref="A4.150.150.122.54.54.54.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.150.150.122.54.54.54.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.150.150.122.54.54.54.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.150.150.122.54.54.54.3" class="ltx_td ltx_align_center">0.87</td>
<td id="A4.150.150.122.54.54.54.4" class="ltx_td ltx_align_center">0.13</td>
<td id="A4.150.150.122.54.54.54.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.150.150.122.54.54.54.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.150.150.122.54.54.54.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.150.150.122.54.54.54.8" class="ltx_td ltx_align_center">5.6</td>
</tr>
<tr id="A4.151.151.123.55.55.55" class="ltx_tr">
<td id="A4.151.151.123.55.55.55.1" class="ltx_td ltx_align_left">AWQ (2-Bit) <math id="A4.151.151.123.55.55.55.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.151.151.123.55.55.55.1.m1.1a"><mo stretchy="false" id="A4.151.151.123.55.55.55.1.m1.1.1" xref="A4.151.151.123.55.55.55.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.151.151.123.55.55.55.1.m1.1b"><ci id="A4.151.151.123.55.55.55.1.m1.1.1.cmml" xref="A4.151.151.123.55.55.55.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.151.151.123.55.55.55.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.151.151.123.55.55.55.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.151.151.123.55.55.55.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.151.151.123.55.55.55.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.151.151.123.55.55.55.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.151.151.123.55.55.55.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.151.151.123.55.55.55.7" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.151.151.123.55.55.55.8" class="ltx_td ltx_align_center">1735678.75</td>
</tr>
<tr id="A4.152.152.124.56.56.56" class="ltx_tr">
<td id="A4.152.152.124.56.56.56.1" class="ltx_td ltx_align_left">AWQ (3-Bit) <math id="A4.152.152.124.56.56.56.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.152.152.124.56.56.56.1.m1.1a"><mo stretchy="false" id="A4.152.152.124.56.56.56.1.m1.1.1" xref="A4.152.152.124.56.56.56.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.152.152.124.56.56.56.1.m1.1b"><ci id="A4.152.152.124.56.56.56.1.m1.1.1.cmml" xref="A4.152.152.124.56.56.56.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.152.152.124.56.56.56.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.152.152.124.56.56.56.2" class="ltx_td ltx_align_center">0.92</td>
<td id="A4.152.152.124.56.56.56.3" class="ltx_td ltx_align_center">0.87</td>
<td id="A4.152.152.124.56.56.56.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.152.152.124.56.56.56.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.152.152.124.56.56.56.6" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.152.152.124.56.56.56.7" class="ltx_td ltx_align_center">0.47</td>
<td id="A4.152.152.124.56.56.56.8" class="ltx_td ltx_align_center">7.57</td>
</tr>
<tr id="A4.153.153.125.57.57.57" class="ltx_tr">
<td id="A4.153.153.125.57.57.57.1" class="ltx_td ltx_align_left">AWQ (4-Bit) <math id="A4.153.153.125.57.57.57.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.153.153.125.57.57.57.1.m1.1a"><mo stretchy="false" id="A4.153.153.125.57.57.57.1.m1.1.1" xref="A4.153.153.125.57.57.57.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.153.153.125.57.57.57.1.m1.1b"><ci id="A4.153.153.125.57.57.57.1.m1.1.1.cmml" xref="A4.153.153.125.57.57.57.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.153.153.125.57.57.57.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.153.153.125.57.57.57.2" class="ltx_td ltx_align_center">0.92</td>
<td id="A4.153.153.125.57.57.57.3" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.153.153.125.57.57.57.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.153.153.125.57.57.57.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.153.153.125.57.57.57.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.153.153.125.57.57.57.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.153.153.125.57.57.57.8" class="ltx_td ltx_align_center">5.97</td>
</tr>
<tr id="A4.154.154.126.58.58.58" class="ltx_tr">
<td id="A4.154.154.126.58.58.58.1" class="ltx_td ltx_align_left">AWQ (5-Bit) <math id="A4.154.154.126.58.58.58.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.154.154.126.58.58.58.1.m1.1a"><mo stretchy="false" id="A4.154.154.126.58.58.58.1.m1.1.1" xref="A4.154.154.126.58.58.58.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.154.154.126.58.58.58.1.m1.1b"><ci id="A4.154.154.126.58.58.58.1.m1.1.1.cmml" xref="A4.154.154.126.58.58.58.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.154.154.126.58.58.58.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.154.154.126.58.58.58.2" class="ltx_td ltx_align_center">0.98</td>
<td id="A4.154.154.126.58.58.58.3" class="ltx_td ltx_align_center">0.96</td>
<td id="A4.154.154.126.58.58.58.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.154.154.126.58.58.58.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.154.154.126.58.58.58.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.154.154.126.58.58.58.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.154.154.126.58.58.58.8" class="ltx_td ltx_align_center">5.68</td>
</tr>
<tr id="A4.155.155.127.59.59.59" class="ltx_tr">
<td id="A4.155.155.127.59.59.59.1" class="ltx_td ltx_align_left">AWQ (6-Bit) <math id="A4.155.155.127.59.59.59.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.155.155.127.59.59.59.1.m1.1a"><mo stretchy="false" id="A4.155.155.127.59.59.59.1.m1.1.1" xref="A4.155.155.127.59.59.59.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.155.155.127.59.59.59.1.m1.1b"><ci id="A4.155.155.127.59.59.59.1.m1.1.1.cmml" xref="A4.155.155.127.59.59.59.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.155.155.127.59.59.59.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.155.155.127.59.59.59.2" class="ltx_td ltx_align_center">0.98</td>
<td id="A4.155.155.127.59.59.59.3" class="ltx_td ltx_align_center">0.92</td>
<td id="A4.155.155.127.59.59.59.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.155.155.127.59.59.59.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.155.155.127.59.59.59.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.155.155.127.59.59.59.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.155.155.127.59.59.59.8" class="ltx_td ltx_align_center">5.62</td>
</tr>
<tr id="A4.156.156.128.60.60.60" class="ltx_tr">
<td id="A4.156.156.128.60.60.60.1" class="ltx_td ltx_align_left">AWQ (8-Bit) <math id="A4.156.156.128.60.60.60.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.156.156.128.60.60.60.1.m1.1a"><mo stretchy="false" id="A4.156.156.128.60.60.60.1.m1.1.1" xref="A4.156.156.128.60.60.60.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.156.156.128.60.60.60.1.m1.1b"><ci id="A4.156.156.128.60.60.60.1.m1.1.1.cmml" xref="A4.156.156.128.60.60.60.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.156.156.128.60.60.60.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.156.156.128.60.60.60.2" class="ltx_td ltx_align_center">0.98</td>
<td id="A4.156.156.128.60.60.60.3" class="ltx_td ltx_align_center">0.94</td>
<td id="A4.156.156.128.60.60.60.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.156.156.128.60.60.60.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.156.156.128.60.60.60.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.156.156.128.60.60.60.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.156.156.128.60.60.60.8" class="ltx_td ltx_align_center">5.59</td>
</tr>
<tr id="A4.157.157.129.61.61.61" class="ltx_tr">
<td id="A4.157.157.129.61.61.61.1" class="ltx_td ltx_align_left">AWQ (2-Bit) <math id="A4.157.157.129.61.61.61.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.157.157.129.61.61.61.1.m1.1a"><mo stretchy="false" id="A4.157.157.129.61.61.61.1.m1.1.1" xref="A4.157.157.129.61.61.61.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.157.157.129.61.61.61.1.m1.1b"><ci id="A4.157.157.129.61.61.61.1.m1.1.1.cmml" xref="A4.157.157.129.61.61.61.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.157.157.129.61.61.61.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.157.157.129.61.61.61.2" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.157.157.129.61.61.61.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.157.157.129.61.61.61.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.157.157.129.61.61.61.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.157.157.129.61.61.61.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.157.157.129.61.61.61.7" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.157.157.129.61.61.61.8" class="ltx_td ltx_align_center">141960.91</td>
</tr>
<tr id="A4.158.158.130.62.62.62" class="ltx_tr">
<td id="A4.158.158.130.62.62.62.1" class="ltx_td ltx_align_left">AWQ (3-Bit) <math id="A4.158.158.130.62.62.62.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.158.158.130.62.62.62.1.m1.1a"><mo stretchy="false" id="A4.158.158.130.62.62.62.1.m1.1.1" xref="A4.158.158.130.62.62.62.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.158.158.130.62.62.62.1.m1.1b"><ci id="A4.158.158.130.62.62.62.1.m1.1.1.cmml" xref="A4.158.158.130.62.62.62.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.158.158.130.62.62.62.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.158.158.130.62.62.62.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.158.158.130.62.62.62.3" class="ltx_td ltx_align_center">0.71</td>
<td id="A4.158.158.130.62.62.62.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.158.158.130.62.62.62.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.158.158.130.62.62.62.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.158.158.130.62.62.62.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.158.158.130.62.62.62.8" class="ltx_td ltx_align_center">55.44</td>
</tr>
<tr id="A4.159.159.131.63.63.63" class="ltx_tr">
<td id="A4.159.159.131.63.63.63.1" class="ltx_td ltx_align_left">AWQ (4-Bit) <math id="A4.159.159.131.63.63.63.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.159.159.131.63.63.63.1.m1.1a"><mo stretchy="false" id="A4.159.159.131.63.63.63.1.m1.1.1" xref="A4.159.159.131.63.63.63.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.159.159.131.63.63.63.1.m1.1b"><ci id="A4.159.159.131.63.63.63.1.m1.1.1.cmml" xref="A4.159.159.131.63.63.63.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.159.159.131.63.63.63.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.159.159.131.63.63.63.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.159.159.131.63.63.63.3" class="ltx_td ltx_align_center">0.74</td>
<td id="A4.159.159.131.63.63.63.4" class="ltx_td ltx_align_center">0.07</td>
<td id="A4.159.159.131.63.63.63.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.159.159.131.63.63.63.6" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.159.159.131.63.63.63.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.159.159.131.63.63.63.8" class="ltx_td ltx_align_center">29.71</td>
</tr>
<tr id="A4.160.160.132.64.64.64" class="ltx_tr">
<td id="A4.160.160.132.64.64.64.1" class="ltx_td ltx_align_left">AWQ (5-Bit) <math id="A4.160.160.132.64.64.64.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.160.160.132.64.64.64.1.m1.1a"><mo stretchy="false" id="A4.160.160.132.64.64.64.1.m1.1.1" xref="A4.160.160.132.64.64.64.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.160.160.132.64.64.64.1.m1.1b"><ci id="A4.160.160.132.64.64.64.1.m1.1.1.cmml" xref="A4.160.160.132.64.64.64.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.160.160.132.64.64.64.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.160.160.132.64.64.64.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.160.160.132.64.64.64.3" class="ltx_td ltx_align_center">0.69</td>
<td id="A4.160.160.132.64.64.64.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.160.160.132.64.64.64.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.160.160.132.64.64.64.6" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.160.160.132.64.64.64.7" class="ltx_td ltx_align_center">0.6</td>
<td id="A4.160.160.132.64.64.64.8" class="ltx_td ltx_align_center">17.32</td>
</tr>
<tr id="A4.161.161.133.65.65.65" class="ltx_tr">
<td id="A4.161.161.133.65.65.65.1" class="ltx_td ltx_align_left">AWQ (6-Bit) <math id="A4.161.161.133.65.65.65.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.161.161.133.65.65.65.1.m1.1a"><mo stretchy="false" id="A4.161.161.133.65.65.65.1.m1.1.1" xref="A4.161.161.133.65.65.65.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.161.161.133.65.65.65.1.m1.1b"><ci id="A4.161.161.133.65.65.65.1.m1.1.1.cmml" xref="A4.161.161.133.65.65.65.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.161.161.133.65.65.65.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.161.161.133.65.65.65.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.161.161.133.65.65.65.3" class="ltx_td ltx_align_center">0.69</td>
<td id="A4.161.161.133.65.65.65.4" class="ltx_td ltx_align_center">0.06</td>
<td id="A4.161.161.133.65.65.65.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.161.161.133.65.65.65.6" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.161.161.133.65.65.65.7" class="ltx_td ltx_align_center">0.61</td>
<td id="A4.161.161.133.65.65.65.8" class="ltx_td ltx_align_center">16.6</td>
</tr>
<tr id="A4.162.162.134.66.66.66" class="ltx_tr">
<td id="A4.162.162.134.66.66.66.1" class="ltx_td ltx_align_left ltx_border_bb">AWQ (8-Bit) <math id="A4.162.162.134.66.66.66.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.162.162.134.66.66.66.1.m1.1a"><mo stretchy="false" id="A4.162.162.134.66.66.66.1.m1.1.1" xref="A4.162.162.134.66.66.66.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.162.162.134.66.66.66.1.m1.1b"><ci id="A4.162.162.134.66.66.66.1.m1.1.1.cmml" xref="A4.162.162.134.66.66.66.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.162.162.134.66.66.66.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.162.162.134.66.66.66.2" class="ltx_td ltx_align_center ltx_border_bb">1.0</td>
<td id="A4.162.162.134.66.66.66.3" class="ltx_td ltx_align_center ltx_border_bb">0.73</td>
<td id="A4.162.162.134.66.66.66.4" class="ltx_td ltx_align_center ltx_border_bb">0.06</td>
<td id="A4.162.162.134.66.66.66.5" class="ltx_td ltx_align_center ltx_border_bb">8.25</td>
<td id="A4.162.162.134.66.66.66.6" class="ltx_td ltx_align_center ltx_border_bb">0.57</td>
<td id="A4.162.162.134.66.66.66.7" class="ltx_td ltx_align_center ltx_border_bb">0.61</td>
<td id="A4.162.162.134.66.66.66.8" class="ltx_td ltx_align_center ltx_border_bb">21.64</td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="A4.322.322.294.227.2.1" class="ltx_text" style="font-size:90%;">Table 10</span>: </span><span id="A4.164.164.136.68.1" class="ltx_text" style="font-size:90%;">Detailed results for compression <math id="A4.164.164.136.68.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.164.164.136.68.1.m1.1b"><mo stretchy="false" id="A4.164.164.136.68.1.m1.1.1" xref="A4.164.164.136.68.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.164.164.136.68.1.m1.1c"><ci id="A4.164.164.136.68.1.m1.1.1.cmml" xref="A4.164.164.136.68.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.164.164.136.68.1.m1.1d">\rightarrow</annotation></semantics></math> editing.</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="A4.322.322.294.226" class="ltx_table ltx_figure_panel">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="A4.230.230.202.134.66" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:875.3pt;vertical-align:-1.5pt;"><span class="ltx_transformed_inner" style="transform:translate(-96.5pt,194.5pt) scale(0.69197624719063,0.69197624719063) ;">
<table id="A4.230.230.202.134.66.66" class="ltx_tabular ltx_align_middle">
<tr id="A4.230.230.202.134.66.66.67" class="ltx_tr">
<td id="A4.230.230.202.134.66.66.67.1" class="ltx_td ltx_border_tt"></td>
<td id="A4.230.230.202.134.66.66.67.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="3">Editing</td>
<td id="A4.230.230.202.134.66.66.67.3" class="ltx_td ltx_align_center ltx_border_tt">Compression</td>
<td id="A4.230.230.202.134.66.66.67.4" class="ltx_td ltx_align_center ltx_border_tt">Unlearning</td>
<td id="A4.230.230.202.134.66.66.67.5" class="ltx_td ltx_align_center ltx_border_tt" colspan="2">Utility</td>
</tr>
<tr id="A4.230.230.202.134.66.66.68" class="ltx_tr">
<td id="A4.230.230.202.134.66.66.68.1" class="ltx_td"></td>
<td id="A4.230.230.202.134.66.66.68.2" class="ltx_td ltx_align_center ltx_border_t">Edit Success</td>
<td id="A4.230.230.202.134.66.66.68.3" class="ltx_td ltx_align_center ltx_border_t">Generalization</td>
<td id="A4.230.230.202.134.66.66.68.4" class="ltx_td ltx_align_center ltx_border_t">Locality</td>
<td id="A4.230.230.202.134.66.66.68.5" class="ltx_td ltx_align_center ltx_border_t">Avg. Bits</td>
<td id="A4.230.230.202.134.66.66.68.6" class="ltx_td ltx_align_center ltx_border_t">Avg. WMDP</td>
<td id="A4.230.230.202.134.66.66.68.7" class="ltx_td ltx_align_center ltx_border_t">MMLU</td>
<td id="A4.230.230.202.134.66.66.68.8" class="ltx_td ltx_align_center ltx_border_t">WikiText PPL</td>
</tr>
<tr id="A4.165.165.137.69.1.1.1" class="ltx_tr">
<td id="A4.165.165.137.69.1.1.1.1" class="ltx_td ltx_align_left ltx_border_t">GA<math id="A4.165.165.137.69.1.1.1.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.165.165.137.69.1.1.1.1.m1.1a"><mo stretchy="false" id="A4.165.165.137.69.1.1.1.1.m1.1.1" xref="A4.165.165.137.69.1.1.1.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.165.165.137.69.1.1.1.1.m1.1b"><ci id="A4.165.165.137.69.1.1.1.1.m1.1.1.cmml" xref="A4.165.165.137.69.1.1.1.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.165.165.137.69.1.1.1.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.25)</td>
<td id="A4.165.165.137.69.1.1.1.2" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.165.165.137.69.1.1.1.3" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.165.165.137.69.1.1.1.4" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.165.165.137.69.1.1.1.5" class="ltx_td ltx_align_center ltx_border_t">12.25</td>
<td id="A4.165.165.137.69.1.1.1.6" class="ltx_td ltx_align_center ltx_border_t">0.46</td>
<td id="A4.165.165.137.69.1.1.1.7" class="ltx_td ltx_align_center ltx_border_t">0.51</td>
<td id="A4.165.165.137.69.1.1.1.8" class="ltx_td ltx_align_center ltx_border_t">inf</td>
</tr>
<tr id="A4.166.166.138.70.2.2.2" class="ltx_tr">
<td id="A4.166.166.138.70.2.2.2.1" class="ltx_td ltx_align_left">GA<math id="A4.166.166.138.70.2.2.2.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.166.166.138.70.2.2.2.1.m1.1a"><mo stretchy="false" id="A4.166.166.138.70.2.2.2.1.m1.1.1" xref="A4.166.166.138.70.2.2.2.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.166.166.138.70.2.2.2.1.m1.1b"><ci id="A4.166.166.138.70.2.2.2.1.m1.1.1.cmml" xref="A4.166.166.138.70.2.2.2.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.166.166.138.70.2.2.2.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.35)</td>
<td id="A4.166.166.138.70.2.2.2.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.166.166.138.70.2.2.2.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.166.166.138.70.2.2.2.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.166.166.138.70.2.2.2.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.166.166.138.70.2.2.2.6" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.166.166.138.70.2.2.2.7" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.166.166.138.70.2.2.2.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.167.167.139.71.3.3.3" class="ltx_tr">
<td id="A4.167.167.139.71.3.3.3.1" class="ltx_td ltx_align_left">GA<math id="A4.167.167.139.71.3.3.3.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.167.167.139.71.3.3.3.1.m1.1a"><mo stretchy="false" id="A4.167.167.139.71.3.3.3.1.m1.1.1" xref="A4.167.167.139.71.3.3.3.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.167.167.139.71.3.3.3.1.m1.1b"><ci id="A4.167.167.139.71.3.3.3.1.m1.1.1.cmml" xref="A4.167.167.139.71.3.3.3.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.167.167.139.71.3.3.3.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.45)</td>
<td id="A4.167.167.139.71.3.3.3.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.167.167.139.71.3.3.3.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.167.167.139.71.3.3.3.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.167.167.139.71.3.3.3.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.167.167.139.71.3.3.3.6" class="ltx_td ltx_align_center">0.42</td>
<td id="A4.167.167.139.71.3.3.3.7" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.167.167.139.71.3.3.3.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.168.168.140.72.4.4.4" class="ltx_tr">
<td id="A4.168.168.140.72.4.4.4.1" class="ltx_td ltx_align_left">GA<math id="A4.168.168.140.72.4.4.4.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.168.168.140.72.4.4.4.1.m1.1a"><mo stretchy="false" id="A4.168.168.140.72.4.4.4.1.m1.1.1" xref="A4.168.168.140.72.4.4.4.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.168.168.140.72.4.4.4.1.m1.1b"><ci id="A4.168.168.140.72.4.4.4.1.m1.1.1.cmml" xref="A4.168.168.140.72.4.4.4.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.168.168.140.72.4.4.4.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.55)</td>
<td id="A4.168.168.140.72.4.4.4.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.168.168.140.72.4.4.4.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.168.168.140.72.4.4.4.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.168.168.140.72.4.4.4.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.168.168.140.72.4.4.4.6" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.168.168.140.72.4.4.4.7" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.168.168.140.72.4.4.4.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.169.169.141.73.5.5.5" class="ltx_tr">
<td id="A4.169.169.141.73.5.5.5.1" class="ltx_td ltx_align_left">GA<math id="A4.169.169.141.73.5.5.5.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.169.169.141.73.5.5.5.1.m1.1a"><mo stretchy="false" id="A4.169.169.141.73.5.5.5.1.m1.1.1" xref="A4.169.169.141.73.5.5.5.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.169.169.141.73.5.5.5.1.m1.1b"><ci id="A4.169.169.141.73.5.5.5.1.m1.1.1.cmml" xref="A4.169.169.141.73.5.5.5.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.169.169.141.73.5.5.5.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.65)</td>
<td id="A4.169.169.141.73.5.5.5.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.169.169.141.73.5.5.5.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.169.169.141.73.5.5.5.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.169.169.141.73.5.5.5.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.169.169.141.73.5.5.5.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.169.169.141.73.5.5.5.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.169.169.141.73.5.5.5.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.170.170.142.74.6.6.6" class="ltx_tr">
<td id="A4.170.170.142.74.6.6.6.1" class="ltx_td ltx_align_left">GA<math id="A4.170.170.142.74.6.6.6.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.170.170.142.74.6.6.6.1.m1.1a"><mo stretchy="false" id="A4.170.170.142.74.6.6.6.1.m1.1.1" xref="A4.170.170.142.74.6.6.6.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.170.170.142.74.6.6.6.1.m1.1b"><ci id="A4.170.170.142.74.6.6.6.1.m1.1.1.cmml" xref="A4.170.170.142.74.6.6.6.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.170.170.142.74.6.6.6.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.75)</td>
<td id="A4.170.170.142.74.6.6.6.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.170.170.142.74.6.6.6.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.170.170.142.74.6.6.6.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.170.170.142.74.6.6.6.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.170.170.142.74.6.6.6.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.170.170.142.74.6.6.6.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.170.170.142.74.6.6.6.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.171.171.143.75.7.7.7" class="ltx_tr">
<td id="A4.171.171.143.75.7.7.7.1" class="ltx_td ltx_align_left">GA<math id="A4.171.171.143.75.7.7.7.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.171.171.143.75.7.7.7.1.m1.1a"><mo stretchy="false" id="A4.171.171.143.75.7.7.7.1.m1.1.1" xref="A4.171.171.143.75.7.7.7.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.171.171.143.75.7.7.7.1.m1.1b"><ci id="A4.171.171.143.75.7.7.7.1.m1.1.1.cmml" xref="A4.171.171.143.75.7.7.7.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.171.171.143.75.7.7.7.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.25)</td>
<td id="A4.171.171.143.75.7.7.7.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.171.171.143.75.7.7.7.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.171.171.143.75.7.7.7.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.171.171.143.75.7.7.7.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.171.171.143.75.7.7.7.6" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.171.171.143.75.7.7.7.7" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.171.171.143.75.7.7.7.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.172.172.144.76.8.8.8" class="ltx_tr">
<td id="A4.172.172.144.76.8.8.8.1" class="ltx_td ltx_align_left">GA<math id="A4.172.172.144.76.8.8.8.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.172.172.144.76.8.8.8.1.m1.1a"><mo stretchy="false" id="A4.172.172.144.76.8.8.8.1.m1.1.1" xref="A4.172.172.144.76.8.8.8.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.172.172.144.76.8.8.8.1.m1.1b"><ci id="A4.172.172.144.76.8.8.8.1.m1.1.1.cmml" xref="A4.172.172.144.76.8.8.8.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.172.172.144.76.8.8.8.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.35)</td>
<td id="A4.172.172.144.76.8.8.8.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.172.172.144.76.8.8.8.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.172.172.144.76.8.8.8.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.172.172.144.76.8.8.8.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.172.172.144.76.8.8.8.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.172.172.144.76.8.8.8.7" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.172.172.144.76.8.8.8.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.173.173.145.77.9.9.9" class="ltx_tr">
<td id="A4.173.173.145.77.9.9.9.1" class="ltx_td ltx_align_left">GA<math id="A4.173.173.145.77.9.9.9.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.173.173.145.77.9.9.9.1.m1.1a"><mo stretchy="false" id="A4.173.173.145.77.9.9.9.1.m1.1.1" xref="A4.173.173.145.77.9.9.9.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.173.173.145.77.9.9.9.1.m1.1b"><ci id="A4.173.173.145.77.9.9.9.1.m1.1.1.cmml" xref="A4.173.173.145.77.9.9.9.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.173.173.145.77.9.9.9.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.45)</td>
<td id="A4.173.173.145.77.9.9.9.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.173.173.145.77.9.9.9.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.173.173.145.77.9.9.9.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.173.173.145.77.9.9.9.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.173.173.145.77.9.9.9.6" class="ltx_td ltx_align_center">0.4</td>
<td id="A4.173.173.145.77.9.9.9.7" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.173.173.145.77.9.9.9.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.174.174.146.78.10.10.10" class="ltx_tr">
<td id="A4.174.174.146.78.10.10.10.1" class="ltx_td ltx_align_left">GA<math id="A4.174.174.146.78.10.10.10.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.174.174.146.78.10.10.10.1.m1.1a"><mo stretchy="false" id="A4.174.174.146.78.10.10.10.1.m1.1.1" xref="A4.174.174.146.78.10.10.10.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.174.174.146.78.10.10.10.1.m1.1b"><ci id="A4.174.174.146.78.10.10.10.1.m1.1.1.cmml" xref="A4.174.174.146.78.10.10.10.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.174.174.146.78.10.10.10.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.55)</td>
<td id="A4.174.174.146.78.10.10.10.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.174.174.146.78.10.10.10.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.174.174.146.78.10.10.10.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.174.174.146.78.10.10.10.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.174.174.146.78.10.10.10.6" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.174.174.146.78.10.10.10.7" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.174.174.146.78.10.10.10.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.175.175.147.79.11.11.11" class="ltx_tr">
<td id="A4.175.175.147.79.11.11.11.1" class="ltx_td ltx_align_left">GA<math id="A4.175.175.147.79.11.11.11.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.175.175.147.79.11.11.11.1.m1.1a"><mo stretchy="false" id="A4.175.175.147.79.11.11.11.1.m1.1.1" xref="A4.175.175.147.79.11.11.11.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.175.175.147.79.11.11.11.1.m1.1b"><ci id="A4.175.175.147.79.11.11.11.1.m1.1.1.cmml" xref="A4.175.175.147.79.11.11.11.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.175.175.147.79.11.11.11.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.65)</td>
<td id="A4.175.175.147.79.11.11.11.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.175.175.147.79.11.11.11.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.175.175.147.79.11.11.11.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.175.175.147.79.11.11.11.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.175.175.147.79.11.11.11.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.175.175.147.79.11.11.11.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.175.175.147.79.11.11.11.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.176.176.148.80.12.12.12" class="ltx_tr">
<td id="A4.176.176.148.80.12.12.12.1" class="ltx_td ltx_align_left">GA<math id="A4.176.176.148.80.12.12.12.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.176.176.148.80.12.12.12.1.m1.1a"><mo stretchy="false" id="A4.176.176.148.80.12.12.12.1.m1.1.1" xref="A4.176.176.148.80.12.12.12.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.176.176.148.80.12.12.12.1.m1.1b"><ci id="A4.176.176.148.80.12.12.12.1.m1.1.1.cmml" xref="A4.176.176.148.80.12.12.12.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.176.176.148.80.12.12.12.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.75)</td>
<td id="A4.176.176.148.80.12.12.12.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.176.176.148.80.12.12.12.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.176.176.148.80.12.12.12.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.176.176.148.80.12.12.12.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.176.176.148.80.12.12.12.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.176.176.148.80.12.12.12.7" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.176.176.148.80.12.12.12.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.177.177.149.81.13.13.13" class="ltx_tr">
<td id="A4.177.177.149.81.13.13.13.1" class="ltx_td ltx_align_left">GA<math id="A4.177.177.149.81.13.13.13.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.177.177.149.81.13.13.13.1.m1.1a"><mo stretchy="false" id="A4.177.177.149.81.13.13.13.1.m1.1.1" xref="A4.177.177.149.81.13.13.13.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.177.177.149.81.13.13.13.1.m1.1b"><ci id="A4.177.177.149.81.13.13.13.1.m1.1.1.cmml" xref="A4.177.177.149.81.13.13.13.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.177.177.149.81.13.13.13.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (2-Bit)</td>
<td id="A4.177.177.149.81.13.13.13.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.177.177.149.81.13.13.13.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.177.177.149.81.13.13.13.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.177.177.149.81.13.13.13.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.177.177.149.81.13.13.13.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.177.177.149.81.13.13.13.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.177.177.149.81.13.13.13.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.178.178.150.82.14.14.14" class="ltx_tr">
<td id="A4.178.178.150.82.14.14.14.1" class="ltx_td ltx_align_left">GA<math id="A4.178.178.150.82.14.14.14.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.178.178.150.82.14.14.14.1.m1.1a"><mo stretchy="false" id="A4.178.178.150.82.14.14.14.1.m1.1.1" xref="A4.178.178.150.82.14.14.14.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.178.178.150.82.14.14.14.1.m1.1b"><ci id="A4.178.178.150.82.14.14.14.1.m1.1.1.cmml" xref="A4.178.178.150.82.14.14.14.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.178.178.150.82.14.14.14.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (3-Bit)</td>
<td id="A4.178.178.150.82.14.14.14.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.178.178.150.82.14.14.14.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.178.178.150.82.14.14.14.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.178.178.150.82.14.14.14.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.178.178.150.82.14.14.14.6" class="ltx_td ltx_align_center">0.33</td>
<td id="A4.178.178.150.82.14.14.14.7" class="ltx_td ltx_align_center">0.39</td>
<td id="A4.178.178.150.82.14.14.14.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.179.179.151.83.15.15.15" class="ltx_tr">
<td id="A4.179.179.151.83.15.15.15.1" class="ltx_td ltx_align_left">GA<math id="A4.179.179.151.83.15.15.15.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.179.179.151.83.15.15.15.1.m1.1a"><mo stretchy="false" id="A4.179.179.151.83.15.15.15.1.m1.1.1" xref="A4.179.179.151.83.15.15.15.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.179.179.151.83.15.15.15.1.m1.1b"><ci id="A4.179.179.151.83.15.15.15.1.m1.1.1.cmml" xref="A4.179.179.151.83.15.15.15.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.179.179.151.83.15.15.15.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (4-Bit)</td>
<td id="A4.179.179.151.83.15.15.15.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.179.179.151.83.15.15.15.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.179.179.151.83.15.15.15.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.179.179.151.83.15.15.15.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.179.179.151.83.15.15.15.6" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.179.179.151.83.15.15.15.7" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.179.179.151.83.15.15.15.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.180.180.152.84.16.16.16" class="ltx_tr">
<td id="A4.180.180.152.84.16.16.16.1" class="ltx_td ltx_align_left">GA<math id="A4.180.180.152.84.16.16.16.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.180.180.152.84.16.16.16.1.m1.1a"><mo stretchy="false" id="A4.180.180.152.84.16.16.16.1.m1.1.1" xref="A4.180.180.152.84.16.16.16.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.180.180.152.84.16.16.16.1.m1.1b"><ci id="A4.180.180.152.84.16.16.16.1.m1.1.1.cmml" xref="A4.180.180.152.84.16.16.16.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.180.180.152.84.16.16.16.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (8-Bit)</td>
<td id="A4.180.180.152.84.16.16.16.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.180.180.152.84.16.16.16.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.180.180.152.84.16.16.16.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.180.180.152.84.16.16.16.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.180.180.152.84.16.16.16.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.180.180.152.84.16.16.16.7" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.180.180.152.84.16.16.16.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.181.181.153.85.17.17.17" class="ltx_tr">
<td id="A4.181.181.153.85.17.17.17.1" class="ltx_td ltx_align_left">GA<math id="A4.181.181.153.85.17.17.17.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.181.181.153.85.17.17.17.1.m1.1a"><mo stretchy="false" id="A4.181.181.153.85.17.17.17.1.m1.1.1" xref="A4.181.181.153.85.17.17.17.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.181.181.153.85.17.17.17.1.m1.1b"><ci id="A4.181.181.153.85.17.17.17.1.m1.1.1.cmml" xref="A4.181.181.153.85.17.17.17.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.181.181.153.85.17.17.17.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (2-Bit)</td>
<td id="A4.181.181.153.85.17.17.17.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.181.181.153.85.17.17.17.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.181.181.153.85.17.17.17.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.181.181.153.85.17.17.17.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.181.181.153.85.17.17.17.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.181.181.153.85.17.17.17.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.181.181.153.85.17.17.17.8" class="ltx_td ltx_align_center">1769133.88</td>
</tr>
<tr id="A4.182.182.154.86.18.18.18" class="ltx_tr">
<td id="A4.182.182.154.86.18.18.18.1" class="ltx_td ltx_align_left">GA<math id="A4.182.182.154.86.18.18.18.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.182.182.154.86.18.18.18.1.m1.1a"><mo stretchy="false" id="A4.182.182.154.86.18.18.18.1.m1.1.1" xref="A4.182.182.154.86.18.18.18.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.182.182.154.86.18.18.18.1.m1.1b"><ci id="A4.182.182.154.86.18.18.18.1.m1.1.1.cmml" xref="A4.182.182.154.86.18.18.18.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.182.182.154.86.18.18.18.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (3-Bit)</td>
<td id="A4.182.182.154.86.18.18.18.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.182.182.154.86.18.18.18.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.182.182.154.86.18.18.18.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.182.182.154.86.18.18.18.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.182.182.154.86.18.18.18.6" class="ltx_td ltx_align_center">0.41</td>
<td id="A4.182.182.154.86.18.18.18.7" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.182.182.154.86.18.18.18.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.183.183.155.87.19.19.19" class="ltx_tr">
<td id="A4.183.183.155.87.19.19.19.1" class="ltx_td ltx_align_left">GA<math id="A4.183.183.155.87.19.19.19.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.183.183.155.87.19.19.19.1.m1.1a"><mo stretchy="false" id="A4.183.183.155.87.19.19.19.1.m1.1.1" xref="A4.183.183.155.87.19.19.19.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.183.183.155.87.19.19.19.1.m1.1b"><ci id="A4.183.183.155.87.19.19.19.1.m1.1.1.cmml" xref="A4.183.183.155.87.19.19.19.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.183.183.155.87.19.19.19.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (4-Bit)</td>
<td id="A4.183.183.155.87.19.19.19.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.183.183.155.87.19.19.19.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.183.183.155.87.19.19.19.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.183.183.155.87.19.19.19.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.183.183.155.87.19.19.19.6" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.183.183.155.87.19.19.19.7" class="ltx_td ltx_align_center">0.47</td>
<td id="A4.183.183.155.87.19.19.19.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.184.184.156.88.20.20.20" class="ltx_tr">
<td id="A4.184.184.156.88.20.20.20.1" class="ltx_td ltx_align_left">GA<math id="A4.184.184.156.88.20.20.20.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.184.184.156.88.20.20.20.1.m1.1a"><mo stretchy="false" id="A4.184.184.156.88.20.20.20.1.m1.1.1" xref="A4.184.184.156.88.20.20.20.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.184.184.156.88.20.20.20.1.m1.1b"><ci id="A4.184.184.156.88.20.20.20.1.m1.1.1.cmml" xref="A4.184.184.156.88.20.20.20.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.184.184.156.88.20.20.20.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (5-Bit)</td>
<td id="A4.184.184.156.88.20.20.20.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.184.184.156.88.20.20.20.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.184.184.156.88.20.20.20.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.184.184.156.88.20.20.20.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.184.184.156.88.20.20.20.6" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.184.184.156.88.20.20.20.7" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.184.184.156.88.20.20.20.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.185.185.157.89.21.21.21" class="ltx_tr">
<td id="A4.185.185.157.89.21.21.21.1" class="ltx_td ltx_align_left">GA<math id="A4.185.185.157.89.21.21.21.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.185.185.157.89.21.21.21.1.m1.1a"><mo stretchy="false" id="A4.185.185.157.89.21.21.21.1.m1.1.1" xref="A4.185.185.157.89.21.21.21.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.185.185.157.89.21.21.21.1.m1.1b"><ci id="A4.185.185.157.89.21.21.21.1.m1.1.1.cmml" xref="A4.185.185.157.89.21.21.21.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.185.185.157.89.21.21.21.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (6-Bit)</td>
<td id="A4.185.185.157.89.21.21.21.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.185.185.157.89.21.21.21.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.185.185.157.89.21.21.21.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.185.185.157.89.21.21.21.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.185.185.157.89.21.21.21.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.185.185.157.89.21.21.21.7" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.185.185.157.89.21.21.21.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.186.186.158.90.22.22.22" class="ltx_tr">
<td id="A4.186.186.158.90.22.22.22.1" class="ltx_td ltx_align_left">GA<math id="A4.186.186.158.90.22.22.22.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.186.186.158.90.22.22.22.1.m1.1a"><mo stretchy="false" id="A4.186.186.158.90.22.22.22.1.m1.1.1" xref="A4.186.186.158.90.22.22.22.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.186.186.158.90.22.22.22.1.m1.1b"><ci id="A4.186.186.158.90.22.22.22.1.m1.1.1.cmml" xref="A4.186.186.158.90.22.22.22.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.186.186.158.90.22.22.22.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (8-Bit)</td>
<td id="A4.186.186.158.90.22.22.22.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.186.186.158.90.22.22.22.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.186.186.158.90.22.22.22.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.186.186.158.90.22.22.22.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.186.186.158.90.22.22.22.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.186.186.158.90.22.22.22.7" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.186.186.158.90.22.22.22.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.187.187.159.91.23.23.23" class="ltx_tr">
<td id="A4.187.187.159.91.23.23.23.1" class="ltx_td ltx_align_left">
<span id="A4.187.187.159.91.23.23.23.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

GD<math id="A4.187.187.159.91.23.23.23.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.187.187.159.91.23.23.23.1.m1.1a"><mo stretchy="false" id="A4.187.187.159.91.23.23.23.1.m1.1.1" xref="A4.187.187.159.91.23.23.23.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.187.187.159.91.23.23.23.1.m1.1b"><ci id="A4.187.187.159.91.23.23.23.1.m1.1.1.cmml" xref="A4.187.187.159.91.23.23.23.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.187.187.159.91.23.23.23.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.25)</td>
<td id="A4.187.187.159.91.23.23.23.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.187.187.159.91.23.23.23.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.187.187.159.91.23.23.23.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.187.187.159.91.23.23.23.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.187.187.159.91.23.23.23.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.187.187.159.91.23.23.23.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.187.187.159.91.23.23.23.8" class="ltx_td ltx_align_center">5.07</td>
</tr>
<tr id="A4.188.188.160.92.24.24.24" class="ltx_tr">
<td id="A4.188.188.160.92.24.24.24.1" class="ltx_td ltx_align_left">GD<math id="A4.188.188.160.92.24.24.24.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.188.188.160.92.24.24.24.1.m1.1a"><mo stretchy="false" id="A4.188.188.160.92.24.24.24.1.m1.1.1" xref="A4.188.188.160.92.24.24.24.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.188.188.160.92.24.24.24.1.m1.1b"><ci id="A4.188.188.160.92.24.24.24.1.m1.1.1.cmml" xref="A4.188.188.160.92.24.24.24.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.188.188.160.92.24.24.24.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.35)</td>
<td id="A4.188.188.160.92.24.24.24.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.188.188.160.92.24.24.24.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.188.188.160.92.24.24.24.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.188.188.160.92.24.24.24.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.188.188.160.92.24.24.24.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.188.188.160.92.24.24.24.7" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.188.188.160.92.24.24.24.8" class="ltx_td ltx_align_center">5.64</td>
</tr>
<tr id="A4.189.189.161.93.25.25.25" class="ltx_tr">
<td id="A4.189.189.161.93.25.25.25.1" class="ltx_td ltx_align_left">GD<math id="A4.189.189.161.93.25.25.25.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.189.189.161.93.25.25.25.1.m1.1a"><mo stretchy="false" id="A4.189.189.161.93.25.25.25.1.m1.1.1" xref="A4.189.189.161.93.25.25.25.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.189.189.161.93.25.25.25.1.m1.1b"><ci id="A4.189.189.161.93.25.25.25.1.m1.1.1.cmml" xref="A4.189.189.161.93.25.25.25.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.189.189.161.93.25.25.25.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.45)</td>
<td id="A4.189.189.161.93.25.25.25.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.189.189.161.93.25.25.25.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.189.189.161.93.25.25.25.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.189.189.161.93.25.25.25.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.189.189.161.93.25.25.25.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.189.189.161.93.25.25.25.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.189.189.161.93.25.25.25.8" class="ltx_td ltx_align_center">7.02</td>
</tr>
<tr id="A4.190.190.162.94.26.26.26" class="ltx_tr">
<td id="A4.190.190.162.94.26.26.26.1" class="ltx_td ltx_align_left">GD<math id="A4.190.190.162.94.26.26.26.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.190.190.162.94.26.26.26.1.m1.1a"><mo stretchy="false" id="A4.190.190.162.94.26.26.26.1.m1.1.1" xref="A4.190.190.162.94.26.26.26.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.190.190.162.94.26.26.26.1.m1.1b"><ci id="A4.190.190.162.94.26.26.26.1.m1.1.1.cmml" xref="A4.190.190.162.94.26.26.26.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.190.190.162.94.26.26.26.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.55)</td>
<td id="A4.190.190.162.94.26.26.26.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.190.190.162.94.26.26.26.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.190.190.162.94.26.26.26.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.190.190.162.94.26.26.26.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.190.190.162.94.26.26.26.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.190.190.162.94.26.26.26.7" class="ltx_td ltx_align_center">0.42</td>
<td id="A4.190.190.162.94.26.26.26.8" class="ltx_td ltx_align_center">10.6</td>
</tr>
<tr id="A4.191.191.163.95.27.27.27" class="ltx_tr">
<td id="A4.191.191.163.95.27.27.27.1" class="ltx_td ltx_align_left">GD<math id="A4.191.191.163.95.27.27.27.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.191.191.163.95.27.27.27.1.m1.1a"><mo stretchy="false" id="A4.191.191.163.95.27.27.27.1.m1.1.1" xref="A4.191.191.163.95.27.27.27.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.191.191.163.95.27.27.27.1.m1.1b"><ci id="A4.191.191.163.95.27.27.27.1.m1.1.1.cmml" xref="A4.191.191.163.95.27.27.27.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.191.191.163.95.27.27.27.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.65)</td>
<td id="A4.191.191.163.95.27.27.27.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.191.191.163.95.27.27.27.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.191.191.163.95.27.27.27.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.191.191.163.95.27.27.27.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.191.191.163.95.27.27.27.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.191.191.163.95.27.27.27.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.191.191.163.95.27.27.27.8" class="ltx_td ltx_align_center">25.97</td>
</tr>
<tr id="A4.192.192.164.96.28.28.28" class="ltx_tr">
<td id="A4.192.192.164.96.28.28.28.1" class="ltx_td ltx_align_left">GD<math id="A4.192.192.164.96.28.28.28.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.192.192.164.96.28.28.28.1.m1.1a"><mo stretchy="false" id="A4.192.192.164.96.28.28.28.1.m1.1.1" xref="A4.192.192.164.96.28.28.28.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.192.192.164.96.28.28.28.1.m1.1b"><ci id="A4.192.192.164.96.28.28.28.1.m1.1.1.cmml" xref="A4.192.192.164.96.28.28.28.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.192.192.164.96.28.28.28.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.75)</td>
<td id="A4.192.192.164.96.28.28.28.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.192.192.164.96.28.28.28.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.192.192.164.96.28.28.28.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.192.192.164.96.28.28.28.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.192.192.164.96.28.28.28.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.192.192.164.96.28.28.28.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.192.192.164.96.28.28.28.8" class="ltx_td ltx_align_center">119.71</td>
</tr>
<tr id="A4.193.193.165.97.29.29.29" class="ltx_tr">
<td id="A4.193.193.165.97.29.29.29.1" class="ltx_td ltx_align_left">GD<math id="A4.193.193.165.97.29.29.29.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.193.193.165.97.29.29.29.1.m1.1a"><mo stretchy="false" id="A4.193.193.165.97.29.29.29.1.m1.1.1" xref="A4.193.193.165.97.29.29.29.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.193.193.165.97.29.29.29.1.m1.1b"><ci id="A4.193.193.165.97.29.29.29.1.m1.1.1.cmml" xref="A4.193.193.165.97.29.29.29.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.193.193.165.97.29.29.29.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.25)</td>
<td id="A4.193.193.165.97.29.29.29.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.193.193.165.97.29.29.29.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.193.193.165.97.29.29.29.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.193.193.165.97.29.29.29.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.193.193.165.97.29.29.29.6" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.193.193.165.97.29.29.29.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.193.193.165.97.29.29.29.8" class="ltx_td ltx_align_center">5.34</td>
</tr>
<tr id="A4.194.194.166.98.30.30.30" class="ltx_tr">
<td id="A4.194.194.166.98.30.30.30.1" class="ltx_td ltx_align_left">GD<math id="A4.194.194.166.98.30.30.30.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.194.194.166.98.30.30.30.1.m1.1a"><mo stretchy="false" id="A4.194.194.166.98.30.30.30.1.m1.1.1" xref="A4.194.194.166.98.30.30.30.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.194.194.166.98.30.30.30.1.m1.1b"><ci id="A4.194.194.166.98.30.30.30.1.m1.1.1.cmml" xref="A4.194.194.166.98.30.30.30.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.194.194.166.98.30.30.30.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.35)</td>
<td id="A4.194.194.166.98.30.30.30.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.194.194.166.98.30.30.30.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.194.194.166.98.30.30.30.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.194.194.166.98.30.30.30.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.194.194.166.98.30.30.30.6" class="ltx_td ltx_align_center">0.38</td>
<td id="A4.194.194.166.98.30.30.30.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.194.194.166.98.30.30.30.8" class="ltx_td ltx_align_center">6.61</td>
</tr>
<tr id="A4.195.195.167.99.31.31.31" class="ltx_tr">
<td id="A4.195.195.167.99.31.31.31.1" class="ltx_td ltx_align_left">GD<math id="A4.195.195.167.99.31.31.31.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.195.195.167.99.31.31.31.1.m1.1a"><mo stretchy="false" id="A4.195.195.167.99.31.31.31.1.m1.1.1" xref="A4.195.195.167.99.31.31.31.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.195.195.167.99.31.31.31.1.m1.1b"><ci id="A4.195.195.167.99.31.31.31.1.m1.1.1.cmml" xref="A4.195.195.167.99.31.31.31.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.195.195.167.99.31.31.31.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.45)</td>
<td id="A4.195.195.167.99.31.31.31.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.195.195.167.99.31.31.31.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.195.195.167.99.31.31.31.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.195.195.167.99.31.31.31.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.195.195.167.99.31.31.31.6" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.195.195.167.99.31.31.31.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.195.195.167.99.31.31.31.8" class="ltx_td ltx_align_center">9.9</td>
</tr>
<tr id="A4.196.196.168.100.32.32.32" class="ltx_tr">
<td id="A4.196.196.168.100.32.32.32.1" class="ltx_td ltx_align_left">GD<math id="A4.196.196.168.100.32.32.32.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.196.196.168.100.32.32.32.1.m1.1a"><mo stretchy="false" id="A4.196.196.168.100.32.32.32.1.m1.1.1" xref="A4.196.196.168.100.32.32.32.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.196.196.168.100.32.32.32.1.m1.1b"><ci id="A4.196.196.168.100.32.32.32.1.m1.1.1.cmml" xref="A4.196.196.168.100.32.32.32.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.196.196.168.100.32.32.32.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.55)</td>
<td id="A4.196.196.168.100.32.32.32.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.196.196.168.100.32.32.32.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.196.196.168.100.32.32.32.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.196.196.168.100.32.32.32.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.196.196.168.100.32.32.32.6" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.196.196.168.100.32.32.32.7" class="ltx_td ltx_align_center">0.39</td>
<td id="A4.196.196.168.100.32.32.32.8" class="ltx_td ltx_align_center">22.22</td>
</tr>
<tr id="A4.197.197.169.101.33.33.33" class="ltx_tr">
<td id="A4.197.197.169.101.33.33.33.1" class="ltx_td ltx_align_left">GD<math id="A4.197.197.169.101.33.33.33.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.197.197.169.101.33.33.33.1.m1.1a"><mo stretchy="false" id="A4.197.197.169.101.33.33.33.1.m1.1.1" xref="A4.197.197.169.101.33.33.33.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.197.197.169.101.33.33.33.1.m1.1b"><ci id="A4.197.197.169.101.33.33.33.1.m1.1.1.cmml" xref="A4.197.197.169.101.33.33.33.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.197.197.169.101.33.33.33.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.65)</td>
<td id="A4.197.197.169.101.33.33.33.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.197.197.169.101.33.33.33.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.197.197.169.101.33.33.33.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.197.197.169.101.33.33.33.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.197.197.169.101.33.33.33.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.197.197.169.101.33.33.33.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.197.197.169.101.33.33.33.8" class="ltx_td ltx_align_center">184.75</td>
</tr>
<tr id="A4.198.198.170.102.34.34.34" class="ltx_tr">
<td id="A4.198.198.170.102.34.34.34.1" class="ltx_td ltx_align_left">GD<math id="A4.198.198.170.102.34.34.34.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.198.198.170.102.34.34.34.1.m1.1a"><mo stretchy="false" id="A4.198.198.170.102.34.34.34.1.m1.1.1" xref="A4.198.198.170.102.34.34.34.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.198.198.170.102.34.34.34.1.m1.1b"><ci id="A4.198.198.170.102.34.34.34.1.m1.1.1.cmml" xref="A4.198.198.170.102.34.34.34.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.198.198.170.102.34.34.34.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.75)</td>
<td id="A4.198.198.170.102.34.34.34.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.198.198.170.102.34.34.34.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.198.198.170.102.34.34.34.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.198.198.170.102.34.34.34.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.198.198.170.102.34.34.34.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.198.198.170.102.34.34.34.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.198.198.170.102.34.34.34.8" class="ltx_td ltx_align_center">3298.92</td>
</tr>
<tr id="A4.199.199.171.103.35.35.35" class="ltx_tr">
<td id="A4.199.199.171.103.35.35.35.1" class="ltx_td ltx_align_left">GD<math id="A4.199.199.171.103.35.35.35.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.199.199.171.103.35.35.35.1.m1.1a"><mo stretchy="false" id="A4.199.199.171.103.35.35.35.1.m1.1.1" xref="A4.199.199.171.103.35.35.35.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.199.199.171.103.35.35.35.1.m1.1b"><ci id="A4.199.199.171.103.35.35.35.1.m1.1.1.cmml" xref="A4.199.199.171.103.35.35.35.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.199.199.171.103.35.35.35.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (2-Bit)</td>
<td id="A4.199.199.171.103.35.35.35.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.199.199.171.103.35.35.35.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.199.199.171.103.35.35.35.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.199.199.171.103.35.35.35.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.199.199.171.103.35.35.35.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.199.199.171.103.35.35.35.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.199.199.171.103.35.35.35.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.200.200.172.104.36.36.36" class="ltx_tr">
<td id="A4.200.200.172.104.36.36.36.1" class="ltx_td ltx_align_left">GD<math id="A4.200.200.172.104.36.36.36.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.200.200.172.104.36.36.36.1.m1.1a"><mo stretchy="false" id="A4.200.200.172.104.36.36.36.1.m1.1.1" xref="A4.200.200.172.104.36.36.36.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.200.200.172.104.36.36.36.1.m1.1b"><ci id="A4.200.200.172.104.36.36.36.1.m1.1.1.cmml" xref="A4.200.200.172.104.36.36.36.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.200.200.172.104.36.36.36.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (3-Bit)</td>
<td id="A4.200.200.172.104.36.36.36.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.200.200.172.104.36.36.36.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.200.200.172.104.36.36.36.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.200.200.172.104.36.36.36.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.200.200.172.104.36.36.36.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.200.200.172.104.36.36.36.7" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.200.200.172.104.36.36.36.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.201.201.173.105.37.37.37" class="ltx_tr">
<td id="A4.201.201.173.105.37.37.37.1" class="ltx_td ltx_align_left">GD<math id="A4.201.201.173.105.37.37.37.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.201.201.173.105.37.37.37.1.m1.1a"><mo stretchy="false" id="A4.201.201.173.105.37.37.37.1.m1.1.1" xref="A4.201.201.173.105.37.37.37.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.201.201.173.105.37.37.37.1.m1.1b"><ci id="A4.201.201.173.105.37.37.37.1.m1.1.1.cmml" xref="A4.201.201.173.105.37.37.37.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.201.201.173.105.37.37.37.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (4-Bit)</td>
<td id="A4.201.201.173.105.37.37.37.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.201.201.173.105.37.37.37.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.201.201.173.105.37.37.37.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.201.201.173.105.37.37.37.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.201.201.173.105.37.37.37.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.201.201.173.105.37.37.37.7" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.201.201.173.105.37.37.37.8" class="ltx_td ltx_align_center">5.106230513313957e+35</td>
</tr>
<tr id="A4.202.202.174.106.38.38.38" class="ltx_tr">
<td id="A4.202.202.174.106.38.38.38.1" class="ltx_td ltx_align_left">GD<math id="A4.202.202.174.106.38.38.38.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.202.202.174.106.38.38.38.1.m1.1a"><mo stretchy="false" id="A4.202.202.174.106.38.38.38.1.m1.1.1" xref="A4.202.202.174.106.38.38.38.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.202.202.174.106.38.38.38.1.m1.1b"><ci id="A4.202.202.174.106.38.38.38.1.m1.1.1.cmml" xref="A4.202.202.174.106.38.38.38.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.202.202.174.106.38.38.38.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (8-Bit)</td>
<td id="A4.202.202.174.106.38.38.38.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.202.202.174.106.38.38.38.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.202.202.174.106.38.38.38.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.202.202.174.106.38.38.38.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.202.202.174.106.38.38.38.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.202.202.174.106.38.38.38.7" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.202.202.174.106.38.38.38.8" class="ltx_td ltx_align_center">2.445586872346403e+26</td>
</tr>
<tr id="A4.203.203.175.107.39.39.39" class="ltx_tr">
<td id="A4.203.203.175.107.39.39.39.1" class="ltx_td ltx_align_left">GD<math id="A4.203.203.175.107.39.39.39.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.203.203.175.107.39.39.39.1.m1.1a"><mo stretchy="false" id="A4.203.203.175.107.39.39.39.1.m1.1.1" xref="A4.203.203.175.107.39.39.39.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.203.203.175.107.39.39.39.1.m1.1b"><ci id="A4.203.203.175.107.39.39.39.1.m1.1.1.cmml" xref="A4.203.203.175.107.39.39.39.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.203.203.175.107.39.39.39.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (2-Bit)</td>
<td id="A4.203.203.175.107.39.39.39.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.203.203.175.107.39.39.39.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.203.203.175.107.39.39.39.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.203.203.175.107.39.39.39.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.203.203.175.107.39.39.39.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.203.203.175.107.39.39.39.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.203.203.175.107.39.39.39.8" class="ltx_td ltx_align_center">2598854.5</td>
</tr>
<tr id="A4.204.204.176.108.40.40.40" class="ltx_tr">
<td id="A4.204.204.176.108.40.40.40.1" class="ltx_td ltx_align_left">GD<math id="A4.204.204.176.108.40.40.40.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.204.204.176.108.40.40.40.1.m1.1a"><mo stretchy="false" id="A4.204.204.176.108.40.40.40.1.m1.1.1" xref="A4.204.204.176.108.40.40.40.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.204.204.176.108.40.40.40.1.m1.1b"><ci id="A4.204.204.176.108.40.40.40.1.m1.1.1.cmml" xref="A4.204.204.176.108.40.40.40.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.204.204.176.108.40.40.40.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (3-Bit)</td>
<td id="A4.204.204.176.108.40.40.40.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.204.204.176.108.40.40.40.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.204.204.176.108.40.40.40.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.204.204.176.108.40.40.40.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.204.204.176.108.40.40.40.6" class="ltx_td ltx_align_center">0.33</td>
<td id="A4.204.204.176.108.40.40.40.7" class="ltx_td ltx_align_center">0.33</td>
<td id="A4.204.204.176.108.40.40.40.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.205.205.177.109.41.41.41" class="ltx_tr">
<td id="A4.205.205.177.109.41.41.41.1" class="ltx_td ltx_align_left">GD<math id="A4.205.205.177.109.41.41.41.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.205.205.177.109.41.41.41.1.m1.1a"><mo stretchy="false" id="A4.205.205.177.109.41.41.41.1.m1.1.1" xref="A4.205.205.177.109.41.41.41.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.205.205.177.109.41.41.41.1.m1.1b"><ci id="A4.205.205.177.109.41.41.41.1.m1.1.1.cmml" xref="A4.205.205.177.109.41.41.41.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.205.205.177.109.41.41.41.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (4-Bit)</td>
<td id="A4.205.205.177.109.41.41.41.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.205.205.177.109.41.41.41.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.205.205.177.109.41.41.41.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.205.205.177.109.41.41.41.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.205.205.177.109.41.41.41.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.205.205.177.109.41.41.41.7" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.205.205.177.109.41.41.41.8" class="ltx_td ltx_align_center">2.0452847856930923e+28</td>
</tr>
<tr id="A4.206.206.178.110.42.42.42" class="ltx_tr">
<td id="A4.206.206.178.110.42.42.42.1" class="ltx_td ltx_align_left">GD<math id="A4.206.206.178.110.42.42.42.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.206.206.178.110.42.42.42.1.m1.1a"><mo stretchy="false" id="A4.206.206.178.110.42.42.42.1.m1.1.1" xref="A4.206.206.178.110.42.42.42.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.206.206.178.110.42.42.42.1.m1.1b"><ci id="A4.206.206.178.110.42.42.42.1.m1.1.1.cmml" xref="A4.206.206.178.110.42.42.42.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.206.206.178.110.42.42.42.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (5-Bit)</td>
<td id="A4.206.206.178.110.42.42.42.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.206.206.178.110.42.42.42.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.206.206.178.110.42.42.42.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.206.206.178.110.42.42.42.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.206.206.178.110.42.42.42.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.206.206.178.110.42.42.42.7" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.206.206.178.110.42.42.42.8" class="ltx_td ltx_align_center">1.0052004503012292e+28</td>
</tr>
<tr id="A4.207.207.179.111.43.43.43" class="ltx_tr">
<td id="A4.207.207.179.111.43.43.43.1" class="ltx_td ltx_align_left">GD<math id="A4.207.207.179.111.43.43.43.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.207.207.179.111.43.43.43.1.m1.1a"><mo stretchy="false" id="A4.207.207.179.111.43.43.43.1.m1.1.1" xref="A4.207.207.179.111.43.43.43.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.207.207.179.111.43.43.43.1.m1.1b"><ci id="A4.207.207.179.111.43.43.43.1.m1.1.1.cmml" xref="A4.207.207.179.111.43.43.43.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.207.207.179.111.43.43.43.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (6-Bit)</td>
<td id="A4.207.207.179.111.43.43.43.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.207.207.179.111.43.43.43.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.207.207.179.111.43.43.43.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.207.207.179.111.43.43.43.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.207.207.179.111.43.43.43.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.207.207.179.111.43.43.43.7" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.207.207.179.111.43.43.43.8" class="ltx_td ltx_align_center">6.499345951176105e+25</td>
</tr>
<tr id="A4.208.208.180.112.44.44.44" class="ltx_tr">
<td id="A4.208.208.180.112.44.44.44.1" class="ltx_td ltx_align_left">GD<math id="A4.208.208.180.112.44.44.44.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.208.208.180.112.44.44.44.1.m1.1a"><mo stretchy="false" id="A4.208.208.180.112.44.44.44.1.m1.1.1" xref="A4.208.208.180.112.44.44.44.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.208.208.180.112.44.44.44.1.m1.1b"><ci id="A4.208.208.180.112.44.44.44.1.m1.1.1.cmml" xref="A4.208.208.180.112.44.44.44.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.208.208.180.112.44.44.44.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (8-Bit)</td>
<td id="A4.208.208.180.112.44.44.44.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.208.208.180.112.44.44.44.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.208.208.180.112.44.44.44.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.208.208.180.112.44.44.44.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.208.208.180.112.44.44.44.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.208.208.180.112.44.44.44.7" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.208.208.180.112.44.44.44.8" class="ltx_td ltx_align_center">9.430016153317283e+25</td>
</tr>
<tr id="A4.209.209.181.113.45.45.45" class="ltx_tr">
<td id="A4.209.209.181.113.45.45.45.1" class="ltx_td ltx_align_left">
<span id="A4.209.209.181.113.45.45.45.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

RMU<math id="A4.209.209.181.113.45.45.45.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.209.209.181.113.45.45.45.1.m1.1a"><mo stretchy="false" id="A4.209.209.181.113.45.45.45.1.m1.1.1" xref="A4.209.209.181.113.45.45.45.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.209.209.181.113.45.45.45.1.m1.1b"><ci id="A4.209.209.181.113.45.45.45.1.m1.1.1.cmml" xref="A4.209.209.181.113.45.45.45.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.209.209.181.113.45.45.45.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.25)</td>
<td id="A4.209.209.181.113.45.45.45.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.209.209.181.113.45.45.45.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.209.209.181.113.45.45.45.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.209.209.181.113.45.45.45.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.209.209.181.113.45.45.45.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.209.209.181.113.45.45.45.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.209.209.181.113.45.45.45.8" class="ltx_td ltx_align_center">5.93</td>
</tr>
<tr id="A4.210.210.182.114.46.46.46" class="ltx_tr">
<td id="A4.210.210.182.114.46.46.46.1" class="ltx_td ltx_align_left">RMU<math id="A4.210.210.182.114.46.46.46.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.210.210.182.114.46.46.46.1.m1.1a"><mo stretchy="false" id="A4.210.210.182.114.46.46.46.1.m1.1.1" xref="A4.210.210.182.114.46.46.46.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.210.210.182.114.46.46.46.1.m1.1b"><ci id="A4.210.210.182.114.46.46.46.1.m1.1.1.cmml" xref="A4.210.210.182.114.46.46.46.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.210.210.182.114.46.46.46.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.35)</td>
<td id="A4.210.210.182.114.46.46.46.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.210.210.182.114.46.46.46.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.210.210.182.114.46.46.46.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.210.210.182.114.46.46.46.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.210.210.182.114.46.46.46.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.210.210.182.114.46.46.46.7" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.210.210.182.114.46.46.46.8" class="ltx_td ltx_align_center">6.39</td>
</tr>
<tr id="A4.211.211.183.115.47.47.47" class="ltx_tr">
<td id="A4.211.211.183.115.47.47.47.1" class="ltx_td ltx_align_left">RMU<math id="A4.211.211.183.115.47.47.47.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.211.211.183.115.47.47.47.1.m1.1a"><mo stretchy="false" id="A4.211.211.183.115.47.47.47.1.m1.1.1" xref="A4.211.211.183.115.47.47.47.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.211.211.183.115.47.47.47.1.m1.1b"><ci id="A4.211.211.183.115.47.47.47.1.m1.1.1.cmml" xref="A4.211.211.183.115.47.47.47.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.211.211.183.115.47.47.47.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.45)</td>
<td id="A4.211.211.183.115.47.47.47.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.211.211.183.115.47.47.47.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.211.211.183.115.47.47.47.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.211.211.183.115.47.47.47.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.211.211.183.115.47.47.47.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.211.211.183.115.47.47.47.7" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.211.211.183.115.47.47.47.8" class="ltx_td ltx_align_center">8.14</td>
</tr>
<tr id="A4.212.212.184.116.48.48.48" class="ltx_tr">
<td id="A4.212.212.184.116.48.48.48.1" class="ltx_td ltx_align_left">RMU<math id="A4.212.212.184.116.48.48.48.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.212.212.184.116.48.48.48.1.m1.1a"><mo stretchy="false" id="A4.212.212.184.116.48.48.48.1.m1.1.1" xref="A4.212.212.184.116.48.48.48.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.212.212.184.116.48.48.48.1.m1.1b"><ci id="A4.212.212.184.116.48.48.48.1.m1.1.1.cmml" xref="A4.212.212.184.116.48.48.48.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.212.212.184.116.48.48.48.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.55)</td>
<td id="A4.212.212.184.116.48.48.48.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.212.212.184.116.48.48.48.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.212.212.184.116.48.48.48.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.212.212.184.116.48.48.48.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.212.212.184.116.48.48.48.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.212.212.184.116.48.48.48.7" class="ltx_td ltx_align_center">0.39</td>
<td id="A4.212.212.184.116.48.48.48.8" class="ltx_td ltx_align_center">10.58</td>
</tr>
<tr id="A4.213.213.185.117.49.49.49" class="ltx_tr">
<td id="A4.213.213.185.117.49.49.49.1" class="ltx_td ltx_align_left">RMU<math id="A4.213.213.185.117.49.49.49.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.213.213.185.117.49.49.49.1.m1.1a"><mo stretchy="false" id="A4.213.213.185.117.49.49.49.1.m1.1.1" xref="A4.213.213.185.117.49.49.49.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.213.213.185.117.49.49.49.1.m1.1b"><ci id="A4.213.213.185.117.49.49.49.1.m1.1.1.cmml" xref="A4.213.213.185.117.49.49.49.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.213.213.185.117.49.49.49.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.65)</td>
<td id="A4.213.213.185.117.49.49.49.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.213.213.185.117.49.49.49.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.213.213.185.117.49.49.49.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.213.213.185.117.49.49.49.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.213.213.185.117.49.49.49.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.213.213.185.117.49.49.49.7" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.213.213.185.117.49.49.49.8" class="ltx_td ltx_align_center">35.97</td>
</tr>
<tr id="A4.214.214.186.118.50.50.50" class="ltx_tr">
<td id="A4.214.214.186.118.50.50.50.1" class="ltx_td ltx_align_left">RMU<math id="A4.214.214.186.118.50.50.50.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.214.214.186.118.50.50.50.1.m1.1a"><mo stretchy="false" id="A4.214.214.186.118.50.50.50.1.m1.1.1" xref="A4.214.214.186.118.50.50.50.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.214.214.186.118.50.50.50.1.m1.1b"><ci id="A4.214.214.186.118.50.50.50.1.m1.1.1.cmml" xref="A4.214.214.186.118.50.50.50.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.214.214.186.118.50.50.50.1.m1.1c">\rightarrow</annotation></semantics></math>SparseGPT (0.75)</td>
<td id="A4.214.214.186.118.50.50.50.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.214.214.186.118.50.50.50.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.214.214.186.118.50.50.50.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.214.214.186.118.50.50.50.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.214.214.186.118.50.50.50.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.214.214.186.118.50.50.50.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.214.214.186.118.50.50.50.8" class="ltx_td ltx_align_center">101.32</td>
</tr>
<tr id="A4.215.215.187.119.51.51.51" class="ltx_tr">
<td id="A4.215.215.187.119.51.51.51.1" class="ltx_td ltx_align_left">RMU<math id="A4.215.215.187.119.51.51.51.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.215.215.187.119.51.51.51.1.m1.1a"><mo stretchy="false" id="A4.215.215.187.119.51.51.51.1.m1.1.1" xref="A4.215.215.187.119.51.51.51.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.215.215.187.119.51.51.51.1.m1.1b"><ci id="A4.215.215.187.119.51.51.51.1.m1.1.1.cmml" xref="A4.215.215.187.119.51.51.51.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.215.215.187.119.51.51.51.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.25)</td>
<td id="A4.215.215.187.119.51.51.51.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.215.215.187.119.51.51.51.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.215.215.187.119.51.51.51.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.215.215.187.119.51.51.51.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.215.215.187.119.51.51.51.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.215.215.187.119.51.51.51.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.215.215.187.119.51.51.51.8" class="ltx_td ltx_align_center">5.88</td>
</tr>
<tr id="A4.216.216.188.120.52.52.52" class="ltx_tr">
<td id="A4.216.216.188.120.52.52.52.1" class="ltx_td ltx_align_left">RMU<math id="A4.216.216.188.120.52.52.52.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.216.216.188.120.52.52.52.1.m1.1a"><mo stretchy="false" id="A4.216.216.188.120.52.52.52.1.m1.1.1" xref="A4.216.216.188.120.52.52.52.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.216.216.188.120.52.52.52.1.m1.1b"><ci id="A4.216.216.188.120.52.52.52.1.m1.1.1.cmml" xref="A4.216.216.188.120.52.52.52.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.216.216.188.120.52.52.52.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.35)</td>
<td id="A4.216.216.188.120.52.52.52.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.216.216.188.120.52.52.52.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.216.216.188.120.52.52.52.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.216.216.188.120.52.52.52.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.216.216.188.120.52.52.52.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.216.216.188.120.52.52.52.7" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.216.216.188.120.52.52.52.8" class="ltx_td ltx_align_center">6.36</td>
</tr>
<tr id="A4.217.217.189.121.53.53.53" class="ltx_tr">
<td id="A4.217.217.189.121.53.53.53.1" class="ltx_td ltx_align_left">RMU<math id="A4.217.217.189.121.53.53.53.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.217.217.189.121.53.53.53.1.m1.1a"><mo stretchy="false" id="A4.217.217.189.121.53.53.53.1.m1.1.1" xref="A4.217.217.189.121.53.53.53.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.217.217.189.121.53.53.53.1.m1.1b"><ci id="A4.217.217.189.121.53.53.53.1.m1.1.1.cmml" xref="A4.217.217.189.121.53.53.53.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.217.217.189.121.53.53.53.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.45)</td>
<td id="A4.217.217.189.121.53.53.53.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.217.217.189.121.53.53.53.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.217.217.189.121.53.53.53.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.217.217.189.121.53.53.53.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.217.217.189.121.53.53.53.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.217.217.189.121.53.53.53.7" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.217.217.189.121.53.53.53.8" class="ltx_td ltx_align_center">7.71</td>
</tr>
<tr id="A4.218.218.190.122.54.54.54" class="ltx_tr">
<td id="A4.218.218.190.122.54.54.54.1" class="ltx_td ltx_align_left">RMU<math id="A4.218.218.190.122.54.54.54.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.218.218.190.122.54.54.54.1.m1.1a"><mo stretchy="false" id="A4.218.218.190.122.54.54.54.1.m1.1.1" xref="A4.218.218.190.122.54.54.54.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.218.218.190.122.54.54.54.1.m1.1b"><ci id="A4.218.218.190.122.54.54.54.1.m1.1.1.cmml" xref="A4.218.218.190.122.54.54.54.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.218.218.190.122.54.54.54.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.55)</td>
<td id="A4.218.218.190.122.54.54.54.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.218.218.190.122.54.54.54.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.218.218.190.122.54.54.54.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.218.218.190.122.54.54.54.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.218.218.190.122.54.54.54.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.218.218.190.122.54.54.54.7" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.218.218.190.122.54.54.54.8" class="ltx_td ltx_align_center">13.73</td>
</tr>
<tr id="A4.219.219.191.123.55.55.55" class="ltx_tr">
<td id="A4.219.219.191.123.55.55.55.1" class="ltx_td ltx_align_left">RMU<math id="A4.219.219.191.123.55.55.55.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.219.219.191.123.55.55.55.1.m1.1a"><mo stretchy="false" id="A4.219.219.191.123.55.55.55.1.m1.1.1" xref="A4.219.219.191.123.55.55.55.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.219.219.191.123.55.55.55.1.m1.1b"><ci id="A4.219.219.191.123.55.55.55.1.m1.1.1.cmml" xref="A4.219.219.191.123.55.55.55.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.219.219.191.123.55.55.55.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.65)</td>
<td id="A4.219.219.191.123.55.55.55.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.219.219.191.123.55.55.55.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.219.219.191.123.55.55.55.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.219.219.191.123.55.55.55.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.219.219.191.123.55.55.55.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.219.219.191.123.55.55.55.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.219.219.191.123.55.55.55.8" class="ltx_td ltx_align_center">52.67</td>
</tr>
<tr id="A4.220.220.192.124.56.56.56" class="ltx_tr">
<td id="A4.220.220.192.124.56.56.56.1" class="ltx_td ltx_align_left">RMU<math id="A4.220.220.192.124.56.56.56.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.220.220.192.124.56.56.56.1.m1.1a"><mo stretchy="false" id="A4.220.220.192.124.56.56.56.1.m1.1.1" xref="A4.220.220.192.124.56.56.56.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.220.220.192.124.56.56.56.1.m1.1b"><ci id="A4.220.220.192.124.56.56.56.1.m1.1.1.cmml" xref="A4.220.220.192.124.56.56.56.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.220.220.192.124.56.56.56.1.m1.1c">\rightarrow</annotation></semantics></math>Wanda (0.75)</td>
<td id="A4.220.220.192.124.56.56.56.2" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.220.220.192.124.56.56.56.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.220.220.192.124.56.56.56.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.220.220.192.124.56.56.56.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.220.220.192.124.56.56.56.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.220.220.192.124.56.56.56.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.220.220.192.124.56.56.56.8" class="ltx_td ltx_align_center">334.45</td>
</tr>
<tr id="A4.221.221.193.125.57.57.57" class="ltx_tr">
<td id="A4.221.221.193.125.57.57.57.1" class="ltx_td ltx_align_left">RMU<math id="A4.221.221.193.125.57.57.57.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.221.221.193.125.57.57.57.1.m1.1a"><mo stretchy="false" id="A4.221.221.193.125.57.57.57.1.m1.1.1" xref="A4.221.221.193.125.57.57.57.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.221.221.193.125.57.57.57.1.m1.1b"><ci id="A4.221.221.193.125.57.57.57.1.m1.1.1.cmml" xref="A4.221.221.193.125.57.57.57.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.221.221.193.125.57.57.57.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (2-Bit)</td>
<td id="A4.221.221.193.125.57.57.57.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.221.221.193.125.57.57.57.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.221.221.193.125.57.57.57.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.221.221.193.125.57.57.57.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.221.221.193.125.57.57.57.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.221.221.193.125.57.57.57.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.221.221.193.125.57.57.57.8" class="ltx_td ltx_align_center">3451.67</td>
</tr>
<tr id="A4.222.222.194.126.58.58.58" class="ltx_tr">
<td id="A4.222.222.194.126.58.58.58.1" class="ltx_td ltx_align_left">RMU<math id="A4.222.222.194.126.58.58.58.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.222.222.194.126.58.58.58.1.m1.1a"><mo stretchy="false" id="A4.222.222.194.126.58.58.58.1.m1.1.1" xref="A4.222.222.194.126.58.58.58.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.222.222.194.126.58.58.58.1.m1.1b"><ci id="A4.222.222.194.126.58.58.58.1.m1.1.1.cmml" xref="A4.222.222.194.126.58.58.58.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.222.222.194.126.58.58.58.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (3-Bit)</td>
<td id="A4.222.222.194.126.58.58.58.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.222.222.194.126.58.58.58.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.222.222.194.126.58.58.58.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.222.222.194.126.58.58.58.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.222.222.194.126.58.58.58.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.222.222.194.126.58.58.58.7" class="ltx_td ltx_align_center">0.42</td>
<td id="A4.222.222.194.126.58.58.58.8" class="ltx_td ltx_align_center">9.09</td>
</tr>
<tr id="A4.223.223.195.127.59.59.59" class="ltx_tr">
<td id="A4.223.223.195.127.59.59.59.1" class="ltx_td ltx_align_left">RMU<math id="A4.223.223.195.127.59.59.59.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.223.223.195.127.59.59.59.1.m1.1a"><mo stretchy="false" id="A4.223.223.195.127.59.59.59.1.m1.1.1" xref="A4.223.223.195.127.59.59.59.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.223.223.195.127.59.59.59.1.m1.1b"><ci id="A4.223.223.195.127.59.59.59.1.m1.1.1.cmml" xref="A4.223.223.195.127.59.59.59.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.223.223.195.127.59.59.59.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (4-Bit)</td>
<td id="A4.223.223.195.127.59.59.59.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.223.223.195.127.59.59.59.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.223.223.195.127.59.59.59.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.223.223.195.127.59.59.59.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.223.223.195.127.59.59.59.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.223.223.195.127.59.59.59.7" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.223.223.195.127.59.59.59.8" class="ltx_td ltx_align_center">12.88</td>
</tr>
<tr id="A4.224.224.196.128.60.60.60" class="ltx_tr">
<td id="A4.224.224.196.128.60.60.60.1" class="ltx_td ltx_align_left">RMU<math id="A4.224.224.196.128.60.60.60.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.224.224.196.128.60.60.60.1.m1.1a"><mo stretchy="false" id="A4.224.224.196.128.60.60.60.1.m1.1.1" xref="A4.224.224.196.128.60.60.60.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.224.224.196.128.60.60.60.1.m1.1b"><ci id="A4.224.224.196.128.60.60.60.1.m1.1.1.cmml" xref="A4.224.224.196.128.60.60.60.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.224.224.196.128.60.60.60.1.m1.1c">\rightarrow</annotation></semantics></math>GPTQ (8-Bit)</td>
<td id="A4.224.224.196.128.60.60.60.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.224.224.196.128.60.60.60.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.224.224.196.128.60.60.60.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.224.224.196.128.60.60.60.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.224.224.196.128.60.60.60.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.224.224.196.128.60.60.60.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.224.224.196.128.60.60.60.8" class="ltx_td ltx_align_center">5.56</td>
</tr>
<tr id="A4.225.225.197.129.61.61.61" class="ltx_tr">
<td id="A4.225.225.197.129.61.61.61.1" class="ltx_td ltx_align_left">RMU<math id="A4.225.225.197.129.61.61.61.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.225.225.197.129.61.61.61.1.m1.1a"><mo stretchy="false" id="A4.225.225.197.129.61.61.61.1.m1.1.1" xref="A4.225.225.197.129.61.61.61.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.225.225.197.129.61.61.61.1.m1.1b"><ci id="A4.225.225.197.129.61.61.61.1.m1.1.1.cmml" xref="A4.225.225.197.129.61.61.61.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.225.225.197.129.61.61.61.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (2-Bit)</td>
<td id="A4.225.225.197.129.61.61.61.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.225.225.197.129.61.61.61.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.225.225.197.129.61.61.61.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.225.225.197.129.61.61.61.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.225.225.197.129.61.61.61.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.225.225.197.129.61.61.61.7" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.225.225.197.129.61.61.61.8" class="ltx_td ltx_align_center">1726409.12</td>
</tr>
<tr id="A4.226.226.198.130.62.62.62" class="ltx_tr">
<td id="A4.226.226.198.130.62.62.62.1" class="ltx_td ltx_align_left">RMU<math id="A4.226.226.198.130.62.62.62.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.226.226.198.130.62.62.62.1.m1.1a"><mo stretchy="false" id="A4.226.226.198.130.62.62.62.1.m1.1.1" xref="A4.226.226.198.130.62.62.62.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.226.226.198.130.62.62.62.1.m1.1b"><ci id="A4.226.226.198.130.62.62.62.1.m1.1.1.cmml" xref="A4.226.226.198.130.62.62.62.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.226.226.198.130.62.62.62.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (3-Bit)</td>
<td id="A4.226.226.198.130.62.62.62.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.226.226.198.130.62.62.62.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.226.226.198.130.62.62.62.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.226.226.198.130.62.62.62.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.226.226.198.130.62.62.62.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.226.226.198.130.62.62.62.7" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.226.226.198.130.62.62.62.8" class="ltx_td ltx_align_center">7.55</td>
</tr>
<tr id="A4.227.227.199.131.63.63.63" class="ltx_tr">
<td id="A4.227.227.199.131.63.63.63.1" class="ltx_td ltx_align_left">RMU<math id="A4.227.227.199.131.63.63.63.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.227.227.199.131.63.63.63.1.m1.1a"><mo stretchy="false" id="A4.227.227.199.131.63.63.63.1.m1.1.1" xref="A4.227.227.199.131.63.63.63.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.227.227.199.131.63.63.63.1.m1.1b"><ci id="A4.227.227.199.131.63.63.63.1.m1.1.1.cmml" xref="A4.227.227.199.131.63.63.63.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.227.227.199.131.63.63.63.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (4-Bit)</td>
<td id="A4.227.227.199.131.63.63.63.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.227.227.199.131.63.63.63.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.227.227.199.131.63.63.63.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.227.227.199.131.63.63.63.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.227.227.199.131.63.63.63.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.227.227.199.131.63.63.63.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.227.227.199.131.63.63.63.8" class="ltx_td ltx_align_center">5.95</td>
</tr>
<tr id="A4.228.228.200.132.64.64.64" class="ltx_tr">
<td id="A4.228.228.200.132.64.64.64.1" class="ltx_td ltx_align_left">RMU<math id="A4.228.228.200.132.64.64.64.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.228.228.200.132.64.64.64.1.m1.1a"><mo stretchy="false" id="A4.228.228.200.132.64.64.64.1.m1.1.1" xref="A4.228.228.200.132.64.64.64.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.228.228.200.132.64.64.64.1.m1.1b"><ci id="A4.228.228.200.132.64.64.64.1.m1.1.1.cmml" xref="A4.228.228.200.132.64.64.64.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.228.228.200.132.64.64.64.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (5-Bit)</td>
<td id="A4.228.228.200.132.64.64.64.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.228.228.200.132.64.64.64.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.228.228.200.132.64.64.64.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.228.228.200.132.64.64.64.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.228.228.200.132.64.64.64.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.228.228.200.132.64.64.64.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.228.228.200.132.64.64.64.8" class="ltx_td ltx_align_center">5.68</td>
</tr>
<tr id="A4.229.229.201.133.65.65.65" class="ltx_tr">
<td id="A4.229.229.201.133.65.65.65.1" class="ltx_td ltx_align_left">RMU<math id="A4.229.229.201.133.65.65.65.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.229.229.201.133.65.65.65.1.m1.1a"><mo stretchy="false" id="A4.229.229.201.133.65.65.65.1.m1.1.1" xref="A4.229.229.201.133.65.65.65.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.229.229.201.133.65.65.65.1.m1.1b"><ci id="A4.229.229.201.133.65.65.65.1.m1.1.1.cmml" xref="A4.229.229.201.133.65.65.65.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.229.229.201.133.65.65.65.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (6-Bit)</td>
<td id="A4.229.229.201.133.65.65.65.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.229.229.201.133.65.65.65.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.229.229.201.133.65.65.65.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.229.229.201.133.65.65.65.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.229.229.201.133.65.65.65.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.229.229.201.133.65.65.65.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.229.229.201.133.65.65.65.8" class="ltx_td ltx_align_center">5.62</td>
</tr>
<tr id="A4.230.230.202.134.66.66.66" class="ltx_tr">
<td id="A4.230.230.202.134.66.66.66.1" class="ltx_td ltx_align_left ltx_border_bb">RMU<math id="A4.230.230.202.134.66.66.66.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.230.230.202.134.66.66.66.1.m1.1a"><mo stretchy="false" id="A4.230.230.202.134.66.66.66.1.m1.1.1" xref="A4.230.230.202.134.66.66.66.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.230.230.202.134.66.66.66.1.m1.1b"><ci id="A4.230.230.202.134.66.66.66.1.m1.1.1.cmml" xref="A4.230.230.202.134.66.66.66.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.230.230.202.134.66.66.66.1.m1.1c">\rightarrow</annotation></semantics></math>AWQ (8-Bit)</td>
<td id="A4.230.230.202.134.66.66.66.2" class="ltx_td ltx_align_center ltx_border_bb">0.02</td>
<td id="A4.230.230.202.134.66.66.66.3" class="ltx_td ltx_align_center ltx_border_bb">0.02</td>
<td id="A4.230.230.202.134.66.66.66.4" class="ltx_td ltx_align_center ltx_border_bb">0.03</td>
<td id="A4.230.230.202.134.66.66.66.5" class="ltx_td ltx_align_center ltx_border_bb">8.25</td>
<td id="A4.230.230.202.134.66.66.66.6" class="ltx_td ltx_align_center ltx_border_bb">0.29</td>
<td id="A4.230.230.202.134.66.66.66.7" class="ltx_td ltx_align_center ltx_border_bb">0.57</td>
<td id="A4.230.230.202.134.66.66.66.8" class="ltx_td ltx_align_center ltx_border_bb">5.56</td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="A4.322.322.294.226.159.2.1" class="ltx_text" style="font-size:90%;">Table 11</span>: </span><span id="A4.232.232.204.136.68.1" class="ltx_text" style="font-size:90%;">Detailed results for unlearning <math id="A4.232.232.204.136.68.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.232.232.204.136.68.1.m1.1b"><mo stretchy="false" id="A4.232.232.204.136.68.1.m1.1.1" xref="A4.232.232.204.136.68.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.232.232.204.136.68.1.m1.1c"><ci id="A4.232.232.204.136.68.1.m1.1.1.cmml" xref="A4.232.232.204.136.68.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.232.232.204.136.68.1.m1.1d">\rightarrow</annotation></semantics></math> compression.</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="A4.322.322.294.226.158" class="ltx_table ltx_figure_panel">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="A4.298.298.270.202.134.66" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:890.4pt;vertical-align:-1.5pt;"><span class="ltx_transformed_inner" style="transform:translate(-94.0pt,192.7pt) scale(0.697541995667234,0.697541995667234) ;">
<table id="A4.298.298.270.202.134.66.66" class="ltx_tabular ltx_align_middle">
<tr id="A4.298.298.270.202.134.66.66.67" class="ltx_tr">
<td id="A4.298.298.270.202.134.66.66.67.1" class="ltx_td ltx_border_tt"></td>
<td id="A4.298.298.270.202.134.66.66.67.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="3">Editing</td>
<td id="A4.298.298.270.202.134.66.66.67.3" class="ltx_td ltx_align_center ltx_border_tt">Compression</td>
<td id="A4.298.298.270.202.134.66.66.67.4" class="ltx_td ltx_align_center ltx_border_tt">Unlearning</td>
<td id="A4.298.298.270.202.134.66.66.67.5" class="ltx_td ltx_align_center ltx_border_tt" colspan="2">Utility</td>
</tr>
<tr id="A4.298.298.270.202.134.66.66.68" class="ltx_tr">
<td id="A4.298.298.270.202.134.66.66.68.1" class="ltx_td"></td>
<td id="A4.298.298.270.202.134.66.66.68.2" class="ltx_td ltx_align_center ltx_border_t">Edit Success</td>
<td id="A4.298.298.270.202.134.66.66.68.3" class="ltx_td ltx_align_center ltx_border_t">Generalization</td>
<td id="A4.298.298.270.202.134.66.66.68.4" class="ltx_td ltx_align_center ltx_border_t">Locality</td>
<td id="A4.298.298.270.202.134.66.66.68.5" class="ltx_td ltx_align_center ltx_border_t">Avg. Bits</td>
<td id="A4.298.298.270.202.134.66.66.68.6" class="ltx_td ltx_align_center ltx_border_t">Avg. WMDP</td>
<td id="A4.298.298.270.202.134.66.66.68.7" class="ltx_td ltx_align_center ltx_border_t">MMLU</td>
<td id="A4.298.298.270.202.134.66.66.68.8" class="ltx_td ltx_align_center ltx_border_t">WikiText PPL</td>
</tr>
<tr id="A4.233.233.205.137.69.1.1.1" class="ltx_tr">
<td id="A4.233.233.205.137.69.1.1.1.1" class="ltx_td ltx_align_left ltx_border_t">SparseGPT (0.25) <math id="A4.233.233.205.137.69.1.1.1.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.233.233.205.137.69.1.1.1.1.m1.1a"><mo stretchy="false" id="A4.233.233.205.137.69.1.1.1.1.m1.1.1" xref="A4.233.233.205.137.69.1.1.1.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.233.233.205.137.69.1.1.1.1.m1.1b"><ci id="A4.233.233.205.137.69.1.1.1.1.m1.1.1.cmml" xref="A4.233.233.205.137.69.1.1.1.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.233.233.205.137.69.1.1.1.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.233.233.205.137.69.1.1.1.2" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.233.233.205.137.69.1.1.1.3" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.233.233.205.137.69.1.1.1.4" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.233.233.205.137.69.1.1.1.5" class="ltx_td ltx_align_center ltx_border_t">12.25</td>
<td id="A4.233.233.205.137.69.1.1.1.6" class="ltx_td ltx_align_center ltx_border_t">0.43</td>
<td id="A4.233.233.205.137.69.1.1.1.7" class="ltx_td ltx_align_center ltx_border_t">0.45</td>
<td id="A4.233.233.205.137.69.1.1.1.8" class="ltx_td ltx_align_center ltx_border_t">inf</td>
</tr>
<tr id="A4.234.234.206.138.70.2.2.2" class="ltx_tr">
<td id="A4.234.234.206.138.70.2.2.2.1" class="ltx_td ltx_align_left">SparseGPT (0.35) <math id="A4.234.234.206.138.70.2.2.2.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.234.234.206.138.70.2.2.2.1.m1.1a"><mo stretchy="false" id="A4.234.234.206.138.70.2.2.2.1.m1.1.1" xref="A4.234.234.206.138.70.2.2.2.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.234.234.206.138.70.2.2.2.1.m1.1b"><ci id="A4.234.234.206.138.70.2.2.2.1.m1.1.1.cmml" xref="A4.234.234.206.138.70.2.2.2.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.234.234.206.138.70.2.2.2.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.234.234.206.138.70.2.2.2.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.234.234.206.138.70.2.2.2.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.234.234.206.138.70.2.2.2.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.234.234.206.138.70.2.2.2.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.234.234.206.138.70.2.2.2.6" class="ltx_td ltx_align_center">0.34</td>
<td id="A4.234.234.206.138.70.2.2.2.7" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.234.234.206.138.70.2.2.2.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.235.235.207.139.71.3.3.3" class="ltx_tr">
<td id="A4.235.235.207.139.71.3.3.3.1" class="ltx_td ltx_align_left">SparseGPT (0.45) <math id="A4.235.235.207.139.71.3.3.3.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.235.235.207.139.71.3.3.3.1.m1.1a"><mo stretchy="false" id="A4.235.235.207.139.71.3.3.3.1.m1.1.1" xref="A4.235.235.207.139.71.3.3.3.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.235.235.207.139.71.3.3.3.1.m1.1b"><ci id="A4.235.235.207.139.71.3.3.3.1.m1.1.1.cmml" xref="A4.235.235.207.139.71.3.3.3.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.235.235.207.139.71.3.3.3.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.235.235.207.139.71.3.3.3.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.235.235.207.139.71.3.3.3.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.235.235.207.139.71.3.3.3.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.235.235.207.139.71.3.3.3.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.235.235.207.139.71.3.3.3.6" class="ltx_td ltx_align_center">0.31</td>
<td id="A4.235.235.207.139.71.3.3.3.7" class="ltx_td ltx_align_center">0.33</td>
<td id="A4.235.235.207.139.71.3.3.3.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.236.236.208.140.72.4.4.4" class="ltx_tr">
<td id="A4.236.236.208.140.72.4.4.4.1" class="ltx_td ltx_align_left">SparseGPT (0.55) <math id="A4.236.236.208.140.72.4.4.4.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.236.236.208.140.72.4.4.4.1.m1.1a"><mo stretchy="false" id="A4.236.236.208.140.72.4.4.4.1.m1.1.1" xref="A4.236.236.208.140.72.4.4.4.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.236.236.208.140.72.4.4.4.1.m1.1b"><ci id="A4.236.236.208.140.72.4.4.4.1.m1.1.1.cmml" xref="A4.236.236.208.140.72.4.4.4.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.236.236.208.140.72.4.4.4.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.236.236.208.140.72.4.4.4.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.236.236.208.140.72.4.4.4.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.236.236.208.140.72.4.4.4.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.236.236.208.140.72.4.4.4.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.236.236.208.140.72.4.4.4.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.236.236.208.140.72.4.4.4.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.236.236.208.140.72.4.4.4.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.237.237.209.141.73.5.5.5" class="ltx_tr">
<td id="A4.237.237.209.141.73.5.5.5.1" class="ltx_td ltx_align_left">SparseGPT (0.65) <math id="A4.237.237.209.141.73.5.5.5.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.237.237.209.141.73.5.5.5.1.m1.1a"><mo stretchy="false" id="A4.237.237.209.141.73.5.5.5.1.m1.1.1" xref="A4.237.237.209.141.73.5.5.5.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.237.237.209.141.73.5.5.5.1.m1.1b"><ci id="A4.237.237.209.141.73.5.5.5.1.m1.1.1.cmml" xref="A4.237.237.209.141.73.5.5.5.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.237.237.209.141.73.5.5.5.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.237.237.209.141.73.5.5.5.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.237.237.209.141.73.5.5.5.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.237.237.209.141.73.5.5.5.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.237.237.209.141.73.5.5.5.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.237.237.209.141.73.5.5.5.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.237.237.209.141.73.5.5.5.7" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.237.237.209.141.73.5.5.5.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.238.238.210.142.74.6.6.6" class="ltx_tr">
<td id="A4.238.238.210.142.74.6.6.6.1" class="ltx_td ltx_align_left">SparseGPT (0.75) <math id="A4.238.238.210.142.74.6.6.6.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.238.238.210.142.74.6.6.6.1.m1.1a"><mo stretchy="false" id="A4.238.238.210.142.74.6.6.6.1.m1.1.1" xref="A4.238.238.210.142.74.6.6.6.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.238.238.210.142.74.6.6.6.1.m1.1b"><ci id="A4.238.238.210.142.74.6.6.6.1.m1.1.1.cmml" xref="A4.238.238.210.142.74.6.6.6.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.238.238.210.142.74.6.6.6.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.238.238.210.142.74.6.6.6.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.238.238.210.142.74.6.6.6.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.238.238.210.142.74.6.6.6.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.238.238.210.142.74.6.6.6.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.238.238.210.142.74.6.6.6.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.238.238.210.142.74.6.6.6.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.238.238.210.142.74.6.6.6.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.239.239.211.143.75.7.7.7" class="ltx_tr">
<td id="A4.239.239.211.143.75.7.7.7.1" class="ltx_td ltx_align_left">SparseGPT (0.25) <math id="A4.239.239.211.143.75.7.7.7.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.239.239.211.143.75.7.7.7.1.m1.1a"><mo stretchy="false" id="A4.239.239.211.143.75.7.7.7.1.m1.1.1" xref="A4.239.239.211.143.75.7.7.7.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.239.239.211.143.75.7.7.7.1.m1.1b"><ci id="A4.239.239.211.143.75.7.7.7.1.m1.1.1.cmml" xref="A4.239.239.211.143.75.7.7.7.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.239.239.211.143.75.7.7.7.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.239.239.211.143.75.7.7.7.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.239.239.211.143.75.7.7.7.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.239.239.211.143.75.7.7.7.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.239.239.211.143.75.7.7.7.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.239.239.211.143.75.7.7.7.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.239.239.211.143.75.7.7.7.7" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.239.239.211.143.75.7.7.7.8" class="ltx_td ltx_align_center">13.32</td>
</tr>
<tr id="A4.240.240.212.144.76.8.8.8" class="ltx_tr">
<td id="A4.240.240.212.144.76.8.8.8.1" class="ltx_td ltx_align_left">SparseGPT (0.35) <math id="A4.240.240.212.144.76.8.8.8.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.240.240.212.144.76.8.8.8.1.m1.1a"><mo stretchy="false" id="A4.240.240.212.144.76.8.8.8.1.m1.1.1" xref="A4.240.240.212.144.76.8.8.8.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.240.240.212.144.76.8.8.8.1.m1.1b"><ci id="A4.240.240.212.144.76.8.8.8.1.m1.1.1.cmml" xref="A4.240.240.212.144.76.8.8.8.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.240.240.212.144.76.8.8.8.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.240.240.212.144.76.8.8.8.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.240.240.212.144.76.8.8.8.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.240.240.212.144.76.8.8.8.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.240.240.212.144.76.8.8.8.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.240.240.212.144.76.8.8.8.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.240.240.212.144.76.8.8.8.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.240.240.212.144.76.8.8.8.8" class="ltx_td ltx_align_center">2712.26</td>
</tr>
<tr id="A4.241.241.213.145.77.9.9.9" class="ltx_tr">
<td id="A4.241.241.213.145.77.9.9.9.1" class="ltx_td ltx_align_left">SparseGPT (0.45) <math id="A4.241.241.213.145.77.9.9.9.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.241.241.213.145.77.9.9.9.1.m1.1a"><mo stretchy="false" id="A4.241.241.213.145.77.9.9.9.1.m1.1.1" xref="A4.241.241.213.145.77.9.9.9.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.241.241.213.145.77.9.9.9.1.m1.1b"><ci id="A4.241.241.213.145.77.9.9.9.1.m1.1.1.cmml" xref="A4.241.241.213.145.77.9.9.9.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.241.241.213.145.77.9.9.9.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.241.241.213.145.77.9.9.9.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.241.241.213.145.77.9.9.9.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.241.241.213.145.77.9.9.9.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.241.241.213.145.77.9.9.9.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.241.241.213.145.77.9.9.9.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.241.241.213.145.77.9.9.9.7" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.241.241.213.145.77.9.9.9.8" class="ltx_td ltx_align_center">1.246813090597988e+19</td>
</tr>
<tr id="A4.242.242.214.146.78.10.10.10" class="ltx_tr">
<td id="A4.242.242.214.146.78.10.10.10.1" class="ltx_td ltx_align_left">SparseGPT (0.55) <math id="A4.242.242.214.146.78.10.10.10.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.242.242.214.146.78.10.10.10.1.m1.1a"><mo stretchy="false" id="A4.242.242.214.146.78.10.10.10.1.m1.1.1" xref="A4.242.242.214.146.78.10.10.10.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.242.242.214.146.78.10.10.10.1.m1.1b"><ci id="A4.242.242.214.146.78.10.10.10.1.m1.1.1.cmml" xref="A4.242.242.214.146.78.10.10.10.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.242.242.214.146.78.10.10.10.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.242.242.214.146.78.10.10.10.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.242.242.214.146.78.10.10.10.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.242.242.214.146.78.10.10.10.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.242.242.214.146.78.10.10.10.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.242.242.214.146.78.10.10.10.6" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.242.242.214.146.78.10.10.10.7" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.242.242.214.146.78.10.10.10.8" class="ltx_td ltx_align_center">9.35</td>
</tr>
<tr id="A4.243.243.215.147.79.11.11.11" class="ltx_tr">
<td id="A4.243.243.215.147.79.11.11.11.1" class="ltx_td ltx_align_left">SparseGPT (0.65) <math id="A4.243.243.215.147.79.11.11.11.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.243.243.215.147.79.11.11.11.1.m1.1a"><mo stretchy="false" id="A4.243.243.215.147.79.11.11.11.1.m1.1.1" xref="A4.243.243.215.147.79.11.11.11.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.243.243.215.147.79.11.11.11.1.m1.1b"><ci id="A4.243.243.215.147.79.11.11.11.1.m1.1.1.cmml" xref="A4.243.243.215.147.79.11.11.11.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.243.243.215.147.79.11.11.11.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.243.243.215.147.79.11.11.11.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.243.243.215.147.79.11.11.11.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.243.243.215.147.79.11.11.11.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.243.243.215.147.79.11.11.11.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.243.243.215.147.79.11.11.11.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.243.243.215.147.79.11.11.11.7" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.243.243.215.147.79.11.11.11.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.244.244.216.148.80.12.12.12" class="ltx_tr">
<td id="A4.244.244.216.148.80.12.12.12.1" class="ltx_td ltx_align_left">SparseGPT (0.75) <math id="A4.244.244.216.148.80.12.12.12.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.244.244.216.148.80.12.12.12.1.m1.1a"><mo stretchy="false" id="A4.244.244.216.148.80.12.12.12.1.m1.1.1" xref="A4.244.244.216.148.80.12.12.12.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.244.244.216.148.80.12.12.12.1.m1.1b"><ci id="A4.244.244.216.148.80.12.12.12.1.m1.1.1.cmml" xref="A4.244.244.216.148.80.12.12.12.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.244.244.216.148.80.12.12.12.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.244.244.216.148.80.12.12.12.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.244.244.216.148.80.12.12.12.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.244.244.216.148.80.12.12.12.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.244.244.216.148.80.12.12.12.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.244.244.216.148.80.12.12.12.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.244.244.216.148.80.12.12.12.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.244.244.216.148.80.12.12.12.8" class="ltx_td ltx_align_center">1437619072.0</td>
</tr>
<tr id="A4.245.245.217.149.81.13.13.13" class="ltx_tr">
<td id="A4.245.245.217.149.81.13.13.13.1" class="ltx_td ltx_align_left">SparseGPT (0.25) <math id="A4.245.245.217.149.81.13.13.13.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.245.245.217.149.81.13.13.13.1.m1.1a"><mo stretchy="false" id="A4.245.245.217.149.81.13.13.13.1.m1.1.1" xref="A4.245.245.217.149.81.13.13.13.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.245.245.217.149.81.13.13.13.1.m1.1b"><ci id="A4.245.245.217.149.81.13.13.13.1.m1.1.1.cmml" xref="A4.245.245.217.149.81.13.13.13.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.245.245.217.149.81.13.13.13.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.245.245.217.149.81.13.13.13.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.245.245.217.149.81.13.13.13.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.245.245.217.149.81.13.13.13.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.245.245.217.149.81.13.13.13.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.245.245.217.149.81.13.13.13.6" class="ltx_td ltx_align_center">0.31</td>
<td id="A4.245.245.217.149.81.13.13.13.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.245.245.217.149.81.13.13.13.8" class="ltx_td ltx_align_center">5.92</td>
</tr>
<tr id="A4.246.246.218.150.82.14.14.14" class="ltx_tr">
<td id="A4.246.246.218.150.82.14.14.14.1" class="ltx_td ltx_align_left">SparseGPT (0.35) <math id="A4.246.246.218.150.82.14.14.14.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.246.246.218.150.82.14.14.14.1.m1.1a"><mo stretchy="false" id="A4.246.246.218.150.82.14.14.14.1.m1.1.1" xref="A4.246.246.218.150.82.14.14.14.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.246.246.218.150.82.14.14.14.1.m1.1b"><ci id="A4.246.246.218.150.82.14.14.14.1.m1.1.1.cmml" xref="A4.246.246.218.150.82.14.14.14.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.246.246.218.150.82.14.14.14.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.246.246.218.150.82.14.14.14.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.246.246.218.150.82.14.14.14.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.246.246.218.150.82.14.14.14.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.246.246.218.150.82.14.14.14.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.246.246.218.150.82.14.14.14.6" class="ltx_td ltx_align_center">0.31</td>
<td id="A4.246.246.218.150.82.14.14.14.7" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.246.246.218.150.82.14.14.14.8" class="ltx_td ltx_align_center">6.35</td>
</tr>
<tr id="A4.247.247.219.151.83.15.15.15" class="ltx_tr">
<td id="A4.247.247.219.151.83.15.15.15.1" class="ltx_td ltx_align_left">SparseGPT (0.45) <math id="A4.247.247.219.151.83.15.15.15.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.247.247.219.151.83.15.15.15.1.m1.1a"><mo stretchy="false" id="A4.247.247.219.151.83.15.15.15.1.m1.1.1" xref="A4.247.247.219.151.83.15.15.15.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.247.247.219.151.83.15.15.15.1.m1.1b"><ci id="A4.247.247.219.151.83.15.15.15.1.m1.1.1.cmml" xref="A4.247.247.219.151.83.15.15.15.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.247.247.219.151.83.15.15.15.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.247.247.219.151.83.15.15.15.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.247.247.219.151.83.15.15.15.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.247.247.219.151.83.15.15.15.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.247.247.219.151.83.15.15.15.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.247.247.219.151.83.15.15.15.6" class="ltx_td ltx_align_center">0.31</td>
<td id="A4.247.247.219.151.83.15.15.15.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.247.247.219.151.83.15.15.15.8" class="ltx_td ltx_align_center">7.86</td>
</tr>
<tr id="A4.248.248.220.152.84.16.16.16" class="ltx_tr">
<td id="A4.248.248.220.152.84.16.16.16.1" class="ltx_td ltx_align_left">SparseGPT (0.55) <math id="A4.248.248.220.152.84.16.16.16.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.248.248.220.152.84.16.16.16.1.m1.1a"><mo stretchy="false" id="A4.248.248.220.152.84.16.16.16.1.m1.1.1" xref="A4.248.248.220.152.84.16.16.16.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.248.248.220.152.84.16.16.16.1.m1.1b"><ci id="A4.248.248.220.152.84.16.16.16.1.m1.1.1.cmml" xref="A4.248.248.220.152.84.16.16.16.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.248.248.220.152.84.16.16.16.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.248.248.220.152.84.16.16.16.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.248.248.220.152.84.16.16.16.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.248.248.220.152.84.16.16.16.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.248.248.220.152.84.16.16.16.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.248.248.220.152.84.16.16.16.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.248.248.220.152.84.16.16.16.7" class="ltx_td ltx_align_center">0.41</td>
<td id="A4.248.248.220.152.84.16.16.16.8" class="ltx_td ltx_align_center">10.39</td>
</tr>
<tr id="A4.249.249.221.153.85.17.17.17" class="ltx_tr">
<td id="A4.249.249.221.153.85.17.17.17.1" class="ltx_td ltx_align_left">SparseGPT (0.65) <math id="A4.249.249.221.153.85.17.17.17.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.249.249.221.153.85.17.17.17.1.m1.1a"><mo stretchy="false" id="A4.249.249.221.153.85.17.17.17.1.m1.1.1" xref="A4.249.249.221.153.85.17.17.17.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.249.249.221.153.85.17.17.17.1.m1.1b"><ci id="A4.249.249.221.153.85.17.17.17.1.m1.1.1.cmml" xref="A4.249.249.221.153.85.17.17.17.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.249.249.221.153.85.17.17.17.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.249.249.221.153.85.17.17.17.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.249.249.221.153.85.17.17.17.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.249.249.221.153.85.17.17.17.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.249.249.221.153.85.17.17.17.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.249.249.221.153.85.17.17.17.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.249.249.221.153.85.17.17.17.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.249.249.221.153.85.17.17.17.8" class="ltx_td ltx_align_center">30.26</td>
</tr>
<tr id="A4.250.250.222.154.86.18.18.18" class="ltx_tr">
<td id="A4.250.250.222.154.86.18.18.18.1" class="ltx_td ltx_align_left">SparseGPT (0.75) <math id="A4.250.250.222.154.86.18.18.18.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.250.250.222.154.86.18.18.18.1.m1.1a"><mo stretchy="false" id="A4.250.250.222.154.86.18.18.18.1.m1.1.1" xref="A4.250.250.222.154.86.18.18.18.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.250.250.222.154.86.18.18.18.1.m1.1b"><ci id="A4.250.250.222.154.86.18.18.18.1.m1.1.1.cmml" xref="A4.250.250.222.154.86.18.18.18.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.250.250.222.154.86.18.18.18.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.250.250.222.154.86.18.18.18.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.250.250.222.154.86.18.18.18.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.250.250.222.154.86.18.18.18.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.250.250.222.154.86.18.18.18.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.250.250.222.154.86.18.18.18.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.250.250.222.154.86.18.18.18.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.250.250.222.154.86.18.18.18.8" class="ltx_td ltx_align_center">97.03</td>
</tr>
<tr id="A4.251.251.223.155.87.19.19.19" class="ltx_tr">
<td id="A4.251.251.223.155.87.19.19.19.1" class="ltx_td ltx_align_left">
<span id="A4.251.251.223.155.87.19.19.19.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

Wanda (0.25) <math id="A4.251.251.223.155.87.19.19.19.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.251.251.223.155.87.19.19.19.1.m1.1a"><mo stretchy="false" id="A4.251.251.223.155.87.19.19.19.1.m1.1.1" xref="A4.251.251.223.155.87.19.19.19.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.251.251.223.155.87.19.19.19.1.m1.1b"><ci id="A4.251.251.223.155.87.19.19.19.1.m1.1.1.cmml" xref="A4.251.251.223.155.87.19.19.19.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.251.251.223.155.87.19.19.19.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.251.251.223.155.87.19.19.19.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.251.251.223.155.87.19.19.19.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.251.251.223.155.87.19.19.19.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.251.251.223.155.87.19.19.19.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.251.251.223.155.87.19.19.19.6" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.251.251.223.155.87.19.19.19.7" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.251.251.223.155.87.19.19.19.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.252.252.224.156.88.20.20.20" class="ltx_tr">
<td id="A4.252.252.224.156.88.20.20.20.1" class="ltx_td ltx_align_left">Wanda (0.35) <math id="A4.252.252.224.156.88.20.20.20.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.252.252.224.156.88.20.20.20.1.m1.1a"><mo stretchy="false" id="A4.252.252.224.156.88.20.20.20.1.m1.1.1" xref="A4.252.252.224.156.88.20.20.20.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.252.252.224.156.88.20.20.20.1.m1.1b"><ci id="A4.252.252.224.156.88.20.20.20.1.m1.1.1.cmml" xref="A4.252.252.224.156.88.20.20.20.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.252.252.224.156.88.20.20.20.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.252.252.224.156.88.20.20.20.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.252.252.224.156.88.20.20.20.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.252.252.224.156.88.20.20.20.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.252.252.224.156.88.20.20.20.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.252.252.224.156.88.20.20.20.6" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.252.252.224.156.88.20.20.20.7" class="ltx_td ltx_align_center">0.51</td>
<td id="A4.252.252.224.156.88.20.20.20.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.253.253.225.157.89.21.21.21" class="ltx_tr">
<td id="A4.253.253.225.157.89.21.21.21.1" class="ltx_td ltx_align_left">Wanda (0.45) <math id="A4.253.253.225.157.89.21.21.21.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.253.253.225.157.89.21.21.21.1.m1.1a"><mo stretchy="false" id="A4.253.253.225.157.89.21.21.21.1.m1.1.1" xref="A4.253.253.225.157.89.21.21.21.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.253.253.225.157.89.21.21.21.1.m1.1b"><ci id="A4.253.253.225.157.89.21.21.21.1.m1.1.1.cmml" xref="A4.253.253.225.157.89.21.21.21.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.253.253.225.157.89.21.21.21.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.253.253.225.157.89.21.21.21.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.253.253.225.157.89.21.21.21.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.253.253.225.157.89.21.21.21.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.253.253.225.157.89.21.21.21.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.253.253.225.157.89.21.21.21.6" class="ltx_td ltx_align_center">0.41</td>
<td id="A4.253.253.225.157.89.21.21.21.7" class="ltx_td ltx_align_center">0.43</td>
<td id="A4.253.253.225.157.89.21.21.21.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.254.254.226.158.90.22.22.22" class="ltx_tr">
<td id="A4.254.254.226.158.90.22.22.22.1" class="ltx_td ltx_align_left">Wanda (0.55) <math id="A4.254.254.226.158.90.22.22.22.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.254.254.226.158.90.22.22.22.1.m1.1a"><mo stretchy="false" id="A4.254.254.226.158.90.22.22.22.1.m1.1.1" xref="A4.254.254.226.158.90.22.22.22.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.254.254.226.158.90.22.22.22.1.m1.1b"><ci id="A4.254.254.226.158.90.22.22.22.1.m1.1.1.cmml" xref="A4.254.254.226.158.90.22.22.22.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.254.254.226.158.90.22.22.22.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.254.254.226.158.90.22.22.22.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.254.254.226.158.90.22.22.22.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.254.254.226.158.90.22.22.22.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.254.254.226.158.90.22.22.22.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.254.254.226.158.90.22.22.22.6" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.254.254.226.158.90.22.22.22.7" class="ltx_td ltx_align_center">0.34</td>
<td id="A4.254.254.226.158.90.22.22.22.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.255.255.227.159.91.23.23.23" class="ltx_tr">
<td id="A4.255.255.227.159.91.23.23.23.1" class="ltx_td ltx_align_left">Wanda (0.65) <math id="A4.255.255.227.159.91.23.23.23.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.255.255.227.159.91.23.23.23.1.m1.1a"><mo stretchy="false" id="A4.255.255.227.159.91.23.23.23.1.m1.1.1" xref="A4.255.255.227.159.91.23.23.23.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.255.255.227.159.91.23.23.23.1.m1.1b"><ci id="A4.255.255.227.159.91.23.23.23.1.m1.1.1.cmml" xref="A4.255.255.227.159.91.23.23.23.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.255.255.227.159.91.23.23.23.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.255.255.227.159.91.23.23.23.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.255.255.227.159.91.23.23.23.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.255.255.227.159.91.23.23.23.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.255.255.227.159.91.23.23.23.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.255.255.227.159.91.23.23.23.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.255.255.227.159.91.23.23.23.7" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.255.255.227.159.91.23.23.23.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.256.256.228.160.92.24.24.24" class="ltx_tr">
<td id="A4.256.256.228.160.92.24.24.24.1" class="ltx_td ltx_align_left">Wanda (0.75) <math id="A4.256.256.228.160.92.24.24.24.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.256.256.228.160.92.24.24.24.1.m1.1a"><mo stretchy="false" id="A4.256.256.228.160.92.24.24.24.1.m1.1.1" xref="A4.256.256.228.160.92.24.24.24.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.256.256.228.160.92.24.24.24.1.m1.1b"><ci id="A4.256.256.228.160.92.24.24.24.1.m1.1.1.cmml" xref="A4.256.256.228.160.92.24.24.24.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.256.256.228.160.92.24.24.24.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.256.256.228.160.92.24.24.24.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.256.256.228.160.92.24.24.24.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.256.256.228.160.92.24.24.24.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.256.256.228.160.92.24.24.24.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.256.256.228.160.92.24.24.24.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.256.256.228.160.92.24.24.24.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.256.256.228.160.92.24.24.24.8" class="ltx_td ltx_align_center">8.12597183809491e+36</td>
</tr>
<tr id="A4.257.257.229.161.93.25.25.25" class="ltx_tr">
<td id="A4.257.257.229.161.93.25.25.25.1" class="ltx_td ltx_align_left">Wanda (0.25) <math id="A4.257.257.229.161.93.25.25.25.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.257.257.229.161.93.25.25.25.1.m1.1a"><mo stretchy="false" id="A4.257.257.229.161.93.25.25.25.1.m1.1.1" xref="A4.257.257.229.161.93.25.25.25.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.257.257.229.161.93.25.25.25.1.m1.1b"><ci id="A4.257.257.229.161.93.25.25.25.1.m1.1.1.cmml" xref="A4.257.257.229.161.93.25.25.25.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.257.257.229.161.93.25.25.25.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.257.257.229.161.93.25.25.25.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.257.257.229.161.93.25.25.25.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.257.257.229.161.93.25.25.25.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.257.257.229.161.93.25.25.25.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.257.257.229.161.93.25.25.25.6" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.257.257.229.161.93.25.25.25.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.257.257.229.161.93.25.25.25.8" class="ltx_td ltx_align_center">13.26</td>
</tr>
<tr id="A4.258.258.230.162.94.26.26.26" class="ltx_tr">
<td id="A4.258.258.230.162.94.26.26.26.1" class="ltx_td ltx_align_left">Wanda (0.35) <math id="A4.258.258.230.162.94.26.26.26.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.258.258.230.162.94.26.26.26.1.m1.1a"><mo stretchy="false" id="A4.258.258.230.162.94.26.26.26.1.m1.1.1" xref="A4.258.258.230.162.94.26.26.26.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.258.258.230.162.94.26.26.26.1.m1.1b"><ci id="A4.258.258.230.162.94.26.26.26.1.m1.1.1.cmml" xref="A4.258.258.230.162.94.26.26.26.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.258.258.230.162.94.26.26.26.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.258.258.230.162.94.26.26.26.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.258.258.230.162.94.26.26.26.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.258.258.230.162.94.26.26.26.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.258.258.230.162.94.26.26.26.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.258.258.230.162.94.26.26.26.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.258.258.230.162.94.26.26.26.7" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.258.258.230.162.94.26.26.26.8" class="ltx_td ltx_align_center">57480503296.0</td>
</tr>
<tr id="A4.259.259.231.163.95.27.27.27" class="ltx_tr">
<td id="A4.259.259.231.163.95.27.27.27.1" class="ltx_td ltx_align_left">Wanda (0.45) <math id="A4.259.259.231.163.95.27.27.27.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.259.259.231.163.95.27.27.27.1.m1.1a"><mo stretchy="false" id="A4.259.259.231.163.95.27.27.27.1.m1.1.1" xref="A4.259.259.231.163.95.27.27.27.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.259.259.231.163.95.27.27.27.1.m1.1b"><ci id="A4.259.259.231.163.95.27.27.27.1.m1.1.1.cmml" xref="A4.259.259.231.163.95.27.27.27.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.259.259.231.163.95.27.27.27.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.259.259.231.163.95.27.27.27.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.259.259.231.163.95.27.27.27.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.259.259.231.163.95.27.27.27.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.259.259.231.163.95.27.27.27.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.259.259.231.163.95.27.27.27.6" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.259.259.231.163.95.27.27.27.7" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.259.259.231.163.95.27.27.27.8" class="ltx_td ltx_align_center">49.77</td>
</tr>
<tr id="A4.260.260.232.164.96.28.28.28" class="ltx_tr">
<td id="A4.260.260.232.164.96.28.28.28.1" class="ltx_td ltx_align_left">Wanda (0.55) <math id="A4.260.260.232.164.96.28.28.28.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.260.260.232.164.96.28.28.28.1.m1.1a"><mo stretchy="false" id="A4.260.260.232.164.96.28.28.28.1.m1.1.1" xref="A4.260.260.232.164.96.28.28.28.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.260.260.232.164.96.28.28.28.1.m1.1b"><ci id="A4.260.260.232.164.96.28.28.28.1.m1.1.1.cmml" xref="A4.260.260.232.164.96.28.28.28.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.260.260.232.164.96.28.28.28.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.260.260.232.164.96.28.28.28.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.260.260.232.164.96.28.28.28.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.260.260.232.164.96.28.28.28.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.260.260.232.164.96.28.28.28.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.260.260.232.164.96.28.28.28.6" class="ltx_td ltx_align_center">0.39</td>
<td id="A4.260.260.232.164.96.28.28.28.7" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.260.260.232.164.96.28.28.28.8" class="ltx_td ltx_align_center">121.07</td>
</tr>
<tr id="A4.261.261.233.165.97.29.29.29" class="ltx_tr">
<td id="A4.261.261.233.165.97.29.29.29.1" class="ltx_td ltx_align_left">Wanda (0.65) <math id="A4.261.261.233.165.97.29.29.29.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.261.261.233.165.97.29.29.29.1.m1.1a"><mo stretchy="false" id="A4.261.261.233.165.97.29.29.29.1.m1.1.1" xref="A4.261.261.233.165.97.29.29.29.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.261.261.233.165.97.29.29.29.1.m1.1b"><ci id="A4.261.261.233.165.97.29.29.29.1.m1.1.1.cmml" xref="A4.261.261.233.165.97.29.29.29.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.261.261.233.165.97.29.29.29.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.261.261.233.165.97.29.29.29.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.261.261.233.165.97.29.29.29.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.261.261.233.165.97.29.29.29.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.261.261.233.165.97.29.29.29.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.261.261.233.165.97.29.29.29.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.261.261.233.165.97.29.29.29.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.261.261.233.165.97.29.29.29.8" class="ltx_td ltx_align_center">16268.01</td>
</tr>
<tr id="A4.262.262.234.166.98.30.30.30" class="ltx_tr">
<td id="A4.262.262.234.166.98.30.30.30.1" class="ltx_td ltx_align_left">Wanda (0.75) <math id="A4.262.262.234.166.98.30.30.30.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.262.262.234.166.98.30.30.30.1.m1.1a"><mo stretchy="false" id="A4.262.262.234.166.98.30.30.30.1.m1.1.1" xref="A4.262.262.234.166.98.30.30.30.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.262.262.234.166.98.30.30.30.1.m1.1b"><ci id="A4.262.262.234.166.98.30.30.30.1.m1.1.1.cmml" xref="A4.262.262.234.166.98.30.30.30.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.262.262.234.166.98.30.30.30.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.262.262.234.166.98.30.30.30.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.262.262.234.166.98.30.30.30.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.262.262.234.166.98.30.30.30.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.262.262.234.166.98.30.30.30.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.262.262.234.166.98.30.30.30.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.262.262.234.166.98.30.30.30.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.262.262.234.166.98.30.30.30.8" class="ltx_td ltx_align_center">10226426.0</td>
</tr>
<tr id="A4.263.263.235.167.99.31.31.31" class="ltx_tr">
<td id="A4.263.263.235.167.99.31.31.31.1" class="ltx_td ltx_align_left">Wanda (0.25) <math id="A4.263.263.235.167.99.31.31.31.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.263.263.235.167.99.31.31.31.1.m1.1a"><mo stretchy="false" id="A4.263.263.235.167.99.31.31.31.1.m1.1.1" xref="A4.263.263.235.167.99.31.31.31.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.263.263.235.167.99.31.31.31.1.m1.1b"><ci id="A4.263.263.235.167.99.31.31.31.1.m1.1.1.cmml" xref="A4.263.263.235.167.99.31.31.31.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.263.263.235.167.99.31.31.31.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.263.263.235.167.99.31.31.31.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.263.263.235.167.99.31.31.31.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.263.263.235.167.99.31.31.31.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.263.263.235.167.99.31.31.31.5" class="ltx_td ltx_align_center">12.25</td>
<td id="A4.263.263.235.167.99.31.31.31.6" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.263.263.235.167.99.31.31.31.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.263.263.235.167.99.31.31.31.8" class="ltx_td ltx_align_center">5.86</td>
</tr>
<tr id="A4.264.264.236.168.100.32.32.32" class="ltx_tr">
<td id="A4.264.264.236.168.100.32.32.32.1" class="ltx_td ltx_align_left">Wanda (0.35) <math id="A4.264.264.236.168.100.32.32.32.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.264.264.236.168.100.32.32.32.1.m1.1a"><mo stretchy="false" id="A4.264.264.236.168.100.32.32.32.1.m1.1.1" xref="A4.264.264.236.168.100.32.32.32.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.264.264.236.168.100.32.32.32.1.m1.1b"><ci id="A4.264.264.236.168.100.32.32.32.1.m1.1.1.cmml" xref="A4.264.264.236.168.100.32.32.32.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.264.264.236.168.100.32.32.32.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.264.264.236.168.100.32.32.32.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.264.264.236.168.100.32.32.32.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.264.264.236.168.100.32.32.32.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.264.264.236.168.100.32.32.32.5" class="ltx_td ltx_align_center">10.75</td>
<td id="A4.264.264.236.168.100.32.32.32.6" class="ltx_td ltx_align_center">0.34</td>
<td id="A4.264.264.236.168.100.32.32.32.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.264.264.236.168.100.32.32.32.8" class="ltx_td ltx_align_center">6.33</td>
</tr>
<tr id="A4.265.265.237.169.101.33.33.33" class="ltx_tr">
<td id="A4.265.265.237.169.101.33.33.33.1" class="ltx_td ltx_align_left">Wanda (0.45) <math id="A4.265.265.237.169.101.33.33.33.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.265.265.237.169.101.33.33.33.1.m1.1a"><mo stretchy="false" id="A4.265.265.237.169.101.33.33.33.1.m1.1.1" xref="A4.265.265.237.169.101.33.33.33.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.265.265.237.169.101.33.33.33.1.m1.1b"><ci id="A4.265.265.237.169.101.33.33.33.1.m1.1.1.cmml" xref="A4.265.265.237.169.101.33.33.33.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.265.265.237.169.101.33.33.33.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.265.265.237.169.101.33.33.33.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.265.265.237.169.101.33.33.33.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.265.265.237.169.101.33.33.33.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.265.265.237.169.101.33.33.33.5" class="ltx_td ltx_align_center">9.25</td>
<td id="A4.265.265.237.169.101.33.33.33.6" class="ltx_td ltx_align_center">0.39</td>
<td id="A4.265.265.237.169.101.33.33.33.7" class="ltx_td ltx_align_center">0.52</td>
<td id="A4.265.265.237.169.101.33.33.33.8" class="ltx_td ltx_align_center">7.55</td>
</tr>
<tr id="A4.266.266.238.170.102.34.34.34" class="ltx_tr">
<td id="A4.266.266.238.170.102.34.34.34.1" class="ltx_td ltx_align_left">Wanda (0.55) <math id="A4.266.266.238.170.102.34.34.34.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.266.266.238.170.102.34.34.34.1.m1.1a"><mo stretchy="false" id="A4.266.266.238.170.102.34.34.34.1.m1.1.1" xref="A4.266.266.238.170.102.34.34.34.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.266.266.238.170.102.34.34.34.1.m1.1b"><ci id="A4.266.266.238.170.102.34.34.34.1.m1.1.1.cmml" xref="A4.266.266.238.170.102.34.34.34.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.266.266.238.170.102.34.34.34.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.266.266.238.170.102.34.34.34.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.266.266.238.170.102.34.34.34.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.266.266.238.170.102.34.34.34.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.266.266.238.170.102.34.34.34.5" class="ltx_td ltx_align_center">7.75</td>
<td id="A4.266.266.238.170.102.34.34.34.6" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.266.266.238.170.102.34.34.34.7" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.266.266.238.170.102.34.34.34.8" class="ltx_td ltx_align_center">12.58</td>
</tr>
<tr id="A4.267.267.239.171.103.35.35.35" class="ltx_tr">
<td id="A4.267.267.239.171.103.35.35.35.1" class="ltx_td ltx_align_left">Wanda (0.65) <math id="A4.267.267.239.171.103.35.35.35.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.267.267.239.171.103.35.35.35.1.m1.1a"><mo stretchy="false" id="A4.267.267.239.171.103.35.35.35.1.m1.1.1" xref="A4.267.267.239.171.103.35.35.35.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.267.267.239.171.103.35.35.35.1.m1.1b"><ci id="A4.267.267.239.171.103.35.35.35.1.m1.1.1.cmml" xref="A4.267.267.239.171.103.35.35.35.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.267.267.239.171.103.35.35.35.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.267.267.239.171.103.35.35.35.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.267.267.239.171.103.35.35.35.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.267.267.239.171.103.35.35.35.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.267.267.239.171.103.35.35.35.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.267.267.239.171.103.35.35.35.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.267.267.239.171.103.35.35.35.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.267.267.239.171.103.35.35.35.8" class="ltx_td ltx_align_center">45.53</td>
</tr>
<tr id="A4.268.268.240.172.104.36.36.36" class="ltx_tr">
<td id="A4.268.268.240.172.104.36.36.36.1" class="ltx_td ltx_align_left">Wanda (0.75) <math id="A4.268.268.240.172.104.36.36.36.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.268.268.240.172.104.36.36.36.1.m1.1a"><mo stretchy="false" id="A4.268.268.240.172.104.36.36.36.1.m1.1.1" xref="A4.268.268.240.172.104.36.36.36.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.268.268.240.172.104.36.36.36.1.m1.1b"><ci id="A4.268.268.240.172.104.36.36.36.1.m1.1.1.cmml" xref="A4.268.268.240.172.104.36.36.36.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.268.268.240.172.104.36.36.36.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.268.268.240.172.104.36.36.36.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.268.268.240.172.104.36.36.36.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.268.268.240.172.104.36.36.36.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.268.268.240.172.104.36.36.36.5" class="ltx_td ltx_align_center">4.75</td>
<td id="A4.268.268.240.172.104.36.36.36.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.268.268.240.172.104.36.36.36.7" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.268.268.240.172.104.36.36.36.8" class="ltx_td ltx_align_center">391.87</td>
</tr>
<tr id="A4.269.269.241.173.105.37.37.37" class="ltx_tr">
<td id="A4.269.269.241.173.105.37.37.37.1" class="ltx_td ltx_align_left">
<span id="A4.269.269.241.173.105.37.37.37.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

GPTQ (2-Bit) <math id="A4.269.269.241.173.105.37.37.37.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.269.269.241.173.105.37.37.37.1.m1.1a"><mo stretchy="false" id="A4.269.269.241.173.105.37.37.37.1.m1.1.1" xref="A4.269.269.241.173.105.37.37.37.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.269.269.241.173.105.37.37.37.1.m1.1b"><ci id="A4.269.269.241.173.105.37.37.37.1.m1.1.1.cmml" xref="A4.269.269.241.173.105.37.37.37.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.269.269.241.173.105.37.37.37.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.269.269.241.173.105.37.37.37.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.269.269.241.173.105.37.37.37.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.269.269.241.173.105.37.37.37.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.269.269.241.173.105.37.37.37.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.269.269.241.173.105.37.37.37.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.269.269.241.173.105.37.37.37.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.269.269.241.173.105.37.37.37.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.270.270.242.174.106.38.38.38" class="ltx_tr">
<td id="A4.270.270.242.174.106.38.38.38.1" class="ltx_td ltx_align_left">GPTQ (3-Bit) <math id="A4.270.270.242.174.106.38.38.38.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.270.270.242.174.106.38.38.38.1.m1.1a"><mo stretchy="false" id="A4.270.270.242.174.106.38.38.38.1.m1.1.1" xref="A4.270.270.242.174.106.38.38.38.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.270.270.242.174.106.38.38.38.1.m1.1b"><ci id="A4.270.270.242.174.106.38.38.38.1.m1.1.1.cmml" xref="A4.270.270.242.174.106.38.38.38.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.270.270.242.174.106.38.38.38.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.270.270.242.174.106.38.38.38.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.270.270.242.174.106.38.38.38.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.270.270.242.174.106.38.38.38.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.270.270.242.174.106.38.38.38.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.270.270.242.174.106.38.38.38.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.270.270.242.174.106.38.38.38.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.270.270.242.174.106.38.38.38.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.271.271.243.175.107.39.39.39" class="ltx_tr">
<td id="A4.271.271.243.175.107.39.39.39.1" class="ltx_td ltx_align_left">GPTQ (4-Bit) <math id="A4.271.271.243.175.107.39.39.39.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.271.271.243.175.107.39.39.39.1.m1.1a"><mo stretchy="false" id="A4.271.271.243.175.107.39.39.39.1.m1.1.1" xref="A4.271.271.243.175.107.39.39.39.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.271.271.243.175.107.39.39.39.1.m1.1b"><ci id="A4.271.271.243.175.107.39.39.39.1.m1.1.1.cmml" xref="A4.271.271.243.175.107.39.39.39.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.271.271.243.175.107.39.39.39.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.271.271.243.175.107.39.39.39.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.271.271.243.175.107.39.39.39.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.271.271.243.175.107.39.39.39.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.271.271.243.175.107.39.39.39.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.271.271.243.175.107.39.39.39.6" class="ltx_td ltx_align_center">0.34</td>
<td id="A4.271.271.243.175.107.39.39.39.7" class="ltx_td ltx_align_center">0.38</td>
<td id="A4.271.271.243.175.107.39.39.39.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.272.272.244.176.108.40.40.40" class="ltx_tr">
<td id="A4.272.272.244.176.108.40.40.40.1" class="ltx_td ltx_align_left">GPTQ (8-Bit) <math id="A4.272.272.244.176.108.40.40.40.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.272.272.244.176.108.40.40.40.1.m1.1a"><mo stretchy="false" id="A4.272.272.244.176.108.40.40.40.1.m1.1.1" xref="A4.272.272.244.176.108.40.40.40.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.272.272.244.176.108.40.40.40.1.m1.1b"><ci id="A4.272.272.244.176.108.40.40.40.1.m1.1.1.cmml" xref="A4.272.272.244.176.108.40.40.40.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.272.272.244.176.108.40.40.40.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.272.272.244.176.108.40.40.40.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.272.272.244.176.108.40.40.40.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.272.272.244.176.108.40.40.40.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.272.272.244.176.108.40.40.40.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.272.272.244.176.108.40.40.40.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.272.272.244.176.108.40.40.40.7" class="ltx_td ltx_align_center">0.5</td>
<td id="A4.272.272.244.176.108.40.40.40.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.273.273.245.177.109.41.41.41" class="ltx_tr">
<td id="A4.273.273.245.177.109.41.41.41.1" class="ltx_td ltx_align_left">GPTQ (2-Bit) <math id="A4.273.273.245.177.109.41.41.41.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.273.273.245.177.109.41.41.41.1.m1.1a"><mo stretchy="false" id="A4.273.273.245.177.109.41.41.41.1.m1.1.1" xref="A4.273.273.245.177.109.41.41.41.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.273.273.245.177.109.41.41.41.1.m1.1b"><ci id="A4.273.273.245.177.109.41.41.41.1.m1.1.1.cmml" xref="A4.273.273.245.177.109.41.41.41.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.273.273.245.177.109.41.41.41.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.273.273.245.177.109.41.41.41.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.273.273.245.177.109.41.41.41.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.273.273.245.177.109.41.41.41.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.273.273.245.177.109.41.41.41.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.273.273.245.177.109.41.41.41.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.273.273.245.177.109.41.41.41.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.273.273.245.177.109.41.41.41.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.274.274.246.178.110.42.42.42" class="ltx_tr">
<td id="A4.274.274.246.178.110.42.42.42.1" class="ltx_td ltx_align_left">GPTQ (3-Bit) <math id="A4.274.274.246.178.110.42.42.42.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.274.274.246.178.110.42.42.42.1.m1.1a"><mo stretchy="false" id="A4.274.274.246.178.110.42.42.42.1.m1.1.1" xref="A4.274.274.246.178.110.42.42.42.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.274.274.246.178.110.42.42.42.1.m1.1b"><ci id="A4.274.274.246.178.110.42.42.42.1.m1.1.1.cmml" xref="A4.274.274.246.178.110.42.42.42.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.274.274.246.178.110.42.42.42.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.274.274.246.178.110.42.42.42.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.274.274.246.178.110.42.42.42.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.274.274.246.178.110.42.42.42.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.274.274.246.178.110.42.42.42.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.274.274.246.178.110.42.42.42.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.274.274.246.178.110.42.42.42.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.274.274.246.178.110.42.42.42.8" class="ltx_td ltx_align_center">628.24</td>
</tr>
<tr id="A4.275.275.247.179.111.43.43.43" class="ltx_tr">
<td id="A4.275.275.247.179.111.43.43.43.1" class="ltx_td ltx_align_left">GPTQ (4-Bit) <math id="A4.275.275.247.179.111.43.43.43.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.275.275.247.179.111.43.43.43.1.m1.1a"><mo stretchy="false" id="A4.275.275.247.179.111.43.43.43.1.m1.1.1" xref="A4.275.275.247.179.111.43.43.43.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.275.275.247.179.111.43.43.43.1.m1.1b"><ci id="A4.275.275.247.179.111.43.43.43.1.m1.1.1.cmml" xref="A4.275.275.247.179.111.43.43.43.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.275.275.247.179.111.43.43.43.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.275.275.247.179.111.43.43.43.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.275.275.247.179.111.43.43.43.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.275.275.247.179.111.43.43.43.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.275.275.247.179.111.43.43.43.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.275.275.247.179.111.43.43.43.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.275.275.247.179.111.43.43.43.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.275.275.247.179.111.43.43.43.8" class="ltx_td ltx_align_center">9.38</td>
</tr>
<tr id="A4.276.276.248.180.112.44.44.44" class="ltx_tr">
<td id="A4.276.276.248.180.112.44.44.44.1" class="ltx_td ltx_align_left">GPTQ (8-Bit) <math id="A4.276.276.248.180.112.44.44.44.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.276.276.248.180.112.44.44.44.1.m1.1a"><mo stretchy="false" id="A4.276.276.248.180.112.44.44.44.1.m1.1.1" xref="A4.276.276.248.180.112.44.44.44.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.276.276.248.180.112.44.44.44.1.m1.1b"><ci id="A4.276.276.248.180.112.44.44.44.1.m1.1.1.cmml" xref="A4.276.276.248.180.112.44.44.44.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.276.276.248.180.112.44.44.44.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.276.276.248.180.112.44.44.44.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.276.276.248.180.112.44.44.44.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.276.276.248.180.112.44.44.44.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.276.276.248.180.112.44.44.44.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.276.276.248.180.112.44.44.44.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.276.276.248.180.112.44.44.44.7" class="ltx_td ltx_align_center">0.47</td>
<td id="A4.276.276.248.180.112.44.44.44.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.277.277.249.181.113.45.45.45" class="ltx_tr">
<td id="A4.277.277.249.181.113.45.45.45.1" class="ltx_td ltx_align_left">GPTQ (2-Bit) <math id="A4.277.277.249.181.113.45.45.45.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.277.277.249.181.113.45.45.45.1.m1.1a"><mo stretchy="false" id="A4.277.277.249.181.113.45.45.45.1.m1.1.1" xref="A4.277.277.249.181.113.45.45.45.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.277.277.249.181.113.45.45.45.1.m1.1b"><ci id="A4.277.277.249.181.113.45.45.45.1.m1.1.1.cmml" xref="A4.277.277.249.181.113.45.45.45.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.277.277.249.181.113.45.45.45.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.277.277.249.181.113.45.45.45.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.277.277.249.181.113.45.45.45.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.277.277.249.181.113.45.45.45.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.277.277.249.181.113.45.45.45.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.277.277.249.181.113.45.45.45.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.277.277.249.181.113.45.45.45.7" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.277.277.249.181.113.45.45.45.8" class="ltx_td ltx_align_center">1017825.94</td>
</tr>
<tr id="A4.278.278.250.182.114.46.46.46" class="ltx_tr">
<td id="A4.278.278.250.182.114.46.46.46.1" class="ltx_td ltx_align_left">GPTQ (3-Bit) <math id="A4.278.278.250.182.114.46.46.46.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.278.278.250.182.114.46.46.46.1.m1.1a"><mo stretchy="false" id="A4.278.278.250.182.114.46.46.46.1.m1.1.1" xref="A4.278.278.250.182.114.46.46.46.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.278.278.250.182.114.46.46.46.1.m1.1b"><ci id="A4.278.278.250.182.114.46.46.46.1.m1.1.1.cmml" xref="A4.278.278.250.182.114.46.46.46.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.278.278.250.182.114.46.46.46.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.278.278.250.182.114.46.46.46.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.278.278.250.182.114.46.46.46.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.278.278.250.182.114.46.46.46.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.278.278.250.182.114.46.46.46.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.278.278.250.182.114.46.46.46.6" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.278.278.250.182.114.46.46.46.7" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.278.278.250.182.114.46.46.46.8" class="ltx_td ltx_align_center">11.1</td>
</tr>
<tr id="A4.279.279.251.183.115.47.47.47" class="ltx_tr">
<td id="A4.279.279.251.183.115.47.47.47.1" class="ltx_td ltx_align_left">GPTQ (4-Bit) <math id="A4.279.279.251.183.115.47.47.47.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.279.279.251.183.115.47.47.47.1.m1.1a"><mo stretchy="false" id="A4.279.279.251.183.115.47.47.47.1.m1.1.1" xref="A4.279.279.251.183.115.47.47.47.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.279.279.251.183.115.47.47.47.1.m1.1b"><ci id="A4.279.279.251.183.115.47.47.47.1.m1.1.1.cmml" xref="A4.279.279.251.183.115.47.47.47.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.279.279.251.183.115.47.47.47.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.279.279.251.183.115.47.47.47.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.279.279.251.183.115.47.47.47.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.279.279.251.183.115.47.47.47.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.279.279.251.183.115.47.47.47.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.279.279.251.183.115.47.47.47.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.279.279.251.183.115.47.47.47.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.279.279.251.183.115.47.47.47.8" class="ltx_td ltx_align_center">6.35</td>
</tr>
<tr id="A4.280.280.252.184.116.48.48.48" class="ltx_tr">
<td id="A4.280.280.252.184.116.48.48.48.1" class="ltx_td ltx_align_left">GPTQ (8-Bit) <math id="A4.280.280.252.184.116.48.48.48.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.280.280.252.184.116.48.48.48.1.m1.1a"><mo stretchy="false" id="A4.280.280.252.184.116.48.48.48.1.m1.1.1" xref="A4.280.280.252.184.116.48.48.48.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.280.280.252.184.116.48.48.48.1.m1.1b"><ci id="A4.280.280.252.184.116.48.48.48.1.m1.1.1.cmml" xref="A4.280.280.252.184.116.48.48.48.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.280.280.252.184.116.48.48.48.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.280.280.252.184.116.48.48.48.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.280.280.252.184.116.48.48.48.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.280.280.252.184.116.48.48.48.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.280.280.252.184.116.48.48.48.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.280.280.252.184.116.48.48.48.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.280.280.252.184.116.48.48.48.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.280.280.252.184.116.48.48.48.8" class="ltx_td ltx_align_center">5.58</td>
</tr>
<tr id="A4.281.281.253.185.117.49.49.49" class="ltx_tr">
<td id="A4.281.281.253.185.117.49.49.49.1" class="ltx_td ltx_align_left">
<span id="A4.281.281.253.185.117.49.49.49.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

AWQ (2-Bit) <math id="A4.281.281.253.185.117.49.49.49.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.281.281.253.185.117.49.49.49.1.m1.1a"><mo stretchy="false" id="A4.281.281.253.185.117.49.49.49.1.m1.1.1" xref="A4.281.281.253.185.117.49.49.49.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.281.281.253.185.117.49.49.49.1.m1.1b"><ci id="A4.281.281.253.185.117.49.49.49.1.m1.1.1.cmml" xref="A4.281.281.253.185.117.49.49.49.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.281.281.253.185.117.49.49.49.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.281.281.253.185.117.49.49.49.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.281.281.253.185.117.49.49.49.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.281.281.253.185.117.49.49.49.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.281.281.253.185.117.49.49.49.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.281.281.253.185.117.49.49.49.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.281.281.253.185.117.49.49.49.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.281.281.253.185.117.49.49.49.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.282.282.254.186.118.50.50.50" class="ltx_tr">
<td id="A4.282.282.254.186.118.50.50.50.1" class="ltx_td ltx_align_left">AWQ (3-Bit) <math id="A4.282.282.254.186.118.50.50.50.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.282.282.254.186.118.50.50.50.1.m1.1a"><mo stretchy="false" id="A4.282.282.254.186.118.50.50.50.1.m1.1.1" xref="A4.282.282.254.186.118.50.50.50.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.282.282.254.186.118.50.50.50.1.m1.1b"><ci id="A4.282.282.254.186.118.50.50.50.1.m1.1.1.cmml" xref="A4.282.282.254.186.118.50.50.50.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.282.282.254.186.118.50.50.50.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.282.282.254.186.118.50.50.50.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.282.282.254.186.118.50.50.50.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.282.282.254.186.118.50.50.50.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.282.282.254.186.118.50.50.50.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.282.282.254.186.118.50.50.50.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.282.282.254.186.118.50.50.50.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.282.282.254.186.118.50.50.50.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.283.283.255.187.119.51.51.51" class="ltx_tr">
<td id="A4.283.283.255.187.119.51.51.51.1" class="ltx_td ltx_align_left">AWQ (4-Bit) <math id="A4.283.283.255.187.119.51.51.51.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.283.283.255.187.119.51.51.51.1.m1.1a"><mo stretchy="false" id="A4.283.283.255.187.119.51.51.51.1.m1.1.1" xref="A4.283.283.255.187.119.51.51.51.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.283.283.255.187.119.51.51.51.1.m1.1b"><ci id="A4.283.283.255.187.119.51.51.51.1.m1.1.1.cmml" xref="A4.283.283.255.187.119.51.51.51.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.283.283.255.187.119.51.51.51.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.283.283.255.187.119.51.51.51.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.283.283.255.187.119.51.51.51.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.283.283.255.187.119.51.51.51.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.283.283.255.187.119.51.51.51.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.283.283.255.187.119.51.51.51.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.283.283.255.187.119.51.51.51.7" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.283.283.255.187.119.51.51.51.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.284.284.256.188.120.52.52.52" class="ltx_tr">
<td id="A4.284.284.256.188.120.52.52.52.1" class="ltx_td ltx_align_left">AWQ (5-Bit) <math id="A4.284.284.256.188.120.52.52.52.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.284.284.256.188.120.52.52.52.1.m1.1a"><mo stretchy="false" id="A4.284.284.256.188.120.52.52.52.1.m1.1.1" xref="A4.284.284.256.188.120.52.52.52.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.284.284.256.188.120.52.52.52.1.m1.1b"><ci id="A4.284.284.256.188.120.52.52.52.1.m1.1.1.cmml" xref="A4.284.284.256.188.120.52.52.52.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.284.284.256.188.120.52.52.52.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.284.284.256.188.120.52.52.52.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.284.284.256.188.120.52.52.52.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.284.284.256.188.120.52.52.52.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.284.284.256.188.120.52.52.52.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.284.284.256.188.120.52.52.52.6" class="ltx_td ltx_align_center">0.35</td>
<td id="A4.284.284.256.188.120.52.52.52.7" class="ltx_td ltx_align_center">0.39</td>
<td id="A4.284.284.256.188.120.52.52.52.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.285.285.257.189.121.53.53.53" class="ltx_tr">
<td id="A4.285.285.257.189.121.53.53.53.1" class="ltx_td ltx_align_left">AWQ (6-Bit) <math id="A4.285.285.257.189.121.53.53.53.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.285.285.257.189.121.53.53.53.1.m1.1a"><mo stretchy="false" id="A4.285.285.257.189.121.53.53.53.1.m1.1.1" xref="A4.285.285.257.189.121.53.53.53.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.285.285.257.189.121.53.53.53.1.m1.1b"><ci id="A4.285.285.257.189.121.53.53.53.1.m1.1.1.cmml" xref="A4.285.285.257.189.121.53.53.53.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.285.285.257.189.121.53.53.53.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.285.285.257.189.121.53.53.53.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.285.285.257.189.121.53.53.53.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.285.285.257.189.121.53.53.53.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.285.285.257.189.121.53.53.53.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.285.285.257.189.121.53.53.53.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.285.285.257.189.121.53.53.53.7" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.285.285.257.189.121.53.53.53.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.286.286.258.190.122.54.54.54" class="ltx_tr">
<td id="A4.286.286.258.190.122.54.54.54.1" class="ltx_td ltx_align_left">AWQ (8-Bit) <math id="A4.286.286.258.190.122.54.54.54.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.286.286.258.190.122.54.54.54.1.m1.1a"><mo stretchy="false" id="A4.286.286.258.190.122.54.54.54.1.m1.1.1" xref="A4.286.286.258.190.122.54.54.54.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.286.286.258.190.122.54.54.54.1.m1.1b"><ci id="A4.286.286.258.190.122.54.54.54.1.m1.1.1.cmml" xref="A4.286.286.258.190.122.54.54.54.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.286.286.258.190.122.54.54.54.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.286.286.258.190.122.54.54.54.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.286.286.258.190.122.54.54.54.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.286.286.258.190.122.54.54.54.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.286.286.258.190.122.54.54.54.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.286.286.258.190.122.54.54.54.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.286.286.258.190.122.54.54.54.7" class="ltx_td ltx_align_center">0.37</td>
<td id="A4.286.286.258.190.122.54.54.54.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.287.287.259.191.123.55.55.55" class="ltx_tr">
<td id="A4.287.287.259.191.123.55.55.55.1" class="ltx_td ltx_align_left">AWQ (2-Bit) <math id="A4.287.287.259.191.123.55.55.55.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.287.287.259.191.123.55.55.55.1.m1.1a"><mo stretchy="false" id="A4.287.287.259.191.123.55.55.55.1.m1.1.1" xref="A4.287.287.259.191.123.55.55.55.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.287.287.259.191.123.55.55.55.1.m1.1b"><ci id="A4.287.287.259.191.123.55.55.55.1.m1.1.1.cmml" xref="A4.287.287.259.191.123.55.55.55.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.287.287.259.191.123.55.55.55.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.287.287.259.191.123.55.55.55.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.287.287.259.191.123.55.55.55.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.287.287.259.191.123.55.55.55.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.287.287.259.191.123.55.55.55.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.287.287.259.191.123.55.55.55.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.287.287.259.191.123.55.55.55.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.287.287.259.191.123.55.55.55.8" class="ltx_td ltx_align_center">4.496028624899119e+33</td>
</tr>
<tr id="A4.288.288.260.192.124.56.56.56" class="ltx_tr">
<td id="A4.288.288.260.192.124.56.56.56.1" class="ltx_td ltx_align_left">AWQ (3-Bit) <math id="A4.288.288.260.192.124.56.56.56.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.288.288.260.192.124.56.56.56.1.m1.1a"><mo stretchy="false" id="A4.288.288.260.192.124.56.56.56.1.m1.1.1" xref="A4.288.288.260.192.124.56.56.56.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.288.288.260.192.124.56.56.56.1.m1.1b"><ci id="A4.288.288.260.192.124.56.56.56.1.m1.1.1.cmml" xref="A4.288.288.260.192.124.56.56.56.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.288.288.260.192.124.56.56.56.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.288.288.260.192.124.56.56.56.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.288.288.260.192.124.56.56.56.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.288.288.260.192.124.56.56.56.4" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.288.288.260.192.124.56.56.56.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.288.288.260.192.124.56.56.56.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.288.288.260.192.124.56.56.56.7" class="ltx_td ltx_align_center">0.34</td>
<td id="A4.288.288.260.192.124.56.56.56.8" class="ltx_td ltx_align_center">166895.91</td>
</tr>
<tr id="A4.289.289.261.193.125.57.57.57" class="ltx_tr">
<td id="A4.289.289.261.193.125.57.57.57.1" class="ltx_td ltx_align_left">AWQ (4-Bit) <math id="A4.289.289.261.193.125.57.57.57.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.289.289.261.193.125.57.57.57.1.m1.1a"><mo stretchy="false" id="A4.289.289.261.193.125.57.57.57.1.m1.1.1" xref="A4.289.289.261.193.125.57.57.57.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.289.289.261.193.125.57.57.57.1.m1.1b"><ci id="A4.289.289.261.193.125.57.57.57.1.m1.1.1.cmml" xref="A4.289.289.261.193.125.57.57.57.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.289.289.261.193.125.57.57.57.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.289.289.261.193.125.57.57.57.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.289.289.261.193.125.57.57.57.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.289.289.261.193.125.57.57.57.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.289.289.261.193.125.57.57.57.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.289.289.261.193.125.57.57.57.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.289.289.261.193.125.57.57.57.7" class="ltx_td ltx_align_center">0.42</td>
<td id="A4.289.289.261.193.125.57.57.57.8" class="ltx_td ltx_align_center">7.05</td>
</tr>
<tr id="A4.290.290.262.194.126.58.58.58" class="ltx_tr">
<td id="A4.290.290.262.194.126.58.58.58.1" class="ltx_td ltx_align_left">AWQ (5-Bit) <math id="A4.290.290.262.194.126.58.58.58.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.290.290.262.194.126.58.58.58.1.m1.1a"><mo stretchy="false" id="A4.290.290.262.194.126.58.58.58.1.m1.1.1" xref="A4.290.290.262.194.126.58.58.58.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.290.290.262.194.126.58.58.58.1.m1.1b"><ci id="A4.290.290.262.194.126.58.58.58.1.m1.1.1.cmml" xref="A4.290.290.262.194.126.58.58.58.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.290.290.262.194.126.58.58.58.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.290.290.262.194.126.58.58.58.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.290.290.262.194.126.58.58.58.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.290.290.262.194.126.58.58.58.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.290.290.262.194.126.58.58.58.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.290.290.262.194.126.58.58.58.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.290.290.262.194.126.58.58.58.7" class="ltx_td ltx_align_center">0.38</td>
<td id="A4.290.290.262.194.126.58.58.58.8" class="ltx_td ltx_align_center">2685384.0</td>
</tr>
<tr id="A4.291.291.263.195.127.59.59.59" class="ltx_tr">
<td id="A4.291.291.263.195.127.59.59.59.1" class="ltx_td ltx_align_left">AWQ (6-Bit) <math id="A4.291.291.263.195.127.59.59.59.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.291.291.263.195.127.59.59.59.1.m1.1a"><mo stretchy="false" id="A4.291.291.263.195.127.59.59.59.1.m1.1.1" xref="A4.291.291.263.195.127.59.59.59.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.291.291.263.195.127.59.59.59.1.m1.1b"><ci id="A4.291.291.263.195.127.59.59.59.1.m1.1.1.cmml" xref="A4.291.291.263.195.127.59.59.59.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.291.291.263.195.127.59.59.59.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.291.291.263.195.127.59.59.59.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.291.291.263.195.127.59.59.59.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.291.291.263.195.127.59.59.59.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.291.291.263.195.127.59.59.59.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.291.291.263.195.127.59.59.59.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.291.291.263.195.127.59.59.59.7" class="ltx_td ltx_align_center">0.31</td>
<td id="A4.291.291.263.195.127.59.59.59.8" class="ltx_td ltx_align_center">27025.07</td>
</tr>
<tr id="A4.292.292.264.196.128.60.60.60" class="ltx_tr">
<td id="A4.292.292.264.196.128.60.60.60.1" class="ltx_td ltx_align_left">AWQ (8-Bit) <math id="A4.292.292.264.196.128.60.60.60.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.292.292.264.196.128.60.60.60.1.m1.1a"><mo stretchy="false" id="A4.292.292.264.196.128.60.60.60.1.m1.1.1" xref="A4.292.292.264.196.128.60.60.60.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.292.292.264.196.128.60.60.60.1.m1.1b"><ci id="A4.292.292.264.196.128.60.60.60.1.m1.1.1.cmml" xref="A4.292.292.264.196.128.60.60.60.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.292.292.264.196.128.60.60.60.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.292.292.264.196.128.60.60.60.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.292.292.264.196.128.60.60.60.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.292.292.264.196.128.60.60.60.4" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.292.292.264.196.128.60.60.60.5" class="ltx_td ltx_align_center">8.25</td>
<td id="A4.292.292.264.196.128.60.60.60.6" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.292.292.264.196.128.60.60.60.7" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.292.292.264.196.128.60.60.60.8" class="ltx_td ltx_align_center">49224794112.0</td>
</tr>
<tr id="A4.293.293.265.197.129.61.61.61" class="ltx_tr">
<td id="A4.293.293.265.197.129.61.61.61.1" class="ltx_td ltx_align_left">AWQ (2-Bit) <math id="A4.293.293.265.197.129.61.61.61.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.293.293.265.197.129.61.61.61.1.m1.1a"><mo stretchy="false" id="A4.293.293.265.197.129.61.61.61.1.m1.1.1" xref="A4.293.293.265.197.129.61.61.61.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.293.293.265.197.129.61.61.61.1.m1.1b"><ci id="A4.293.293.265.197.129.61.61.61.1.m1.1.1.cmml" xref="A4.293.293.265.197.129.61.61.61.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.293.293.265.197.129.61.61.61.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.293.293.265.197.129.61.61.61.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.293.293.265.197.129.61.61.61.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.293.293.265.197.129.61.61.61.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.293.293.265.197.129.61.61.61.5" class="ltx_td ltx_align_center">2.25</td>
<td id="A4.293.293.265.197.129.61.61.61.6" class="ltx_td ltx_align_center">0.24</td>
<td id="A4.293.293.265.197.129.61.61.61.7" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.293.293.265.197.129.61.61.61.8" class="ltx_td ltx_align_center">1749321.75</td>
</tr>
<tr id="A4.294.294.266.198.130.62.62.62" class="ltx_tr">
<td id="A4.294.294.266.198.130.62.62.62.1" class="ltx_td ltx_align_left">AWQ (3-Bit) <math id="A4.294.294.266.198.130.62.62.62.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.294.294.266.198.130.62.62.62.1.m1.1a"><mo stretchy="false" id="A4.294.294.266.198.130.62.62.62.1.m1.1.1" xref="A4.294.294.266.198.130.62.62.62.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.294.294.266.198.130.62.62.62.1.m1.1b"><ci id="A4.294.294.266.198.130.62.62.62.1.m1.1.1.cmml" xref="A4.294.294.266.198.130.62.62.62.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.294.294.266.198.130.62.62.62.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.294.294.266.198.130.62.62.62.2" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.294.294.266.198.130.62.62.62.3" class="ltx_td ltx_align_center">0.01</td>
<td id="A4.294.294.266.198.130.62.62.62.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.294.294.266.198.130.62.62.62.5" class="ltx_td ltx_align_center">3.25</td>
<td id="A4.294.294.266.198.130.62.62.62.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.294.294.266.198.130.62.62.62.7" class="ltx_td ltx_align_center">0.46</td>
<td id="A4.294.294.266.198.130.62.62.62.8" class="ltx_td ltx_align_center">7.51</td>
</tr>
<tr id="A4.295.295.267.199.131.63.63.63" class="ltx_tr">
<td id="A4.295.295.267.199.131.63.63.63.1" class="ltx_td ltx_align_left">AWQ (4-Bit) <math id="A4.295.295.267.199.131.63.63.63.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.295.295.267.199.131.63.63.63.1.m1.1a"><mo stretchy="false" id="A4.295.295.267.199.131.63.63.63.1.m1.1.1" xref="A4.295.295.267.199.131.63.63.63.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.295.295.267.199.131.63.63.63.1.m1.1b"><ci id="A4.295.295.267.199.131.63.63.63.1.m1.1.1.cmml" xref="A4.295.295.267.199.131.63.63.63.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.295.295.267.199.131.63.63.63.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.295.295.267.199.131.63.63.63.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.295.295.267.199.131.63.63.63.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.295.295.267.199.131.63.63.63.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.295.295.267.199.131.63.63.63.5" class="ltx_td ltx_align_center">4.25</td>
<td id="A4.295.295.267.199.131.63.63.63.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.295.295.267.199.131.63.63.63.7" class="ltx_td ltx_align_center">0.55</td>
<td id="A4.295.295.267.199.131.63.63.63.8" class="ltx_td ltx_align_center">6.06</td>
</tr>
<tr id="A4.296.296.268.200.132.64.64.64" class="ltx_tr">
<td id="A4.296.296.268.200.132.64.64.64.1" class="ltx_td ltx_align_left">AWQ (5-Bit) <math id="A4.296.296.268.200.132.64.64.64.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.296.296.268.200.132.64.64.64.1.m1.1a"><mo stretchy="false" id="A4.296.296.268.200.132.64.64.64.1.m1.1.1" xref="A4.296.296.268.200.132.64.64.64.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.296.296.268.200.132.64.64.64.1.m1.1b"><ci id="A4.296.296.268.200.132.64.64.64.1.m1.1.1.cmml" xref="A4.296.296.268.200.132.64.64.64.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.296.296.268.200.132.64.64.64.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.296.296.268.200.132.64.64.64.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.296.296.268.200.132.64.64.64.3" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.296.296.268.200.132.64.64.64.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.296.296.268.200.132.64.64.64.5" class="ltx_td ltx_align_center">5.25</td>
<td id="A4.296.296.268.200.132.64.64.64.6" class="ltx_td ltx_align_center">0.27</td>
<td id="A4.296.296.268.200.132.64.64.64.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.296.296.268.200.132.64.64.64.8" class="ltx_td ltx_align_center">5.68</td>
</tr>
<tr id="A4.297.297.269.201.133.65.65.65" class="ltx_tr">
<td id="A4.297.297.269.201.133.65.65.65.1" class="ltx_td ltx_align_left">AWQ (6-Bit) <math id="A4.297.297.269.201.133.65.65.65.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.297.297.269.201.133.65.65.65.1.m1.1a"><mo stretchy="false" id="A4.297.297.269.201.133.65.65.65.1.m1.1.1" xref="A4.297.297.269.201.133.65.65.65.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.297.297.269.201.133.65.65.65.1.m1.1b"><ci id="A4.297.297.269.201.133.65.65.65.1.m1.1.1.cmml" xref="A4.297.297.269.201.133.65.65.65.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.297.297.269.201.133.65.65.65.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.297.297.269.201.133.65.65.65.2" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.297.297.269.201.133.65.65.65.3" class="ltx_td ltx_align_center">0.02</td>
<td id="A4.297.297.269.201.133.65.65.65.4" class="ltx_td ltx_align_center">0.04</td>
<td id="A4.297.297.269.201.133.65.65.65.5" class="ltx_td ltx_align_center">6.25</td>
<td id="A4.297.297.269.201.133.65.65.65.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.297.297.269.201.133.65.65.65.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.297.297.269.201.133.65.65.65.8" class="ltx_td ltx_align_center">5.6</td>
</tr>
<tr id="A4.298.298.270.202.134.66.66.66" class="ltx_tr">
<td id="A4.298.298.270.202.134.66.66.66.1" class="ltx_td ltx_align_left ltx_border_bb">AWQ (8-Bit) <math id="A4.298.298.270.202.134.66.66.66.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.298.298.270.202.134.66.66.66.1.m1.1a"><mo stretchy="false" id="A4.298.298.270.202.134.66.66.66.1.m1.1.1" xref="A4.298.298.270.202.134.66.66.66.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.298.298.270.202.134.66.66.66.1.m1.1b"><ci id="A4.298.298.270.202.134.66.66.66.1.m1.1.1.cmml" xref="A4.298.298.270.202.134.66.66.66.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.298.298.270.202.134.66.66.66.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.298.298.270.202.134.66.66.66.2" class="ltx_td ltx_align_center ltx_border_bb">0.02</td>
<td id="A4.298.298.270.202.134.66.66.66.3" class="ltx_td ltx_align_center ltx_border_bb">0.03</td>
<td id="A4.298.298.270.202.134.66.66.66.4" class="ltx_td ltx_align_center ltx_border_bb">0.03</td>
<td id="A4.298.298.270.202.134.66.66.66.5" class="ltx_td ltx_align_center ltx_border_bb">8.25</td>
<td id="A4.298.298.270.202.134.66.66.66.6" class="ltx_td ltx_align_center ltx_border_bb">0.28</td>
<td id="A4.298.298.270.202.134.66.66.66.7" class="ltx_td ltx_align_center ltx_border_bb">0.57</td>
<td id="A4.298.298.270.202.134.66.66.66.8" class="ltx_td ltx_align_center ltx_border_bb">5.71</td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="A4.322.322.294.226.158.91.2.1" class="ltx_text" style="font-size:90%;">Table 12</span>: </span><span id="A4.300.300.272.204.136.68.1" class="ltx_text" style="font-size:90%;">Detailed results for compression <math id="A4.300.300.272.204.136.68.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.300.300.272.204.136.68.1.m1.1b"><mo stretchy="false" id="A4.300.300.272.204.136.68.1.m1.1.1" xref="A4.300.300.272.204.136.68.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.300.300.272.204.136.68.1.m1.1c"><ci id="A4.300.300.272.204.136.68.1.m1.1.1.cmml" xref="A4.300.300.272.204.136.68.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.300.300.272.204.136.68.1.m1.1d">\rightarrow</annotation></semantics></math> unlearning.</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="A4.322.322.294.226.158.90" class="ltx_table ltx_figure_panel">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="A4.309.309.281.213.145.77.9" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:164.5pt;vertical-align:-1.6pt;"><span class="ltx_transformed_inner" style="transform:translate(-73.2pt,27.5pt) scale(0.747494056590366,0.747494056590366) ;">
<table id="A4.309.309.281.213.145.77.9.9" class="ltx_tabular ltx_align_middle">
<tr id="A4.309.309.281.213.145.77.9.9.10" class="ltx_tr">
<td id="A4.309.309.281.213.145.77.9.9.10.1" class="ltx_td ltx_border_tt"></td>
<td id="A4.309.309.281.213.145.77.9.9.10.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="3">Editing</td>
<td id="A4.309.309.281.213.145.77.9.9.10.3" class="ltx_td ltx_align_center ltx_border_tt">Compression</td>
<td id="A4.309.309.281.213.145.77.9.9.10.4" class="ltx_td ltx_align_center ltx_border_tt">Unlearning</td>
<td id="A4.309.309.281.213.145.77.9.9.10.5" class="ltx_td ltx_align_center ltx_border_tt" colspan="2">Utility</td>
</tr>
<tr id="A4.309.309.281.213.145.77.9.9.11" class="ltx_tr">
<td id="A4.309.309.281.213.145.77.9.9.11.1" class="ltx_td"></td>
<td id="A4.309.309.281.213.145.77.9.9.11.2" class="ltx_td ltx_align_center ltx_border_t">Edit Success</td>
<td id="A4.309.309.281.213.145.77.9.9.11.3" class="ltx_td ltx_align_center ltx_border_t">Generalization</td>
<td id="A4.309.309.281.213.145.77.9.9.11.4" class="ltx_td ltx_align_center ltx_border_t">Locality</td>
<td id="A4.309.309.281.213.145.77.9.9.11.5" class="ltx_td ltx_align_center ltx_border_t">Avg. Bits</td>
<td id="A4.309.309.281.213.145.77.9.9.11.6" class="ltx_td ltx_align_center ltx_border_t">Avg. WMDP</td>
<td id="A4.309.309.281.213.145.77.9.9.11.7" class="ltx_td ltx_align_center ltx_border_t">MMLU</td>
<td id="A4.309.309.281.213.145.77.9.9.11.8" class="ltx_td ltx_align_center ltx_border_t">WikiText PPL</td>
</tr>
<tr id="A4.301.301.273.205.137.69.1.1.1" class="ltx_tr">
<td id="A4.301.301.273.205.137.69.1.1.1.1" class="ltx_td ltx_align_left ltx_border_t">FT<math id="A4.301.301.273.205.137.69.1.1.1.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.301.301.273.205.137.69.1.1.1.1.m1.1a"><mo stretchy="false" id="A4.301.301.273.205.137.69.1.1.1.1.m1.1.1" xref="A4.301.301.273.205.137.69.1.1.1.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.301.301.273.205.137.69.1.1.1.1.m1.1b"><ci id="A4.301.301.273.205.137.69.1.1.1.1.m1.1.1.cmml" xref="A4.301.301.273.205.137.69.1.1.1.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.301.301.273.205.137.69.1.1.1.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.301.301.273.205.137.69.1.1.1.2" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.301.301.273.205.137.69.1.1.1.3" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.301.301.273.205.137.69.1.1.1.4" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.301.301.273.205.137.69.1.1.1.5" class="ltx_td ltx_align_center ltx_border_t">16.0</td>
<td id="A4.301.301.273.205.137.69.1.1.1.6" class="ltx_td ltx_align_center ltx_border_t">0.47</td>
<td id="A4.301.301.273.205.137.69.1.1.1.7" class="ltx_td ltx_align_center ltx_border_t">0.53</td>
<td id="A4.301.301.273.205.137.69.1.1.1.8" class="ltx_td ltx_align_center ltx_border_t">inf</td>
</tr>
<tr id="A4.302.302.274.206.138.70.2.2.2" class="ltx_tr">
<td id="A4.302.302.274.206.138.70.2.2.2.1" class="ltx_td ltx_align_left">FT<math id="A4.302.302.274.206.138.70.2.2.2.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.302.302.274.206.138.70.2.2.2.1.m1.1a"><mo stretchy="false" id="A4.302.302.274.206.138.70.2.2.2.1.m1.1.1" xref="A4.302.302.274.206.138.70.2.2.2.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.302.302.274.206.138.70.2.2.2.1.m1.1b"><ci id="A4.302.302.274.206.138.70.2.2.2.1.m1.1.1.cmml" xref="A4.302.302.274.206.138.70.2.2.2.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.302.302.274.206.138.70.2.2.2.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.302.302.274.206.138.70.2.2.2.2" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.302.302.274.206.138.70.2.2.2.3" class="ltx_td ltx_align_center">0.25</td>
<td id="A4.302.302.274.206.138.70.2.2.2.4" class="ltx_td ltx_align_center">0.07</td>
<td id="A4.302.302.274.206.138.70.2.2.2.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.302.302.274.206.138.70.2.2.2.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.302.302.274.206.138.70.2.2.2.7" class="ltx_td ltx_align_center">0.41</td>
<td id="A4.302.302.274.206.138.70.2.2.2.8" class="ltx_td ltx_align_center">2.8236466916108585e+31</td>
</tr>
<tr id="A4.303.303.275.207.139.71.3.3.3" class="ltx_tr">
<td id="A4.303.303.275.207.139.71.3.3.3.1" class="ltx_td ltx_align_left">FT<math id="A4.303.303.275.207.139.71.3.3.3.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.303.303.275.207.139.71.3.3.3.1.m1.1a"><mo stretchy="false" id="A4.303.303.275.207.139.71.3.3.3.1.m1.1.1" xref="A4.303.303.275.207.139.71.3.3.3.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.303.303.275.207.139.71.3.3.3.1.m1.1b"><ci id="A4.303.303.275.207.139.71.3.3.3.1.m1.1.1.cmml" xref="A4.303.303.275.207.139.71.3.3.3.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.303.303.275.207.139.71.3.3.3.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.303.303.275.207.139.71.3.3.3.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.303.303.275.207.139.71.3.3.3.3" class="ltx_td ltx_align_center">0.82</td>
<td id="A4.303.303.275.207.139.71.3.3.3.4" class="ltx_td ltx_align_center">0.1</td>
<td id="A4.303.303.275.207.139.71.3.3.3.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.303.303.275.207.139.71.3.3.3.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.303.303.275.207.139.71.3.3.3.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.303.303.275.207.139.71.3.3.3.8" class="ltx_td ltx_align_center">5.61</td>
</tr>
<tr id="A4.304.304.276.208.140.72.4.4.4" class="ltx_tr">
<td id="A4.304.304.276.208.140.72.4.4.4.1" class="ltx_td ltx_align_left">
<span id="A4.304.304.276.208.140.72.4.4.4.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

MEMIT<math id="A4.304.304.276.208.140.72.4.4.4.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.304.304.276.208.140.72.4.4.4.1.m1.1a"><mo stretchy="false" id="A4.304.304.276.208.140.72.4.4.4.1.m1.1.1" xref="A4.304.304.276.208.140.72.4.4.4.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.304.304.276.208.140.72.4.4.4.1.m1.1b"><ci id="A4.304.304.276.208.140.72.4.4.4.1.m1.1.1.cmml" xref="A4.304.304.276.208.140.72.4.4.4.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.304.304.276.208.140.72.4.4.4.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.304.304.276.208.140.72.4.4.4.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.304.304.276.208.140.72.4.4.4.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.304.304.276.208.140.72.4.4.4.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.304.304.276.208.140.72.4.4.4.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.304.304.276.208.140.72.4.4.4.6" class="ltx_td ltx_align_center">0.4</td>
<td id="A4.304.304.276.208.140.72.4.4.4.7" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.304.304.276.208.140.72.4.4.4.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.305.305.277.209.141.73.5.5.5" class="ltx_tr">
<td id="A4.305.305.277.209.141.73.5.5.5.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.305.305.277.209.141.73.5.5.5.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.305.305.277.209.141.73.5.5.5.1.m1.1a"><mo stretchy="false" id="A4.305.305.277.209.141.73.5.5.5.1.m1.1.1" xref="A4.305.305.277.209.141.73.5.5.5.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.305.305.277.209.141.73.5.5.5.1.m1.1b"><ci id="A4.305.305.277.209.141.73.5.5.5.1.m1.1.1.cmml" xref="A4.305.305.277.209.141.73.5.5.5.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.305.305.277.209.141.73.5.5.5.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.305.305.277.209.141.73.5.5.5.2" class="ltx_td ltx_align_center">0.53</td>
<td id="A4.305.305.277.209.141.73.5.5.5.3" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.305.305.277.209.141.73.5.5.5.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.305.305.277.209.141.73.5.5.5.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.305.305.277.209.141.73.5.5.5.6" class="ltx_td ltx_align_center">0.26</td>
<td id="A4.305.305.277.209.141.73.5.5.5.7" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.305.305.277.209.141.73.5.5.5.8" class="ltx_td ltx_align_center">27645056122880.0</td>
</tr>
<tr id="A4.306.306.278.210.142.74.6.6.6" class="ltx_tr">
<td id="A4.306.306.278.210.142.74.6.6.6.1" class="ltx_td ltx_align_left">MEMIT<math id="A4.306.306.278.210.142.74.6.6.6.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.306.306.278.210.142.74.6.6.6.1.m1.1a"><mo stretchy="false" id="A4.306.306.278.210.142.74.6.6.6.1.m1.1.1" xref="A4.306.306.278.210.142.74.6.6.6.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.306.306.278.210.142.74.6.6.6.1.m1.1b"><ci id="A4.306.306.278.210.142.74.6.6.6.1.m1.1.1.cmml" xref="A4.306.306.278.210.142.74.6.6.6.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.306.306.278.210.142.74.6.6.6.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.306.306.278.210.142.74.6.6.6.2" class="ltx_td ltx_align_center">0.96</td>
<td id="A4.306.306.278.210.142.74.6.6.6.3" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.306.306.278.210.142.74.6.6.6.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.306.306.278.210.142.74.6.6.6.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.306.306.278.210.142.74.6.6.6.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.306.306.278.210.142.74.6.6.6.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.306.306.278.210.142.74.6.6.6.8" class="ltx_td ltx_align_center">5.58</td>
</tr>
<tr id="A4.307.307.279.211.143.75.7.7.7" class="ltx_tr">
<td id="A4.307.307.279.211.143.75.7.7.7.1" class="ltx_td ltx_align_left">
<span id="A4.307.307.279.211.143.75.7.7.7.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

LoRA<math id="A4.307.307.279.211.143.75.7.7.7.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.307.307.279.211.143.75.7.7.7.1.m1.1a"><mo stretchy="false" id="A4.307.307.279.211.143.75.7.7.7.1.m1.1.1" xref="A4.307.307.279.211.143.75.7.7.7.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.307.307.279.211.143.75.7.7.7.1.m1.1b"><ci id="A4.307.307.279.211.143.75.7.7.7.1.m1.1.1.cmml" xref="A4.307.307.279.211.143.75.7.7.7.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.307.307.279.211.143.75.7.7.7.1.m1.1c">\rightarrow</annotation></semantics></math>GA</td>
<td id="A4.307.307.279.211.143.75.7.7.7.2" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.307.307.279.211.143.75.7.7.7.3" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.307.307.279.211.143.75.7.7.7.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.307.307.279.211.143.75.7.7.7.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.307.307.279.211.143.75.7.7.7.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.307.307.279.211.143.75.7.7.7.7" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.307.307.279.211.143.75.7.7.7.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.308.308.280.212.144.76.8.8.8" class="ltx_tr">
<td id="A4.308.308.280.212.144.76.8.8.8.1" class="ltx_td ltx_align_left">LoRA<math id="A4.308.308.280.212.144.76.8.8.8.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.308.308.280.212.144.76.8.8.8.1.m1.1a"><mo stretchy="false" id="A4.308.308.280.212.144.76.8.8.8.1.m1.1.1" xref="A4.308.308.280.212.144.76.8.8.8.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.308.308.280.212.144.76.8.8.8.1.m1.1b"><ci id="A4.308.308.280.212.144.76.8.8.8.1.m1.1.1.cmml" xref="A4.308.308.280.212.144.76.8.8.8.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.308.308.280.212.144.76.8.8.8.1.m1.1c">\rightarrow</annotation></semantics></math>GD</td>
<td id="A4.308.308.280.212.144.76.8.8.8.2" class="ltx_td ltx_align_center">0.44</td>
<td id="A4.308.308.280.212.144.76.8.8.8.3" class="ltx_td ltx_align_center">0.23</td>
<td id="A4.308.308.280.212.144.76.8.8.8.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.308.308.280.212.144.76.8.8.8.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.308.308.280.212.144.76.8.8.8.6" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.308.308.280.212.144.76.8.8.8.7" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.308.308.280.212.144.76.8.8.8.8" class="ltx_td ltx_align_center">29.46</td>
</tr>
<tr id="A4.309.309.281.213.145.77.9.9.9" class="ltx_tr">
<td id="A4.309.309.281.213.145.77.9.9.9.1" class="ltx_td ltx_align_left ltx_border_bb">LoRA<math id="A4.309.309.281.213.145.77.9.9.9.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.309.309.281.213.145.77.9.9.9.1.m1.1a"><mo stretchy="false" id="A4.309.309.281.213.145.77.9.9.9.1.m1.1.1" xref="A4.309.309.281.213.145.77.9.9.9.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.309.309.281.213.145.77.9.9.9.1.m1.1b"><ci id="A4.309.309.281.213.145.77.9.9.9.1.m1.1.1.cmml" xref="A4.309.309.281.213.145.77.9.9.9.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.309.309.281.213.145.77.9.9.9.1.m1.1c">\rightarrow</annotation></semantics></math>RMU</td>
<td id="A4.309.309.281.213.145.77.9.9.9.2" class="ltx_td ltx_align_center ltx_border_bb">1.0</td>
<td id="A4.309.309.281.213.145.77.9.9.9.3" class="ltx_td ltx_align_center ltx_border_bb">0.68</td>
<td id="A4.309.309.281.213.145.77.9.9.9.4" class="ltx_td ltx_align_center ltx_border_bb">0.05</td>
<td id="A4.309.309.281.213.145.77.9.9.9.5" class="ltx_td ltx_align_center ltx_border_bb">16.0</td>
<td id="A4.309.309.281.213.145.77.9.9.9.6" class="ltx_td ltx_align_center ltx_border_bb">0.3</td>
<td id="A4.309.309.281.213.145.77.9.9.9.7" class="ltx_td ltx_align_center ltx_border_bb">0.52</td>
<td id="A4.309.309.281.213.145.77.9.9.9.8" class="ltx_td ltx_align_center ltx_border_bb">34.56</td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="A4.322.322.294.226.158.90.23.2.1" class="ltx_text" style="font-size:90%;">Table 13</span>: </span><span id="A4.311.311.283.215.147.79.11.1" class="ltx_text" style="font-size:90%;">Detailed results for editing <math id="A4.311.311.283.215.147.79.11.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.311.311.283.215.147.79.11.1.m1.1b"><mo stretchy="false" id="A4.311.311.283.215.147.79.11.1.m1.1.1" xref="A4.311.311.283.215.147.79.11.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.311.311.283.215.147.79.11.1.m1.1c"><ci id="A4.311.311.283.215.147.79.11.1.m1.1.1.cmml" xref="A4.311.311.283.215.147.79.11.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.311.311.283.215.147.79.11.1.m1.1d">\rightarrow</annotation></semantics></math> unlearning.</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<figure id="A4.322.322.294.226.158.90.22" class="ltx_table ltx_figure_panel">
<div class="ltx_flex_figure ltx_flex_table">
<div class="ltx_flex_cell ltx_flex_size_1">
<div id="A4.320.320.292.224.156.88.20.9" class="ltx_inline-block ltx_figure_panel ltx_transformed_outer" style="width:433.6pt;height:178.6pt;vertical-align:-1.7pt;"><span class="ltx_transformed_inner" style="transform:translate(-50.4pt,20.6pt) scale(0.811409156741764,0.811409156741764) ;">
<table id="A4.320.320.292.224.156.88.20.9.9" class="ltx_tabular ltx_align_middle">
<tr id="A4.320.320.292.224.156.88.20.9.9.10" class="ltx_tr">
<td id="A4.320.320.292.224.156.88.20.9.9.10.1" class="ltx_td ltx_border_tt"></td>
<td id="A4.320.320.292.224.156.88.20.9.9.10.2" class="ltx_td ltx_align_center ltx_border_tt" colspan="3">Editing</td>
<td id="A4.320.320.292.224.156.88.20.9.9.10.3" class="ltx_td ltx_align_center ltx_border_tt">Compression</td>
<td id="A4.320.320.292.224.156.88.20.9.9.10.4" class="ltx_td ltx_align_center ltx_border_tt">Unlearning</td>
<td id="A4.320.320.292.224.156.88.20.9.9.10.5" class="ltx_td ltx_align_center ltx_border_tt" colspan="2">Utility</td>
</tr>
<tr id="A4.320.320.292.224.156.88.20.9.9.11" class="ltx_tr">
<td id="A4.320.320.292.224.156.88.20.9.9.11.1" class="ltx_td"></td>
<td id="A4.320.320.292.224.156.88.20.9.9.11.2" class="ltx_td ltx_align_center ltx_border_t">Edit Success</td>
<td id="A4.320.320.292.224.156.88.20.9.9.11.3" class="ltx_td ltx_align_center ltx_border_t">Generalization</td>
<td id="A4.320.320.292.224.156.88.20.9.9.11.4" class="ltx_td ltx_align_center ltx_border_t">Locality</td>
<td id="A4.320.320.292.224.156.88.20.9.9.11.5" class="ltx_td ltx_align_center ltx_border_t">Avg. Bits</td>
<td id="A4.320.320.292.224.156.88.20.9.9.11.6" class="ltx_td ltx_align_center ltx_border_t">Avg. WMDP</td>
<td id="A4.320.320.292.224.156.88.20.9.9.11.7" class="ltx_td ltx_align_center ltx_border_t">MMLU</td>
<td id="A4.320.320.292.224.156.88.20.9.9.11.8" class="ltx_td ltx_align_center ltx_border_t">WikiText PPL</td>
</tr>
<tr id="A4.312.312.284.216.148.80.12.1.1.1" class="ltx_tr">
<td id="A4.312.312.284.216.148.80.12.1.1.1.1" class="ltx_td ltx_align_left ltx_border_t">GA<math id="A4.312.312.284.216.148.80.12.1.1.1.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.312.312.284.216.148.80.12.1.1.1.1.m1.1a"><mo stretchy="false" id="A4.312.312.284.216.148.80.12.1.1.1.1.m1.1.1" xref="A4.312.312.284.216.148.80.12.1.1.1.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.312.312.284.216.148.80.12.1.1.1.1.m1.1b"><ci id="A4.312.312.284.216.148.80.12.1.1.1.1.m1.1.1.cmml" xref="A4.312.312.284.216.148.80.12.1.1.1.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.312.312.284.216.148.80.12.1.1.1.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.312.312.284.216.148.80.12.1.1.1.2" class="ltx_td ltx_align_center ltx_border_t">0.07</td>
<td id="A4.312.312.284.216.148.80.12.1.1.1.3" class="ltx_td ltx_align_center ltx_border_t">0.04</td>
<td id="A4.312.312.284.216.148.80.12.1.1.1.4" class="ltx_td ltx_align_center ltx_border_t">0.0</td>
<td id="A4.312.312.284.216.148.80.12.1.1.1.5" class="ltx_td ltx_align_center ltx_border_t">16.0</td>
<td id="A4.312.312.284.216.148.80.12.1.1.1.6" class="ltx_td ltx_align_center ltx_border_t">0.47</td>
<td id="A4.312.312.284.216.148.80.12.1.1.1.7" class="ltx_td ltx_align_center ltx_border_t">0.52</td>
<td id="A4.312.312.284.216.148.80.12.1.1.1.8" class="ltx_td ltx_align_center ltx_border_t">inf</td>
</tr>
<tr id="A4.313.313.285.217.149.81.13.2.2.2" class="ltx_tr">
<td id="A4.313.313.285.217.149.81.13.2.2.2.1" class="ltx_td ltx_align_left">GA<math id="A4.313.313.285.217.149.81.13.2.2.2.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.313.313.285.217.149.81.13.2.2.2.1.m1.1a"><mo stretchy="false" id="A4.313.313.285.217.149.81.13.2.2.2.1.m1.1.1" xref="A4.313.313.285.217.149.81.13.2.2.2.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.313.313.285.217.149.81.13.2.2.2.1.m1.1b"><ci id="A4.313.313.285.217.149.81.13.2.2.2.1.m1.1.1.cmml" xref="A4.313.313.285.217.149.81.13.2.2.2.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.313.313.285.217.149.81.13.2.2.2.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.313.313.285.217.149.81.13.2.2.2.2" class="ltx_td ltx_align_center">0.48</td>
<td id="A4.313.313.285.217.149.81.13.2.2.2.3" class="ltx_td ltx_align_center">0.41</td>
<td id="A4.313.313.285.217.149.81.13.2.2.2.4" class="ltx_td ltx_align_center">0.0</td>
<td id="A4.313.313.285.217.149.81.13.2.2.2.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.313.313.285.217.149.81.13.2.2.2.6" class="ltx_td ltx_align_center">0.45</td>
<td id="A4.313.313.285.217.149.81.13.2.2.2.7" class="ltx_td ltx_align_center">0.49</td>
<td id="A4.313.313.285.217.149.81.13.2.2.2.8" class="ltx_td ltx_align_center">inf</td>
</tr>
<tr id="A4.314.314.286.218.150.82.14.3.3.3" class="ltx_tr">
<td id="A4.314.314.286.218.150.82.14.3.3.3.1" class="ltx_td ltx_align_left">GA<math id="A4.314.314.286.218.150.82.14.3.3.3.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.314.314.286.218.150.82.14.3.3.3.1.m1.1a"><mo stretchy="false" id="A4.314.314.286.218.150.82.14.3.3.3.1.m1.1.1" xref="A4.314.314.286.218.150.82.14.3.3.3.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.314.314.286.218.150.82.14.3.3.3.1.m1.1b"><ci id="A4.314.314.286.218.150.82.14.3.3.3.1.m1.1.1.cmml" xref="A4.314.314.286.218.150.82.14.3.3.3.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.314.314.286.218.150.82.14.3.3.3.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.314.314.286.218.150.82.14.3.3.3.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.314.314.286.218.150.82.14.3.3.3.3" class="ltx_td ltx_align_center">0.78</td>
<td id="A4.314.314.286.218.150.82.14.3.3.3.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.314.314.286.218.150.82.14.3.3.3.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.314.314.286.218.150.82.14.3.3.3.6" class="ltx_td ltx_align_center">0.34</td>
<td id="A4.314.314.286.218.150.82.14.3.3.3.7" class="ltx_td ltx_align_center">0.36</td>
<td id="A4.314.314.286.218.150.82.14.3.3.3.8" class="ltx_td ltx_align_center">56.91</td>
</tr>
<tr id="A4.315.315.287.219.151.83.15.4.4.4" class="ltx_tr">
<td id="A4.315.315.287.219.151.83.15.4.4.4.1" class="ltx_td ltx_align_left">
<span id="A4.315.315.287.219.151.83.15.4.4.4.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

GD<math id="A4.315.315.287.219.151.83.15.4.4.4.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.315.315.287.219.151.83.15.4.4.4.1.m1.1a"><mo stretchy="false" id="A4.315.315.287.219.151.83.15.4.4.4.1.m1.1.1" xref="A4.315.315.287.219.151.83.15.4.4.4.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.315.315.287.219.151.83.15.4.4.4.1.m1.1b"><ci id="A4.315.315.287.219.151.83.15.4.4.4.1.m1.1.1.cmml" xref="A4.315.315.287.219.151.83.15.4.4.4.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.315.315.287.219.151.83.15.4.4.4.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.315.315.287.219.151.83.15.4.4.4.2" class="ltx_td ltx_align_center">0.99</td>
<td id="A4.315.315.287.219.151.83.15.4.4.4.3" class="ltx_td ltx_align_center">0.81</td>
<td id="A4.315.315.287.219.151.83.15.4.4.4.4" class="ltx_td ltx_align_center">0.11</td>
<td id="A4.315.315.287.219.151.83.15.4.4.4.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.315.315.287.219.151.83.15.4.4.4.6" class="ltx_td ltx_align_center">0.29</td>
<td id="A4.315.315.287.219.151.83.15.4.4.4.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.315.315.287.219.151.83.15.4.4.4.8" class="ltx_td ltx_align_center">4.64</td>
</tr>
<tr id="A4.316.316.288.220.152.84.16.5.5.5" class="ltx_tr">
<td id="A4.316.316.288.220.152.84.16.5.5.5.1" class="ltx_td ltx_align_left">GD<math id="A4.316.316.288.220.152.84.16.5.5.5.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.316.316.288.220.152.84.16.5.5.5.1.m1.1a"><mo stretchy="false" id="A4.316.316.288.220.152.84.16.5.5.5.1.m1.1.1" xref="A4.316.316.288.220.152.84.16.5.5.5.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.316.316.288.220.152.84.16.5.5.5.1.m1.1b"><ci id="A4.316.316.288.220.152.84.16.5.5.5.1.m1.1.1.cmml" xref="A4.316.316.288.220.152.84.16.5.5.5.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.316.316.288.220.152.84.16.5.5.5.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.316.316.288.220.152.84.16.5.5.5.2" class="ltx_td ltx_align_center">0.93</td>
<td id="A4.316.316.288.220.152.84.16.5.5.5.3" class="ltx_td ltx_align_center">0.89</td>
<td id="A4.316.316.288.220.152.84.16.5.5.5.4" class="ltx_td ltx_align_center">0.05</td>
<td id="A4.316.316.288.220.152.84.16.5.5.5.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.316.316.288.220.152.84.16.5.5.5.6" class="ltx_td ltx_align_center">0.28</td>
<td id="A4.316.316.288.220.152.84.16.5.5.5.7" class="ltx_td ltx_align_center">0.58</td>
<td id="A4.316.316.288.220.152.84.16.5.5.5.8" class="ltx_td ltx_align_center">4.65</td>
</tr>
<tr id="A4.317.317.289.221.153.85.17.6.6.6" class="ltx_tr">
<td id="A4.317.317.289.221.153.85.17.6.6.6.1" class="ltx_td ltx_align_left">GD<math id="A4.317.317.289.221.153.85.17.6.6.6.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.317.317.289.221.153.85.17.6.6.6.1.m1.1a"><mo stretchy="false" id="A4.317.317.289.221.153.85.17.6.6.6.1.m1.1.1" xref="A4.317.317.289.221.153.85.17.6.6.6.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.317.317.289.221.153.85.17.6.6.6.1.m1.1b"><ci id="A4.317.317.289.221.153.85.17.6.6.6.1.m1.1.1.cmml" xref="A4.317.317.289.221.153.85.17.6.6.6.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.317.317.289.221.153.85.17.6.6.6.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.317.317.289.221.153.85.17.6.6.6.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.317.317.289.221.153.85.17.6.6.6.3" class="ltx_td ltx_align_center">0.71</td>
<td id="A4.317.317.289.221.153.85.17.6.6.6.4" class="ltx_td ltx_align_center">0.08</td>
<td id="A4.317.317.289.221.153.85.17.6.6.6.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.317.317.289.221.153.85.17.6.6.6.6" class="ltx_td ltx_align_center">0.54</td>
<td id="A4.317.317.289.221.153.85.17.6.6.6.7" class="ltx_td ltx_align_center">0.59</td>
<td id="A4.317.317.289.221.153.85.17.6.6.6.8" class="ltx_td ltx_align_center">4.99</td>
</tr>
<tr id="A4.318.318.290.222.154.86.18.7.7.7" class="ltx_tr">
<td id="A4.318.318.290.222.154.86.18.7.7.7.1" class="ltx_td ltx_align_left">
<span id="A4.318.318.290.222.154.86.18.7.7.7.1.1" class="ltx_ERROR undefined">\cdashline</span>1-8

RMU<math id="A4.318.318.290.222.154.86.18.7.7.7.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.318.318.290.222.154.86.18.7.7.7.1.m1.1a"><mo stretchy="false" id="A4.318.318.290.222.154.86.18.7.7.7.1.m1.1.1" xref="A4.318.318.290.222.154.86.18.7.7.7.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.318.318.290.222.154.86.18.7.7.7.1.m1.1b"><ci id="A4.318.318.290.222.154.86.18.7.7.7.1.m1.1.1.cmml" xref="A4.318.318.290.222.154.86.18.7.7.7.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.318.318.290.222.154.86.18.7.7.7.1.m1.1c">\rightarrow</annotation></semantics></math>FT</td>
<td id="A4.318.318.290.222.154.86.18.7.7.7.2" class="ltx_td ltx_align_center">1.0</td>
<td id="A4.318.318.290.222.154.86.18.7.7.7.3" class="ltx_td ltx_align_center">0.79</td>
<td id="A4.318.318.290.222.154.86.18.7.7.7.4" class="ltx_td ltx_align_center">0.13</td>
<td id="A4.318.318.290.222.154.86.18.7.7.7.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.318.318.290.222.154.86.18.7.7.7.6" class="ltx_td ltx_align_center">0.32</td>
<td id="A4.318.318.290.222.154.86.18.7.7.7.7" class="ltx_td ltx_align_center">0.57</td>
<td id="A4.318.318.290.222.154.86.18.7.7.7.8" class="ltx_td ltx_align_center">5.6</td>
</tr>
<tr id="A4.319.319.291.223.155.87.19.8.8.8" class="ltx_tr">
<td id="A4.319.319.291.223.155.87.19.8.8.8.1" class="ltx_td ltx_align_left">RMU<math id="A4.319.319.291.223.155.87.19.8.8.8.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.319.319.291.223.155.87.19.8.8.8.1.m1.1a"><mo stretchy="false" id="A4.319.319.291.223.155.87.19.8.8.8.1.m1.1.1" xref="A4.319.319.291.223.155.87.19.8.8.8.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.319.319.291.223.155.87.19.8.8.8.1.m1.1b"><ci id="A4.319.319.291.223.155.87.19.8.8.8.1.m1.1.1.cmml" xref="A4.319.319.291.223.155.87.19.8.8.8.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.319.319.291.223.155.87.19.8.8.8.1.m1.1c">\rightarrow</annotation></semantics></math>MEMIT</td>
<td id="A4.319.319.291.223.155.87.19.8.8.8.2" class="ltx_td ltx_align_center">0.97</td>
<td id="A4.319.319.291.223.155.87.19.8.8.8.3" class="ltx_td ltx_align_center">0.93</td>
<td id="A4.319.319.291.223.155.87.19.8.8.8.4" class="ltx_td ltx_align_center">0.03</td>
<td id="A4.319.319.291.223.155.87.19.8.8.8.5" class="ltx_td ltx_align_center">16.0</td>
<td id="A4.319.319.291.223.155.87.19.8.8.8.6" class="ltx_td ltx_align_center">0.3</td>
<td id="A4.319.319.291.223.155.87.19.8.8.8.7" class="ltx_td ltx_align_center">0.56</td>
<td id="A4.319.319.291.223.155.87.19.8.8.8.8" class="ltx_td ltx_align_center">5.58</td>
</tr>
<tr id="A4.320.320.292.224.156.88.20.9.9.9" class="ltx_tr">
<td id="A4.320.320.292.224.156.88.20.9.9.9.1" class="ltx_td ltx_align_left ltx_border_bb">RMU<math id="A4.320.320.292.224.156.88.20.9.9.9.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.320.320.292.224.156.88.20.9.9.9.1.m1.1a"><mo stretchy="false" id="A4.320.320.292.224.156.88.20.9.9.9.1.m1.1.1" xref="A4.320.320.292.224.156.88.20.9.9.9.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.320.320.292.224.156.88.20.9.9.9.1.m1.1b"><ci id="A4.320.320.292.224.156.88.20.9.9.9.1.m1.1.1.cmml" xref="A4.320.320.292.224.156.88.20.9.9.9.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.320.320.292.224.156.88.20.9.9.9.1.m1.1c">\rightarrow</annotation></semantics></math>LoRA</td>
<td id="A4.320.320.292.224.156.88.20.9.9.9.2" class="ltx_td ltx_align_center ltx_border_bb">1.0</td>
<td id="A4.320.320.292.224.156.88.20.9.9.9.3" class="ltx_td ltx_align_center ltx_border_bb">0.71</td>
<td id="A4.320.320.292.224.156.88.20.9.9.9.4" class="ltx_td ltx_align_center ltx_border_bb">0.05</td>
<td id="A4.320.320.292.224.156.88.20.9.9.9.5" class="ltx_td ltx_align_center ltx_border_bb">16.0</td>
<td id="A4.320.320.292.224.156.88.20.9.9.9.6" class="ltx_td ltx_align_center ltx_border_bb">0.29</td>
<td id="A4.320.320.292.224.156.88.20.9.9.9.7" class="ltx_td ltx_align_center ltx_border_bb">0.56</td>
<td id="A4.320.320.292.224.156.88.20.9.9.9.8" class="ltx_td ltx_align_center ltx_border_bb">12.83</td>
</tr>
</table>
</span></div>
</div>
</div>
<figcaption class="ltx_caption"><span class="ltx_tag ltx_tag_table"><span id="A4.322.322.294.226.158.90.22.12.2.1" class="ltx_text" style="font-size:90%;">Table 14</span>: </span><span id="A4.322.322.294.226.158.90.22.11.1" class="ltx_text" style="font-size:90%;">Detailed results for unlearning <math id="A4.322.322.294.226.158.90.22.11.1.m1.1" class="ltx_Math" alttext="\rightarrow" display="inline"><semantics id="A4.322.322.294.226.158.90.22.11.1.m1.1b"><mo stretchy="false" id="A4.322.322.294.226.158.90.22.11.1.m1.1.1" xref="A4.322.322.294.226.158.90.22.11.1.m1.1.1.cmml">→</mo><annotation-xml encoding="MathML-Content" id="A4.322.322.294.226.158.90.22.11.1.m1.1c"><ci id="A4.322.322.294.226.158.90.22.11.1.m1.1.1.cmml" xref="A4.322.322.294.226.158.90.22.11.1.m1.1.1">→</ci></annotation-xml><annotation encoding="application/x-tex" id="A4.322.322.294.226.158.90.22.11.1.m1.1d">\rightarrow</annotation></semantics></math> editing.</span></figcaption><div class="ltx_flex_figure">
<div class="ltx_flex_cell ltx_flex_size_1">
<div class="ltx_pagination ltx_figure_panel ltx_role_newpage"></div>
</div>
</div>
</figure>
</div>
</div>
</figure>
</div>
</div>
</figure>
</div>
</div>
</figure>
</div>
</div>
</figure>
</div>
</div>
</figure>
</div>
</div>
</figure>
</section>
</section>
</section>
</section>
</div>
</div>
</figure>
</section>
</section>
</div>
</div>
</figure>
</section>
</section>
</article>
</div>
<div class="ar5iv-footer"><a href="/html/2407.06482" class="ar5iv-nav-button ar5iv-nav-button-prev">◄</a>
    <a class="ar5iv-home-button" href="/"><img height="40" alt="ar5iv homepage" src="/assets/ar5iv.png"></a>
    <a href="/feeling_lucky" class="ar5iv-text-button">Feeling<br>lucky?</a>
    <a href="/log/2407.06483" class="ar5iv-text-button ar5iv-severity-error">Conversion<br>report</a>
    <a class="ar5iv-text-button" target="_blank" href="https://github.com/dginev/ar5iv/issues/new?template=improve-article--arxiv-id-.md&title=Improve+article+2407.06483">Report<br>an issue</a>
    <a href="https://arxiv.org/abs/2407.06483" class="ar5iv-text-button arxiv-ui-theme">View&nbsp;original<br>on&nbsp;arXiv</a><a href="/html/2407.06484" class="ar5iv-nav-button ar5iv-nav-button-next">►</a>
</div><footer class="ltx_page_footer">
<a class="ar5iv-toggle-color-scheme" href="javascript:toggleColorScheme()" title="Toggle ar5iv color scheme"><span class="color-scheme-icon"></span></a>
<a class="ar5iv-footer-button" href="https://arxiv.org/help/license" target="_blank">Copyright</a>
<a class="ar5iv-footer-button" href="https://arxiv.org/help/policies/privacy_policy" target="_blank">Privacy Policy</a>

<div class="ltx_page_logo">Generated  on Mon Aug  5 15:32:25 2024 by <a target="_blank" href="http://dlmf.nist.gov/LaTeXML/" class="ltx_LaTeXML_logo"><span style="letter-spacing:-0.2em; margin-right:0.1em;">L<span class="ltx_font_smallcaps" style="position:relative; bottom:2.2pt;">a</span>T<span class="ltx_font_smallcaps" style="font-size:120%;position:relative; bottom:-0.2ex;">e</span></span><span style="font-size:90%; position:relative; bottom:-0.2ex;">XML</span><img src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAsAAAAOCAYAAAD5YeaVAAAAAXNSR0IArs4c6QAAAAZiS0dEAP8A/wD/oL2nkwAAAAlwSFlzAAALEwAACxMBAJqcGAAAAAd0SU1FB9wKExQZLWTEaOUAAAAddEVYdENvbW1lbnQAQ3JlYXRlZCB3aXRoIFRoZSBHSU1Q72QlbgAAAdpJREFUKM9tkL+L2nAARz9fPZNCKFapUn8kyI0e4iRHSR1Kb8ng0lJw6FYHFwv2LwhOpcWxTjeUunYqOmqd6hEoRDhtDWdA8ApRYsSUCDHNt5ul13vz4w0vWCgUnnEc975arX6ORqN3VqtVZbfbTQC4uEHANM3jSqXymFI6yWazP2KxWAXAL9zCUa1Wy2tXVxheKA9YNoR8Pt+aTqe4FVVVvz05O6MBhqUIBGk8Hn8HAOVy+T+XLJfLS4ZhTiRJgqIoVBRFIoric47jPnmeB1mW/9rr9ZpSSn3Lsmir1fJZlqWlUonKsvwWwD8ymc/nXwVBeLjf7xEKhdBut9Hr9WgmkyGEkJwsy5eHG5vN5g0AKIoCAEgkEkin0wQAfN9/cXPdheu6P33fBwB4ngcAcByHJpPJl+fn54mD3Gg0NrquXxeLRQAAwzAYj8cwTZPwPH9/sVg8PXweDAauqqr2cDjEer1GJBLBZDJBs9mE4zjwfZ85lAGg2+06hmGgXq+j3+/DsixYlgVN03a9Xu8jgCNCyIegIAgx13Vfd7vdu+FweG8YRkjXdWy329+dTgeSJD3ieZ7RNO0VAXAPwDEAO5VKndi2fWrb9jWl9Esul6PZbDY9Go1OZ7PZ9z/lyuD3OozU2wAAAABJRU5ErkJggg==" alt="Mascot Sammy"></a>
</div></footer>
</div>

    <script>
      var canMathML = typeof(MathMLElement) == "function";
      if (!canMathML) {
        var body = document.querySelector("body");
        body.firstElementChild.setAttribute('style', 'opacity: 0;');
        var loading = document.createElement("div");
        loading.setAttribute("id", "mathjax-loading-spinner");
        var message = document.createElement("div");
        message.setAttribute("id", "mathjax-loading-message");
        message.innerText = "Typesetting Equations...";
        body.prepend(loading);
        body.prepend(message);

        var el = document.createElement("script");
        el.src = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js";
        document.querySelector("head").appendChild(el);

        window.MathJax = {
          startup: {
            pageReady: () => {
              return MathJax.startup.defaultPageReady().then(() => {
                body.removeChild(loading);
                body.removeChild(message);
                body.firstElementChild.removeAttribute('style');
              }); } } };
      }
    </script>
    <script>
    // Auxiliary function, building the preview feature when
    // an inline citation is clicked
    function clicked_cite(e) {
      e.preventDefault();
      let cite = this.closest('.ltx_cite');
      let next = cite.nextSibling;
      if (next && next.nodeType == Node.ELEMENT_NODE && next.getAttribute('class') == "ar5iv-bibitem-preview") {
        next.remove();
        return; }
      // Before adding a preview modal,
      // cleanup older previews, in case they're still open
      document.querySelectorAll('span.ar5iv-bibitem-preview').forEach(function(node) {
        node.remove();
      })

      // Create the preview
      preview = document.createElement('span');
      preview.setAttribute('class','ar5iv-bibitem-preview');
      let target = document.getElementById(this.getAttribute('href').slice(1));
      target.childNodes.forEach(function (child) {
        preview.append(child.cloneNode(true));
      });
      let close_x = document.createElement('button');
      close_x.setAttribute("aria-label","Close modal for bibliography item preview");
      close_x.textContent = "×";
      close_x.setAttribute('class', 'ar5iv-button-close-preview');
      close_x.setAttribute('onclick','this.parentNode.remove()');
      preview.append(close_x);
      preview.querySelectorAll('.ltx_tag_bibitem').forEach(function(node) {
        node.remove();
      });
      cite.parentNode.insertBefore(preview, cite.nextSibling);
      return;
    }
    // Global Document initialization:
    // - assign the preview feature to all inline citation links
    document.querySelectorAll(".ltx_cite .ltx_ref").forEach(function (link) {
      link.addEventListener("click", clicked_cite);
    });
    </script>
    </body>
</html>

2407.06483
Test-time interventions for language models can enhance factual accuracy, mitigate harmful outputs, and improve model efficiency without costly retraining.
But despite a flood of new methods, different types of interventions are largely developing independently.
In practice, multiple interventions must be applied sequentially to the same model, yet we lack standardized ways to study how interventions interact.
We fill this gap by introducingcomposable interventions, a framework to study the effects of using multiple interventions on the same language models, featuring new metrics and a unified codebase.
Using our framework, we conduct extensive experiments and compose popular methods from three emerging intervention categories—knowledge editing,model compression, andmachine unlearning.
Our results from 310 different compositions uncover meaningful interactions: compression hinders editing and unlearning, composing interventions hinges on their order of application, and popular general-purpose metrics are inadequate for assessing composability.
Taken together, our findings showcase clear gaps in composability, suggesting a need for new multi-objective interventions.111All of our code is public:github.com/hartvigsen-group/composable-interventions††Correspondence:{arinbjorn,hartvigsen}@virginia.edu,kyobrien@microsoft.com
abstract
Composable Interventions for Language Models
Abstract
2407.06483
1IntroductionLanguage models (LMs)  exhibit striking capabilities on important tasks in myriad domains including medicine, finance, science, and entertainment. But despite high performance, deployed LMs can still misbehave unpredictably and require updates.
For example, LMs generate content that ishallucinatory,factually incorrect, andharmful.
Beyond unwanted behaviors, user requirements also change over time.
For example, regulations arise, computational resources constrict, knowledge gets outdated, and copyrighted training materials are identified.
Without ways to quickly address these issues, models can be left miscalibrated, outdated, and biased, limiting their widespread responsible use.Many recent works study efficient, in-place updates for LMs.
We broadly refer to these asinterventions—updates to targeted properties of LMs applied after pretraining (and optional fine-tuning) .
For example, we can view model compressionas an intervention to make language models more inference- or memory-efficient.
Other rapidly-advancing examples include knowledge editing, detoxification, and unlearning.
However, these interventions are largely advancing independently.
In practice, we usually havemultiplerequirements for properties of our models (e.g.,factualityandefficiency) .
And as requirements change over time, new interventions must be applied.
While some works have started studying interactions between training objectives, practical widespread use is limited without unified evaluations for how interventions interact.As depicted in Figure1, we propose that practical interventions should becomposable: When an intervention is applied to a model, it should not interfere with prior or future interventions.
For example, we might repair a factual error in an LM using a knowledge editing method.
Later on, we might quantize that model.
Acomposablequantization method should specifically preserve the edited fact and a composable editor should withstand quantization.
However, it remains unknown how well existing interventions compose, and we lack formal notions of intervention composition.
We therefore propose two metrics for composability: 1) Order-free Error, where an intervention is composable if its application leaves others’ success unimpacted, and 2) Order Sensitivity, where the combined success of multiple interventions should not depend on the order in which they are applied.
We instantiate these metrics in a codebase that includes popular interventions, tackling a major lack of standardized code that has inhibited cross-intervention evaluations until now.Figure 1:Interventions aim to update targeted properties of language models without impacting unrelated behaviors or adding excessive compute. We introduce and extensively experiment withcomposabilityof different interventions used on the same model.We use our framework to extensively study composability of state-of-the-artknowledge editing,model compression, andmachine unlearninginterventions on Llama3-8B.
Our experiments with 310 different compositions unearth novel insights and offer guidance for composing recent methods.
Three key results are as follows:1Model compression often limits the success of other interventions.
This suggests a significant drawback of existing compression methods, which are nearly universal in LM deployments.2The order in which interventions are applied dramatically alters their success.
Therefore, we need new interventions explicitly designed for composability.3General-purpose post-intervention model performance is a poor proxy for composability.
This indicates that targeting composability as a metric has the potential to drive the development of new, practically-grounded interventions.
Taken alongside our other findings (Section4) , our experiments suggest we sorely need to broaden intervention method evaluations and to design new methods explicitly for composability.Our key contributions are as follows:•We introduce the notion ofcomposabilityto language model interventions, formalizing the sequential applications of multiple interventions to LMs. Our work opens doors to address crucial, practical challenges in online LM updates and broadens evaluation criteria for interventions.•Our main contribution is extensive experimentation, identifying unknown interactions between knowledge editing, model compression, and machine unlearning.•Our findings suggest a clear need to develop novel interventions that target composability, a crucial property of practical interventions.•We release an extendable codebase that unifies many state-of-the-art implementations to enable others to develop new multi-objective interventions.
introduction
Composable Interventions for Language Models
1Introduction
2407.06483
2An Evaluation Framework for Composable Interventions2.1Preliminaries: Interventions for Language ModelsAssume we are given a language model, wherecontains its pre-trained parameters andis the parameter space.
We define aninterventionto be the combination of four things:
1)  an operatorthat produces a new model,
2)  hyperparameters, which typically control the strength of intervention,
3)  a lossthat specifies the targeted update,222Some interventions require data, some do not. We remove direct data dependencies for simplicity.and 4)  evaluation criteria, where higher scores indicate better performance.
Each intervention’s operatoraims to minimize lossand ultimately achieve high performance according to.
For readability, we denote the application of interventionto modelas.2.2Composable Interventions for Language ModelsNow considerpossible interventions.
We define interventioncompositionas the application of multiple interventions to a modelin sequence.333For readability, we describe our framework in terms of two interventions.For example, we might apply interventionto model, thento the intervened-upon model.
We can write this composition as eitheror, and here we use the latter format.Most interventionsare developed specifically for their own criterion.
For simplicity, we let each intervention have one criterion, though in practice they often have multiple.
We propose that the quality of many interventions in practice should also relate tohow an intervention impacts our ability to apply other interventions.
To meet this need, we propose two metrics that measure suchcomposability.
Each is computed for a given criterionand can be computed for a single hyperparameter setting or ranged over multiple settings to attain a richer description of intervention interactions.Our evaluation strategy is straightforward: After applying an intervention, we measure its success as.
Then, we can add another interventionand measure.
The difference between these measures then elicits howimpacts the success ofaccording to.
To disentangle absolute performance onfrom the interventions, we explore the impact oforderingby reversing the interventions and computing.
By pairing absolute changes with results from different orderings, we isolate direct interactions between interventions.
Overall, this approach provides guidance when choosing interventions to compose and illuminates failure modes of existing methods.
We next describe two concrete composability metrics.
The first describes best-case composability regardless of order, and the second isolates the impact of ordering.Composability Error Metric 1: Order-free Error.Interventions are more composable if they achieve high performance when applied in either order.
We therefore measure the impacts of adding a second intervention to a model regardless of order for a chosen criterionusingorder-free error, computed as the areaabovethe maximum of two curves.
A small area means higher overall performance on the metric.
We use the maximum criterion value as an upper bound for this area, and assume it is 1, though this is easily changed in practice.
Each curve corresponds to an order in which each intervention is applied and sweeps over hyperparameter choices for:(1) A smaller area indicates better composability.
By sweeping over hyperparameters, this measure provides a general sense of composability without choosing hyperparameters ahead of time.
The area can be computed for any criteria—in our experiments, we evaluate all criteria for each composition.
In practice, not all hyperparameters are practical.
For example, harsh model compression can have catastrophic impacts on model utility and so comparing interventions at extreme levels may be unrealistic.
Therefore, we can also set a bound on acceptable performance decay with respect toor any other criterion when using Equation1and can also easily be measured for hand-chosen hyperparameters, and for a single setting, Order-free Error reduces to the simple error score:.Composability Error Metric 2: Order Sensitivity.Interventions are more composable if their performance is invariant to their order of application.
We measureorder invariancefor criterionas the areabetweenthe two curves defined by each direction of intervention application.
Again, we compute this across hyperparameter choices for:(2) A smaller area again indicates better composability.
And again, if hyperparameters are already chosen, this simplifies to a simple error:.Overall, measuring Order-free Error and Order Sensitivity provides the first insights into intervention composability.
Each reduces to a simple form for point measures of the impact of composition.
While more computationally costly, the broader sweeps over hyperparameters enable clearer pictures of how interventions interact.
As shown in Section3, the curves produced by these sweeps also enable intuitive, visual understandings of composability.
body
Composable Interventions for Language Models
2An Evaluation Framework for Composable Interventions
2407.06483
3Experimental SetupWe study the effects of sequentially applying multiple interventions to the same model on performance as measured by intervention-specific metrics and overall utility. Understanding these effects is crucial for the practical application of test-time interventions. The following sections detail our experiments to illuminate how composable various interventions are across model editing, unlearning, and compression.3.1Intervention Datasets, Methods, and MetricsWe next detail the ten techniques selected from the three categories of interventions, along with their task-specific and overall utility metrics. AppendixBfurther describes the backgrounds of the three types of intervention and Table1summarizes our experimental setup.InterventionsMethodsDatasetsInterventionMetricsComposabilityMetricsKnowledgeEditingFinetuning, LoRA,MEMITzsREEdit Success, Edit Generalization,Strict Edit Locality, MMLUOrder-free Error(Equation1) Order Sensitivity(Equation2) ModelCompressionPruning:Wanda, SparseGPTQuantization:GPTQ, AWQ–MMLUMachineUnlearningGradient Ascent (GA) ,Gradient Difference (GD) ,Representation MisdirectionUnlearning (RMU) WMDPWMDP,MMLUTable 1:Summary of the methods, data, and metrics used in our experiments. All experiments are conducted with Llama3-8B. Composability metrics apply to all experiments.Knowledge Editing.LMs’ knowledge can be fundamentally incorrect or grow outdated. Knowledge editing aims to updatefactsaccording to LMs, typically framed as question–answering.
Here, where models are edited to return updated answers to questions by increasing the likelihood of the correct answers.
We use three editors, including the popular MEMIT, as well as LoRAand finetuning, which have recently been shown to be surprisingly successful editors despite their simplicity.
We use the zsREdataset, which is a popular question-answering benchmark for knowledge editing.To evaluate editors’ performance, we use three standard metrics:Edit successmeasures whether the post-edit model successfully outputs the correct, edited response.Edit generalizationmeasures how well the edit generalizes to other semantically-equivalent inputs. Following recent work, we compute generalization as the average edit success across a set of 10 holdout edit rephrasings.Strict Edit localitymeasures the impact of edits on unrelated inputs as the average success on random, unedited samples.
All editing metrics are computed as the F1 score on the correct output logits as a strict measure of editing performance—intended outputs must be the most-likely tokens after editing.
Each metric’s highest possible value is 1.0.Machine Unlearning.LMs can learn undesirable knowledge from their pretraining data, including information that is potentially dangerouscopyrighted,
toxic,
or memorized.
Machine unlearning methods aim to remove undesirable knowledge from LMs without regressing performance on extraneous domains. We experiment with three popular and representative methods: Gradient Ascent (GA) , Gradient Difference (GD) , and Representation Misdirection for Unlearning (RMU) .
GA is a traditional approach that minimizes the likelihood of chosen unlearning targets.
GD augments GA by also maintaining the LM’s predictions on unrelated retention data.
RMU, the most recent method, perturbs the model’s representations for inputs related to the learning target.
Further details are in AppendixC.4.We evaluate unlearning with the recent QA dataset Weapons of Mass Description Proxy (WMDP) , updating LMs to forget potentially hazardous biosecurity and cybersecurity knowledge.
We average the performance on WMDP’s cyber and bio splits, totaling 3,260 questions. We report individual split performance in AppendixD.
Optimal unlearning yields 25% accuracy as this is random performance on the four-choice benchmark.Model Compression.LMs require significant compute resources during pretraining, finetuning, and inference.
Model compression methods aim to reduce resources needed at test-time.
We experiment with four popular, state-of-the-art weight pruning and quantization methods.
For pruning, we choose SparseGPTand Wanda.
For quantization, we choose GPTQand AWQ.
Pruning and quantization are used for different reasons, so we generally avoid comparisons between the two approaches.
Each algorithm has a hyperparameter that controls how much to compress the model—% of zeroed weights for pruning,bitsfor quantization.
Naturally, efficiency grows alongside compression, but at the expense of task performance.
We vary this hyperparameter to explore the impacts of various levels of compression, choosing sparsity levels of .25, .35, .45, .55, .65, and .75 for pruning and 2, 3, 4, and 8 bits for quantization.
The uncompressed model is 16 bits.
Editing or unlearning some compressed models requires decompression, in which case we recompress afterwards using the same compression technique.Measuring General Model Utility.All interventions aim to avoid degrading overall model utility.
While assessing general performance of LMs is an active area, we make the standard choice to evaluate question–answering accuracy on MMLUusing the LM Eval Harness.
Accuracy of 25% indicates random predictions.3.2Implementation detailsAll experiments are performed with Llama3-8B, which is well-studied and highly capable for its parameter count.
All results for editing methods are averaged over 10 batches of 50 random edits from zsRE.
Further details on implementations and hyperparameters are in AppendixC.
body
Composable Interventions for Language Models
3Experimental Setup
2407.06483
4ResultsIn the following subsections, we first examine how each pair of interventions compose individually, then summarize general trends.
We measure composability using our metrics from Section2for each intervention’s criteria and report impacts of compression at different levels.
All intervention pairs are applied in both directions (e.g.,MEMIT before AWQ and AWQ before MEMIT) .
For composability metrics, we compare methods regardless of metric ranges by counting the number of times each intervention method outperforms the others (# Wins)  with ties counting as losses.4.1Composing Model Compression with Knowledge Editing(a) (b) Figure 2:Composing Knowledge Editing with Model Compression with varying degrees of compression.
Higher values are better for all metrics.Key Takeaways:Editing post-compression generally outperforms the reverse order, as compression tends to degrade edit performance and all editors exhibit order sensitivity.Edit SuccessEdit GeneralizationOrder-free Error () Order Sensitivity () Order-free Error () Order Sensitivity () MethodFTMEMITLoRAFTMEMITLoRAFTMEMITLoRAFTMEMITLoRA# WinsWanda.01.05.05.03.24.08.20.10.46.04.21.016SparseGPT.01.09.14.03.04.01.23.14.48.02.03.074\cdashline1-14AWQ.01.07.00.04.01.08.16.11.26.05.00.1412GPTQ.21.16.42.36.03.37.40.20.59.36.11.270# Wins211121040131Strict Edit LocalityMMLUOrder-free Error () Order Sensitivity () Order-free Error () Order Sensitivity () MethodFTMEMITLoRAFTMEMITLoRAFTMEMITLoRAFTMEMITLoRA# WinsWanda.89.97.96.04.00.01.40.41.40.00.00.000SparseGPT.89.96.96.03.00.00.40.40.40.00.00.004\cdashline1-14AWQ.85.96.93.04.00.03.41.41.41.02.01.005GPTQ.89.96.95.07.00.01.41.41.42.00.01.012# Wins400040000101Table 2:Composability metrics for Knowledge Editing composed with Model Compression. Order-free Error (↓)  measures best-case error caused by interventions, while Order Sensitivity (↓)  measures unwanted effects of intervention ordering.
# Wins denotes the number of times each method outperforms other methods of its intervention category, counting all ties as losses.
We use 25% sparsity and 4-bit quantization to showcase practical compression levels.Key Takeaways:AWQ composes better than GPTQ and LoRA is surprisingly non-composable.Our results are shown in Figure2, where we find that model compression and knowledge editing interact in meaningful ways at different levels of compression.
We report three editing metrics and MMLU to measure general model degradation. All results are averaged (mean)  over 10 batches of 50 edits each.
Illustrated below, these experiments reveal that there is a large gap in composability of existing methods. Our findings are as follows.1Model compression degrades editing performance.Across Figure2, we see that edits made by MEMIT, LoRA, and Finetuning deteriorate as compression increases.
In weight pruning, the editing metrics, Success, Generalization, and Locality, steadily drop with higher sparsity.
For quantization, the steepest drops occur between 2 and 4 Bits, and GPTQ decays editing performance faster than AWQ.
While general performance decay from compression is unsurprising, the steepness of the decay differs dramatically between editors and compression methods.
Decay curves also depend on the order of interventions.
We also note that when compression is applied after editing, theonlything that can impact the editing metrics is the compression.
Interestingly, Finetuning is also a surprisingly strong baseline at higher levels of compression, often achieving better editing performance than MEMIT or LoRA.
Overall, these results suggests that model compression can limit a model’s editability.2Editing performance hinges on the order of interventions.The shaded area between the curves in Figure2shows the impact of ordering.
Interestingly, the ordering is highly impactful in most cases and varies dramatically between editing methods.
For pruning methods (left side) , compressing first is generally better than editing first.
For example, pruning after Finetuning or LoRA destroys their edits much faster than if the model had been compressed first.
For quantization (right side) , the order also clearly matters.
However, the best order differs by quantization method.
For GPTQ, editing should be done first.
For AWQ, compression should be done first.
This finding suggests that these interventions interact in meaningful ways and successful composition will benefit from tailored, composable methods that have expected behaviors.3Composability can vary within the same intervention category.We next summarize the general composability of each pair of interventions in Table4.1, using the metrics proposed in Section2: Order-free Error ()  measures best-case composition error, while Order Sensitivity ()  measures unwanted effects of intervention ordering.
These metrics let us compare methods according to composability, and we find that 1)  MEMIT is generally the most composable editor, totaling 14 wins compared to 9 by Finetuning and 4 by LoRA, 2)  SparseGPT is the most composable pruning method with 8 wins compared to Wanda’s 6, and 3)  AWQ far surpasses GPTQ, achieving 17 wins compared to GPTQ’s 2.
However, many wins are quite close and composability can change dramatically between metrics.
For example, Finetuning has better Order-free Error than MEMIT on 2/3 metrics but underforms MEMIT on Order Sensitivity for all metrics, so MEMIT’s edits are more robust.
These results indicate a need to use many metrics to explicitly target and evaluate composability.4Overall utility evaluations fail to measure composability.Despite similar MMLU scores, methods vary significantly in composability. For example, Table4.1shows that MEMIT and Wanda achieve near perfect Order Sensitivity on MMLU but have 0.24 Order Sensitivity on Edit Success.
This is further backed by Figure2, where all compositions look very similar according to MMLU, but not according to editing metrics.
The disparity between overall utility and composability metrics underscores the need for more nuanced evaluations explicitly targeting composability as a design criterion.4.2Composing Model Compression with Machine UnlearningWe next compose three model compression and three machine unlearning interventions.
We measure unlearning success with WMDP (Section3.1) .(a) (b) Figure 3:Composing Machine Unlearning with Model Compression at different levels of compression.
Lower values are better for WMDP, while higher values are better for MMLU.Key Takeaways:Unlearning should generally be appliedbeforecompression, and performance varies significantly by composition.5Compression hinders unlearning.Our main results in Figure3show that applying unlearning to a compressed model can degrade performance. With RMU, pruning the model before unlearning can significantly regress unlearning performance, especially at higher sparsity levels. We observe a similar trend with quantization when composing RMU and GPTQ. With GD, we similarly find that pruning after unlearning is best, though quantizing first can be optimal. We observe the opposite trend with GA, though GA’s poor unlearning performance, as seen in other studies, is a possible confounder. We overall find that unlearning a compressed model is more challenging but that the degree to which unlearning after compression hurts performance is highly contingent on the composition and compression level.6Order Sensitivity can determine overall composability.Table4.3reports the composability metrics of each unlearning technique. As measured by Order-free Error on WMDP, GD and RMU outperform GA, which struggles to unlearn without significantly degrading utility. GD exhibits higher WMDP Order Sensitivity for pruning than quantization, whereas RMU has low Order Sensitivity across all techniques except when quantized by GPTQ. Despite both GD and RMU’s successful unlearning, RMU has lower overall Order Sensitivity than GD, suggesting that while task performance is critical, it alone does not fully capture the practical challenges of applying interventions—RMU is more robust to order.4.3Composing Knowledge Editing with Machine UnlearningWe next examine the composability between editing and unlearning. Both categories of interventions aim to modify the model’s knowledge without degrading overall utility. The need to keep LMs factually updated while forgetting undesirable knowledge makes pairing editing and unlearning a practical composition.WMDP (Unlearning) MMLUOrder-free Error () Order Sensitivity () Order-free Error () Order Sensitivity () MethodGAGDRMUGAGDRMUGAGDRMUGAGDRMU# WinsWanda.46.35.29.03.17.03.46.42.43.02.01.014SparseGPT.43.28.28.03.22.02.49.42.43.06.09.014\cdashline1-14AWQ.43.24.27.02.00.01.53.58.44.02.07.015GPTQ.34.24.27.09.02.18.52.44.42.09.24.054# Wins021021022003Table 3:Composability metrics for machine unlearning and model compression.Key Takeaways:GD and RMU have overall comparable performance and far outperform GA. RMU is less sensitive to ordering than GD.Edit SuccessEdit GeneralizationOrder-free Error () Order Sensitivity () Order-free Error () Order Sensitivity () MethodFTMEMITLoRAFTMEMITLoRAFTMEMITLoRAFTMEMITLoRA# WinsGA.93.52.00.07.481.0.96.59.22.04.41.781GD.01.07.00.67.40.56.19.11.29.56.41.480RMU.00.03.00.01.01.00.18.07.29.03.04.0410# Wins003111021210WMDP (Unlearning) MMLUOrder-free Error () Order Sensitivity () Order-free Error () Order Sensitivity () MethodFTMEMITLoRAFTMEMITLoRAFTMEMITLoRAFTMEMITLoRA# WinsGA.47.40.28.00.05.07.47.51.64.01.04.071GD.29.26.30.00.02.24.41.42.41.18.22.144RMU.28.29.29.04.01.00.43.44.44.01.00.045# Wins111201200111Table 4:Composability metrics for Machine Unlearning composed with Model Editing. We study compression at 25% sparsity and 4-bit quantization. Note that lower values of WMDP MCS indicate more unlearning and therefore better performance while higher values are better for the other metrics’ MCS.Key Takeaways:Editing and unlearning are composable for GD and RMU, with RMU having the lowest Order Sensitivity. These results demonstrate that different techniques can modify LMs’ knowledge sequentially.7Editing and unlearning are highly composable for some unlearning methods.Table4shows that the editing metrics’ Order-free Error and Order Sensitivity vary greatly between unlearning techniques. RMU has a lower overall Order-free Error and Order Sensitivity than every other unlearning technique across all editors. Similar to Section4.2, we find that GD can perform comparably to RMU but with a worse Order Sensitivity. GA’s catastrophic forgetting makes it unsuitable for composition. Excluding GA, editing does not disrupt the model’s ability to forget the unlearning target, possibly due to knowledge editing’s relatively surgical modification to the LM and targeting knowledge semantically distinct from the WMDP unlearning target.
Together, these results imply that editing and unlearning can be composable, but composability depends most on the unlearning technique.4.4General Trends in ComposabilityWe finally summarize trends revealed when considering all findings.Compression consistently hinders other interventions.Compressing an edited model can regress editing performance. Applying unlearning to a compressed model is often more challenging than unlearning the base model. These trends suggest that compression may modify how models encode knowledge in a way that makes targeted updates more challenging. Previous works have studied how compression can cause models to forget some knowledge. Others have studied how editing modifies model internals in a way that makes the model more fragile. However, the relationships betweenhow models are compressedandhow knowledge is modifiedremain underexplored. An improved understanding of these dynamics may help guide more composable interventions and shed further light on how LMs encode knowledge.RMU is the most composable unlearning technique.RMU has comparable unlearning performance to GD without regressing overall utility as harshly and while being more order invariant. A possible factor is the RMU training objective, which directs activations for inputs related to the unrelating to a random direction in activation space while minimally modifying unrelated activations.
RMU achieves strong performance, even when only modifying the LM’s third layer.
These results suggest that intervening on model representationsis a promising direction for effective and composable interventions.MMLU is insufficient for measuring composability.MMLU is a common measure of a model’s general performance.
However, similar scores on MMLU can mask significant differences in intervention performance.
Across the board, interventions appear far more composable on MMLU than other metrics.
We therefore advocate for thorough multi-metric and multi-dataset evaluations for composable interventions.5ConclusionWe conduct an extensive study of composing test-time interventions for language models.
Such interventions are an important step towards keeping expensively trained models up-to-date over extended deployments with respect to factuality, efficiency, and social responsibility.
Our study lays the foundation for a new range of intervention methods that arecomposablewith one another, proposing two metrics for measuring the composability of interventions.
Our setup aligns with the practical need to update the same models multiple times as use cases evolve and training remains expensive.
We deploy this new framework to investigate composability between popular interventions spanning knowledge editing, machine unlearning, and model compression.
We show that these interventions interact in non-trivial ways, pointing to concrete directions for future work.
We further discover guidance for current practice: model compression harms other interventions, unlearning compressed models is especially hard, and the order in which interventions are applied changes their outcomes substantially.
Our composability metrics also pave the way for formal investigations and practical principles.
Finally, our framework is general and naturally extends to a broad range of test-time interventions.AcknowledgementsWe thank the University of Virginia Research Computing team for providing access to excellent high-performance computing resources.References[1]K. Singhal, T. Tu, J. Gottweis, R. Sayres, E. Wulczyn, L. Hou, K. Clark, S. Pfohl, H. Cole-Lewis, D. Neal,et al., “Towards expert-level medical question answering with large language models,”arXiv preprint arXiv:2305.09617, 2023.[2]S. Wu, O. Irsoy, S. Lu, V. Dabravolski, M. Dredze, S. Gehrmann, P. Kambadur, D. Rosenberg, and G. Mann, “Bloomberggpt: A large language model for finance,”arXiv preprint arXiv:2303.17564, 2023.[3]R. Taylor, M. Kardas, G. Cucurull, T. Scialom, A. Hartshorn, E. Saravia, A. Poulton, V. Kerkez, and R. Stojnic, “Galactica: A large language model for science,”arXiv preprint arXiv:2211.09085, 2022.[4]S. Zhong, Z. Huang, S. Gao, W. Wen, L. Lin, M. Zitnik, and P. Zhou, “Let’s think outside the box: Exploring leap-of-thought in large language models with creative humor generation,”arXiv preprint arXiv:2312.02439, 2023.[5]Z. Ji, N. Lee, R. Frieske, T. Yu, D. Su, Y. Xu, E. Ishii, Y. J. Bang, A. Madotto, and P. Fung, “Survey of hallucination in natural language generation,”ACM Computing Surveys, vol. 55, no. 12, pp. 1–38, 2023.[6]Y. Zhao, J. Zhang, I. Chern, S. Gao, P. Liu, J. He,et al., “Felm: Benchmarking factuality evaluation of large language models,”Advances in Neural Information Processing Systems, vol. 36, 2023.[7]J. Mendelsohn, R. L. Bras, Y. Choi, and M. Sap, “From dogwhistles to bullhorns: Unveiling coded rhetoric with language models,”arXiv preprint arXiv:2305.17174, 2023.[8]D. Jain, P. Kumar, S. Gehman, X. Zhou, T. Hartvigsen, and M. Sap, “PolygloToxicityPrompts: Multilingual evaluation of neural toxic degeneration in large language models,”arXiv preprint arXiv:2405.09373, 2024.[9]T. Hartvigsen, S. Gabriel, H. Palangi, M. Sap, D. Ray, and E. Kamar, “Toxigen: A large-scale machine-generated dataset for adversarial and implicit hate speech detection,” inAnnual Meeting of the Association for Computational Linguistics, 2022.[10]The President of the United States, “Executive order 14110 of october 30, 2023: Safe, secure, and trustworthy development and use of artificial intelligence.” Federal Register, 2023.Published November 1, 2023.[11]J. Tack, J. Kim, E. Mitchell, J. Shin, Y. W. Teh, and J. R. Schwarz, “Online adaptation of language models with a memory of amortized contexts,”arXiv preprint arXiv:2403.04317, 2024.[12]M. Grynbaum and R. Mac, “The times sues openai and microsoft over a.i. use of copyrighted work,”The New York Times, 2023.[13]J. Kaddour, J. Harris, M. Mozes, H. Bradley, R. Raileanu, and R. McHardy, “Challenges and applications of large language models,”arXiv preprint arXiv:2307.10169, 2023.[14]X. Zhu, J. Li, Y. Liu, C. Ma, and W. Wang, “A survey on model compression for large language models,”arXiv preprint arXiv:2308.07633, 2023.[15]E. Frantar, S. Ashkboos, T. Hoefler, and D. Alistarh, “GPTQ: Accurate post-training quantization for generative pre-trained transformers,” inInternational Conference on Learning Representations, 2023.[16]E. Frantar and D. Alistarh, “SparseGPT: Massive language models can be accurately pruned in one-shot,” inInternational Conference on Machine Learning, 2023.[17]V. Mazzia, A. Pedrani, A. Caciolai, K. Rottmann, and D. Bernardi, “A survey on knowledge editing of neural networks,”arXiv preprint arXiv:2310.19704, 2023.[18]T. Hartvigsen, S. Sankaranarayanan, H. Palangi, Y. Kim, and M. Ghassemi, “Aging with grace: Lifelong model editing with discrete key-value adaptors,” inAdvances in Neural Information Processing Systems, 2023.[19]K. Meng, D. Bau, A. Andonian, and Y. Belinkov, “Locating and editing factual associations in gpt,” inAdvances in Neural Information Processing Systems, 2022.[20]K. Meng, A. S. Sharma, A. Andonian, Y. Belinkov, and D. Bau, “Mass-editing memory in a transformer,” inInternational Conference on Learning Representations, 2023.[21]L. Yu, Q. Chen, J. Zhou, and L. He, “Melo: Enhancing model editing with neuron-indexed dynamic lora,” inProceedings of the AAAI Conference on Artificial Intelligence, vol. 38, pp. 19449–19457, 2024.[22]C. Tan, G. Zhang, and J. Fu, “Massive editing for large language models via meta learning,” inInternational Conference on Learning Representations, 2024.[23]J. Welbl, A. Glaese, J. Uesato, S. Dathathri, J. Mellor, L. A. Hendricks, K. Anderson, P. Kohli, B. Coppin, and P.-S. Huang, “Challenges in detoxifying language models,” inFindings of the Association for Computational Linguistics: EMNLP 2021, pp. 2447–2469, 2021.[24]C. Yu, S. Jeoung, A. Kasi, P. Yu, and H. Ji, “Unlearning bias in language models by partitioning gradients,” inFindings of the Association for Computational Linguistics: ACL 2023, pp. 6032–6048, 2023.[25]B. Wang, W. Ping, C. Xiao, P. Xu, M. Patwary, M. Shoeybi, B. Li, A. Anandkumar, and B. Catanzaro, “Exploring the limits of domain-adaptive training for detoxifying large-scale language models,”Advances in Neural Information Processing Systems, vol. 35, pp. 35811–35824, 2022.[26]X. Suau, P. Delobelle, K. Metcalf, A. Joulin, N. Apostoloff, L. Zappella, and P. Rodriguez, “Whispering experts: Neural interventions for toxicity mitigation in language models,” inForty-first International Conference on Machine Learning.[27]S. Liu, Y. Yao, J. Jia, S. Casper, N. Baracaldo, P. Hase, X. Xu, Y. Yao, H. Li, K. R. Varshney, M. Bansal, S. Koyejo, and Y. Liu, “Rethinking machine unlearning for large language models,”ArXiv, vol. abs/2402.08787, 2024.[28]A. Lynch, P. Guo, A. Ewart, S. Casper, and D. Hadfield-Menell, “Eight methods to evaluate robust unlearning in llms,”arXiv preprint arXiv:2402.16835, 2024.[29]R. Eldan and M. Russinovich, “Who’s harry potter? approximate unlearning in llms,”ArXiv, vol. abs/2310.02238, 2023.[30]C. Matzken, S. Eger, and I. Habernal, “Trade-offs between fairness and privacy in language modeling,”arXiv preprint arXiv:2305.14936, 2023.[31]Z. Xu, Z. Liu, B. Chen, Y. Tang, J. Wang, K. Zhou, X. Hu, and A. Shrivastava, “Compress, then prompt: Improving accuracy-efficiency trade-off of llm inference with transferable prompt,”arXiv preprint arXiv:2305.11186, 2023.[32]Y. Li, Y. Yu, C. Liang, P. He, N. Karampatziakis, W. Chen, and T. Zhao, “LoftQ: Lora-fine-tuning-aware quantization for large language models,” inInternational Conference on Learning Representations, 2024.[33]H. Touvron, L. Martin, K. Stone, P. Albert, A. Almahairi, Y. Babaei, N. Bashlykov, S. Batra, P. Bhargava, S. Bhosale,et al., “Llama 2: Open foundation and fine-tuned chat models,”arXiv preprint arXiv:2307.09288, 2023.[34]E. J. Hu, Y. Shen, P. Wallis, Z. Allen-Zhu, Y. Li, S. Wang, L. Wang, and W. Chen, “Lora: Low-rank adaptation of large language models,”arXiv preprint arXiv:2106.09685, 2021.[35]O. Levy, M. Seo, E. Choi, and L. Zettlemoyer, “Zero-shot relation extraction via reading comprehension,” inProceedings of the 21st Conference on Computational Natural Language Learning (CoNLL 2017) , pp. 333–342, 2017.[36]M. Sun, Z. Liu, A. Bair, and J. Z. Kolter, “A simple and effective pruning approach for large language models,”arXiv preprint arXiv:2306.11695, 2023.[37]J. Lin, J. Tang, H. Tang, S. Yang, X. Dang, and S. Han, “AWQ: Activation-aware weight quantization for llm compression and acceleration,”arXiv preprint arXiv:2306.00978, 2023.[38]J. Jang, D. Yoon, S. Yang, S. Cha, M. Lee, L. Logeswaran, and M. Seo, “Knowledge unlearning for mitigating privacy risks in language models,” inAnnual Meeting of the Association for Computational Linguistics, 2022.[39]P. Maini, Z. Feng, A. Schwarzschild, Z. C. Lipton, and J. Z. Kolter, “Tofu: A task of fictitious unlearning for llms,”ArXiv, vol. abs/2401.06121, 2024.[40]N. Li, A. Pan, A. Gopal, S. Yue, D. Berrios, A. Gatti, J. D. Li, A.-K. Dombrowski, S. Goel, L. Phan, G. Mukobi, N. Helm-Burger, R. Lababidi, L. Justen, A. B. Liu, M. Chen, I. Barrass, O. Zhang, X. Zhu, R. Tamirisa, B. Bharathi, A. Khoja, Z. Zhao, A. Herbert-Voss, C. B. Breuer, S. Marks, O. Patel, A. Zou, M. Mazeika, Z. Wang, P. Oswal, W. Liu, A. A. Hunt, J. Tienken-Harder, K. Y. Shih, K. Talley, J. Guan, R. Kaplan, I. Steneker, D. Campbell, B. Jokubaitis, A. Levinson, J. Wang, W. Qian, K. K. Karmakar, S. Basart, S. Fitz, M. Levine, P. Kumaraguru, U. Tupakula, V. Varadharajan, Y. Shoshitaishvili, J. Ba, K. M. Esvelt, A. Wang, and D. Hendrycks, “The wmdp benchmark: Measuring and reducing malicious use with unlearning,” 2024.[41]G. Gangadhar and K. Stratos, “Model editing by pure fine-tuning,” 2024.[42]C.-H. Hsueh, P. K.-M. Huang, T.-H. Lin, C.-W. Liao, H.-C. Fang, C.-W. Huang, and Y.-N. Chen, “Editing the mind of giants: An in-depth exploration of pitfalls of knowledge editing in large language models,”arXiv preprint arXiv:2406.01436, 2024.[43]W. Hua, J. Guo, M. Dong, H. Zhu, P. Ng, and Z. Wang, “Propagation and pitfalls: Reasoning-based assessment of knowledge editing through counterfactual tasks,”arXiv preprint arXiv:2401.17585, 2024.[44]Y. Yao, P. Wang, B. Tian, S. Cheng, Z. Li, S. Deng, H. Chen, and N. Zhang, “Editing large language models: Problems, methods, and opportunities,” 2023.[45]A. Karamolegkou, J. Li, L. Zhou, and A. Sogaard, “Copyright violations and large language models,”ArXiv, vol. abs/2310.13771, 2023.[46]M. Meeus, I. Shilov, M. Faysse, and Y.-A. de Montjoye, “Copyright traps for large language models,”ArXiv, vol. abs/2402.09363, 2024.[47]E. Sheng, K.-W. Chang, P. Natarajan, and N. Peng, “The woman worked as a babysitter: On biases in language generation,”ArXiv, vol. abs/1909.01326, 2019.[48]S. Gehman, S. Gururangan, M. Sap, Y. Choi, and N. A. Smith, “Realtoxicityprompts: Evaluating neural toxic degeneration in language models,” inFindings, 2020.[49]N. Carlini, C. Liu, Ú. Erlingsson, J. Kos, and D. X. Song, “The secret sharer: Evaluating and testing unintended memorization in neural networks,” inUSENIX Security Symposium, 2018.[50]D. Ippolito, F. Tramèr, M. Nasr, C. Zhang, M. Jagielski, K. Lee, C. A. Choquette-Choo, and N. Carlini, “Preventing verbatim memorization in language models gives a false sense of privacy,”ArXiv, vol. abs/2210.17546, 2022.[51]S. Biderman, U. S. Prashanth, L. Sutawika, H. Schoelkopf, Q. G. Anthony, S. Purohit, and E. Raf, “Emergent and predictable memorization in large language models,”ArXiv, vol. abs/2304.11158, 2023.[52]U. S. Prashanth, A. Deng, K. O’Brien, V. JyothirS, M. A. Khan, J. Borkar, C. A. Choquette-Choo, J. R. Fuehne, S. Biderman, T. Ke, K. Lee, and N. Saphra, “Recite, reconstruct, recollect: Memorization in lms as a multifaceted phenomenon,” 2024.[53]Y. Cao and J. Yang, “Towards making systems forget with machine unlearning,”2015 IEEE Symposium on Security and Privacy, pp. 463–480, 2015.[54]M. Dehghani, Y. Tay, A. A. Gritsenko, Z. Zhao, N. Houlsby, F. Diaz, D. Metzler, and O. Vinyals, “The benchmark lottery,”ArXiv, vol. abs/2107.07002, 2021.[55]S. Biderman, H. Schoelkopf, L. Sutawika, L. Gao, J. Tow, B. Abbasi, A. F. Aji, P. S. Ammanamanchi, S. Black, J. Clive, A. DiPofi, J. Etxaniz, B. Fattori, J. Z. Forde, C. Foster, M. Jaiswal, W. Y. Lee, H. Li, C. Lovering, N. Muennighoff, E. Pavlick, J. Phang, A. Skowron, S. Tan, X. Tang, K. A. Wang, G. I. Winata, F. Yvon, and A. Zou, “Lessons from the trenches on reproducible evaluation of language models,” 2024.[56]V. Gupta, D. Pantoja, C. Ross, A. Williams, and M. Ung, “Changing answer order can decrease mmlu accuracy,” 2024.[57]D. Hendrycks, C. Burns, S. Basart, A. Zou, M. Mazeika, D. X. Song, and J. Steinhardt, “Measuring massive multitask language understanding,”ArXiv, vol. abs/2009.03300, 2020.[58]L. Gao, J. Tow, B. Abbasi, S. Biderman, S. Black, A. DiPofi, C. Foster, L. Golding, J. Hsu, A. Le Noac’h, H. Li, K. McDonell, N. Muennighoff, C. Ociepa, J. Phang, L. Reynolds, H. Schoelkopf, A. Skowron, L. Sutawika, E. Tang, A. Thite, B. Wang, K. Wang, and A. Zou, “A framework for few-shot language model evaluation,” 12 2023.[59]AI@Meta, “Llama 3 model card,” 2024.[60]R. Zhang, L. Lin, Y. Bai, and S. Mei, “Negative preference optimization: From catastrophic collapse to effective unlearning,”ArXiv, vol. abs/2404.05868, 2024.[61]Q. Wang, B. Han, P. Yang, J. Zhu, T. Liu, and M. Sugiyama, “Unlearning with control: Assessing real-world utility for large language model unlearning,” 2024.[62]S. Hooker, A. C. Courville, G. Clark, Y. Dauphin, and A. Frome, “What do compressed deep neural networks forget,”arXiv: Learning, 2019.[63]M. Du, S. Mukherjee, Y. Cheng, M. Shokouhi, X. Hu, and A. H. Awadallah, “What do compressed large language models forget? robustness challenges in model compression,”ArXiv, vol. abs/2110.08419, 2021.[64]A. H. Azeemi, I. A. Qazi, and A. A. Raza, “Data pruning for efficient model pruning in neural machine translation,” inConference on Empirical Methods in Natural Language Processing, 2023.[65]D. N. M. Hoang, M. Cho, T. Merth, M. Rastegari, and Z. Wang, “Do compressed llms forget knowledge? an experimental study with practical implications,” 2023.[66]S. Sai, S. Namburi, M. N. Sreedhar, S. Srinivasan, and F. Sala, “The cost of compression: Investigating the impact of compression on parametric knowledge in language models,”ArXiv, vol. abs/2312.00960, 2023.[67]D. Brown, C. Godfrey, C. Nizinski, J. Tu, and H. Kvinge, “Edit at your own risk: evaluating the robustness of edited models to distribution shifts,” 2023.[68]J.-C. Gu, H. Xu, J.-Y. Ma, P. Lu, Z.-H. Ling, K.-W. Chang, and N. Peng, “Model editing harms general abilities of large language models: Regularization to the rescue,” 2024.[69]A. Zou, L. Phan, S. Chen, J. Campbell, P. Guo, R. Ren, A. Pan, X. Yin, M. Mazeika, A.-K. Dombrowski, S. Goel, N. Li, M. J. Byun, Z. Wang, A. Mallen, S. Basart, S. Koyejo, D. Song, M. Fredrikson, Z. Kolter, and D. Hendrycks, “Representation engineering: A top-down approach to ai transparency,”ArXiv, vol. abs/2310.01405, 2023.[70]S. Wang, Y. Zhu, H. Liu, Z. Zheng, C. Chen, and J. Li, “Knowledge editing for large language models: A survey,”arXiv preprint arXiv:2310.16218, 2023.[71]E. Mitchell, C. Lin, A. Bosselut, C. D. Manning, and C. Finn, “Memory-based model editing at scale,” inInternational Conference on Machine Learning, PMLR, 2022.[72]A. Sinitsin, V. Plokhotnyuk, D. Pyrkin, S. Popov, and A. Babenko, “Editable neural networks,” inInternational Conference on Learning Representations, 2020.[73]E. Mitchell, C. Lin, A. Bosselut, C. Finn, and C. D. Manning, “Fast model editing at scale,” inInternational Conference on Learning Representations, 2022.[74]Z. Zhong, Z. Wu, C. D. Manning, C. Potts, and D. Chen, “Mquake: Assessing knowledge editing in language models via multi-hop questions,” inProceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, 2023.[75]R. Cohen, E. Biran, O. Yoran, A. Globerson, and M. Geva, “Evaluating the ripple effects of knowledge editing in language models,”Transactions of the Association for Computational Linguistics, vol. 12, pp. 283–298, 2024.[76]N. De Cao, W. Aziz, and I. Titov, “Editing factual knowledge in language models,” inProceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pp. 6491–6506, 2021.[77]Z. Huang, Y. Shen, X. Zhang, J. Zhou, W. Rong, and Z. Xiong, “Transformer-patcher: One mistake worth one neuron,” inInternational Conference on Learning Representations, 2023.[78]J.-C. Gu, H.-X. Xu, J.-Y. Ma, P. Lu, Z.-H. Ling, K.-W. Chang, and N. Peng, “Model editing can hurt general abilities of large language models,”arXiv preprint arXiv:2401.04700, 2024.[79]J. Hoelscher-Obermaier, J. Persson, E. Kran, I. Konstas, and F. Barez, “Detecting edit failures in large language models: An improved specificity benchmark,”arXiv preprint arXiv:2305.17553, 2023.[80]Z. Li, N. Zhang, Y. Yao, M. Wang, X. Chen, and H. Chen, “Unveiling the pitfalls of knowledge editing for large language models,”arXiv preprint arXiv:2310.02129, 2023.[81]R. Hazra, S. Layek, S. Banerjee, and S. Poria, “Sowing the wind, reaping the whirlwind: The impact of editing language models,”arXiv preprint arXiv:2401.10647, 2024.[82]D. Brown, C. Godfrey, C. Nizinski, J. Tu, and H. Kvinge, “Robustness of edited neural networks,”arXiv preprint arXiv:2303.00046, 2023.[83]P. Hase, M. Bansal, B. Kim, and A. Ghandeharioun, “Does localization inform editing? surprising differences in causality-based localization vs. knowledge editing in language models,” inAdvances in Neural Information Processing Systems, 2023.[84]D. Powell, W. Gerych, and T. Hartvigsen, “TAXI: Evaluating categorical knowledge editing for language models,” inFindings of the Association for Computational Linguistics: ACL 2024, 2024.[85]T. B. Shaik, X. Tao, H. Xie, L. Li, X. Zhu, and Q. Li, “Exploring the landscape of machine unlearning: A comprehensive survey and taxonomy,”ArXiv, vol. abs/2305.06360, 2023.[86]J. Zhao, Z. Deng, D. Madras, J. Zou, and M. Ren, “Learning and forgetting unsafe examples in large language models,”ArXiv, vol. abs/2312.12736, 2023.[87]Z. Liu, G. Dou, Z. Tan, Y. Tian, and M. Jiang, “Towards safer large language models through machine unlearning,”ArXiv, vol. abs/2402.10058, 2024.[88]J. Jia, J. Liu, P. Ram, Y. Yao, G. Liu, Y. Liu, P. Sharma, and S. Liu, “Model sparsity can simplify machine unlearning,” inNeural Information Processing Systems, 2023.[89]G. Xiao, J. Lin, M. Seznec, H. Wu, J. Demouth, and S. Han, “Smoothquant: Accurate and efficient post-training quantization for large language models,” inInternational Conference on Machine Learning, pp. 38087–38099, PMLR, 2023.[90]L. Yin, Y. Wu, Z. Zhang, C.-Y. Hsieh, Y. Wang, Y. Jia, M. Pechenizkiy, Y. Liang, Z. Wang, and S. Liu, “Outlier weighed layerwise sparsity (owl) : A missing secret sauce for pruning llms to high sparsity,”arXiv preprint arXiv:2310.05175, 2023.[91]X. Ma, G. Fang, and X. Wang, “Llm-pruner: On the structural pruning of large language models,”arXiv preprint arXiv:2305.11627, 2023.[92]P. Wang, Z. Wang, Z. Li, Y. Gao, B. Yin, and X. Ren, “Scott: Self-consistent chain-of-thought distillation,”arXiv preprint arXiv:2305.01879, 2023.[93]S. Saha, P. Hase, and M. Bansal, “Can language models teach weaker agents? teacher explanations improve students via theory of mind,”arXiv preprint arXiv:2306.09299, 2023.[94]C.-Y. Hsieh, C.-L. Li, C.-K. Yeh, H. Nakhost, Y. Fujii, A. Ratner, R. Krishna, C.-Y. Lee, and T. Pfister, “Distilling step-by-step! outperforming larger language models with less training data and smaller model sizes,”arXiv preprint arXiv:2305.02301, 2023.[95]M. Xu, Y. L. Xu, and D. P. Mandic, “Tensorgpt: Efficient compression of the embedding layer in llms based on the tensor-train decomposition,”arXiv preprint arXiv:2307.00526, 2023.[96]P. Sharma, J. T. Ash, and D. Misra, “The truth is in there: Improving reasoning in language models with layer-selective rank reduction,”arXiv preprint arXiv:2312.13558, 2023.[97]Z. Yao, R. Yazdani Aminabadi, M. Zhang, X. Wu, C. Li, and Y. He, “Zeroquant: Efficient and affordable post-training quantization for large-scale transformers,”Advances in Neural Information Processing Systems, vol. 35, pp. 27168–27183, 2022.[98]T. Dettmers, M. Lewis, Y. Belkada, and L. Zettlemoyer, “Llm. int8 () : 8-bit matrix multiplication for transformers at scale,”arXiv preprint arXiv:2208.07339, 2022.[99]P. Wang, N. Zhang, X. Xie, Y. Yao, B. Tian, M. Wang, Z. Xi, S. Cheng, K. Liu, G. Zheng,et al., “Easyedit: An easy-to-use knowledge editing framework for large language models,”arXiv preprint arXiv:2308.07269, 2023.[100]S. Merity, C. Xiong, J. Bradbury, and R. Socher, “Pointer sentinel mixture models,” inInternational Conference on Learning Representations, 2016.[101]A. Golatkar, A. Achille, and S. Soatto, “Eternal sunshine of the spotless net: Selective forgetting in deep networks,”2020 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) , pp. 9301–9309, 2019.[102]J. Yao, E. Chien, M. Du, X. Niu, T. Wang, Z. Cheng, and X. Yue, “Machine unlearning of pre-trained large language models,”ArXiv, vol. abs/2402.15159, 2024.AppendixAppendix ALimitationsWhile our work provides the first look at how state-of-the-art methods compose, there are far more possible combinations of interventions, methods, models, and datasets than we can consider in one work.
So some popular methods are omitted.
We also prioritize broad experiments with many different intervention methods.
We hope future works will closely examine individual methods to study, improve, and extend beyond the mechanisms that drive their composability.
Lastly, we only experiment with Llama3-8B.
While our code is extendable and Llama3-8B is popular, it is unknown whether our findings generalize between language models.
We hope our codebase empowers others to explore more model sizes and architectures in future research.Appendix BBackground on InterventionsDesigning interventions for language models is a highly-active research area.
We focus on three key types of intervention.Knowledge Editing.
Knowledge editors aim to address factuality decay in LLMs: As the world changes, some facts an LLM learned during training become inaccurate.
For example, a model trained before 2020 would still predictThe latest pandemic in the US is“Swine Flu” until updated to predict “COVID”.
There are four general approaches to model editing: 1)  Memory-based methods that cache knowledge, 2)  Locate-then-Edit methods that selectively finetune parameter subsets, 3)  Hypernetwork methods that predict new model weights, and 4)  Prompt editors that add facts to prompts.
Most editors update singular facts, though recent works apply many simultaneously.
Others works embrace the practical need forsequentialedits.
Recent works have also begun investigating unintended impacts of editing on pre-trained models, while others are exploring multi-hop editing.
However, these works study impacts on model performance, not interactions with other types of interventions.Machine Unlearning.Machine Unlearning (MU)  refers to a broad set of techniques for modifying a model post-training to remove the influence of specific training examples or other undesirable properties444Most MU techniques for LMs are considered approximate unlearning in that they aim to modify the model to approximate the behavior of it not being trained on the unlearning target without retraining from scratch.. These undesirable properties are commonly referred to as the unlearning target. Crucially, MU must excel at causing the model to forget the unlearning target without adversely affecting overall utility by reducing performance on other domains. Another requirement is that MU must cause models to deeply forget knowledge such that the unlearning target can not be elicited by adversaries or probing. With the rise of generative models, recent work has explored MU for forgetting potentially copyrighted work, information about individuals, and potentially dangerous knowledge. Existing works have studied which types of examples models forget when compressed, whether compression hinders unlearning, and whether compression in itself can be used as an unlearning technique. While related to knowledge editing, the two areas have largely developed independently.Model Compression.
To mitigate the huge computational and memory demands of LLMs, plenty of classical techniques in model compression have been explored, such as quantization, network pruning, knowledge distillation, and low-rank factorization. We focus on quantization and network pruning,
which both make in-place updates to pre-trained model weights.
Quantization involves lowering the bit-precision of model parameters, effectively shrinking model size and expediting inference. Early works, such as ZeroQuantand LLM.int8, focused on fine-tuning quantization granularity, showing promising results predominantly at higher bit-widths, like 8-bit. More recent innovations like GPTQand AWQhave successfully pushed the boundaries by reducing bit-width to as low as 3 or 4 bits per weight, with negligible performance degradation.
Pruning is another compression methods, which aims to eliminate redundant components like weights and channels.
Techniques like SparseGPTand Wandasuccessfully maintain much of an LLM’s performance, even when discarding approximately 50% of its weights. OWLachieves higher levels of sparsity by reallocating sparsities across layers.Appendix CImplementation DetailsIn this section we outline the set up and hyperparameters used for knowledge editing, unlearning, and compression.C.1Knowledge Editing DetailsC.1.1Knowledge Editing MetricsWe adopt three widely-used editing metrics and three compression metrics to measure the effectiveness of these interventions. The editing metrics are:Edit Success:Assesses the post-edit model’s ability to provide the correct, expected answer. An issue arises here with the different levels of verbosity for the models. An unedited model might return the correct token, but use a differently phrased answer or arrange the correct tokens differently. We therefore adopt and F1-style measure used in recent workfor this.First, we determine the common tokens between the generated response and the ground truth, which are represented by the sets(generated response tokens without special tokens)  and(target tokens without padding) , respectively. The common tokens are given by.Ifis empty is empty, the F1 score is set to 0, as there are no common elements to compare.Otherwise, we calculate precision and recall. Precision is the ratio of the number of common tokens to the number of tokens in the generated response, and recall is the ratio of the number of common tokens to the number of target tokens. They are defined as:PrecisionRecallThe F1 score, which is the harmonic mean of precision and recall, is then calculated as:with the condition that if the denominatoris zero, the F1 score is set to 0.The final F1 score is the average of the F1 scores for all items. If there are no F1 scores (i.e., the list of scores is empty) , the average F1 score is set to 0.Edit Generalization:Measures the model’s capability to genuinely update knowledge, rather than just repeating keywords. This is tested by evaluating edit success on paraphrased prompts given by the respective dataset.Edit Locality:Examines the precision of edits, ensuring that only relevant, in-distribution knowledge is altered while out-of-distribution facts remain unaffected. In this setting, the F1 score is critical as testing on the pre-trained knowledge can result in varied answer phrasings.C.1.2Knowledge Editing DatasetZsRE: This dataset focuses on context-free question-answering. Given a question related to a subject and a relation, the model’s task is to provide the correct object as the answer.follow the procedure outlined in, to improve the locality sets used. As the initial locality sets were solely based on Natural Question annotations, we use the locality sets provided by.In all three datasets, original prompt rephrasings and locality sets are used to derive the respective metrics.C.2Model Editing DetailsC.2.1MEMIT DetailsWe use the state-of-the-art MEMITmodel editor, which applies batches of edits simultaneously.
The editing process was applied to layers 4 through 8 of the model, with a clamp normalization factor set at 4. The learning parameters adhered closely to the original implementation:v_num_grad_stepswas set to 25, accompanied by a learning rate (lr)  of 0.5, and using the last layer for loss calculation. Additionally, a weight decay (weight_decay)  of 0.001 was employed. The KL divergence contribution to the overall loss was controlled by aof 0.0625. Moreover, a second momentum adjustment was enabled, with an update weight of, to fine-tune the optimization process. The model generated a maximum length of 40 tokens and a batch size of 50, matching the number of edits being made. 10 repeats were made for each edit and the results averaged.C.2.2LoRA DetailsWe also employed LoRA as a model editor, specifically using the adalora variant for our experiments. The editing process utilized 70 gradient steps, with a learning rate of 0.005. The rank for the low-rank adaptation was set at 8, with a scaling factor of 32. There was no dropout applied to the adaptation matrices, and the normalization constraint was disabled. The target modules for the edits consisted of query and value projection layers. Consistent with the MEMIT method, the model generated sequences with a maximum length of 40 tokens and utilized a batch size of 50, matching the number of edits per batch. Each edit was repeated 10 times, and the results were averaged to ensure reliability. The data type for the computations was set totorch.float.C.2.3Fine-Tuning (FT)  DetailsFor fine-tuning (FT) , we focused on layers 4 through 8 of the model. The specific modules targeted for editing included the MLP down-projection (model.layers.{}.mlp.down_proj) , the general layer module (model.layers.{}) , the MLP module (model.layers.{}.mlp) , the self-attention module (model.layers.{}.self_attn) , the final layer normalization (model.norm) , and the language model head (lm_head) .The fine-tuning process was conducted with 25 optimization steps, using a learning rate of. No weight decay was applied. The normalization constraint was disabled. The optimization objective was set to target the new data and the data type for the computations was set to bfloat16.As with the MEMIT and LoRA methods, the model generated sequences with a maximum length of 40 tokens and used a batch size of 50, matching the number of edits per batch. Each edit was repeated 10 times, and the results were averaged for consistency and reliability.C.3Model Compression DetailsC.3.1Compression MetricsThe compression metrics include:Sparsity Ratio.The fraction of parameters in the model that are zero, indicating the extent to which the model is sparse.Average Bits.The mean number of bits utilized to represent each parameter in a quantized or sparsified neural network, reflecting the level of compression.C.3.2Compressor detailsWe use four state-of-the-art compression methods including two pruning methods:SparseGPT: an efficient one-shot pruning method tailored for large models. It converts the pruning process into solving large-scale sparse regression problems using an approximate solver. This approach enables rapid pruning on a single GPU with minimal accuracy loss, achieving 50-60% on large models.Wanda: is another popular method for pruning large language models that relies on a pruning metric that combines a weight’s magnitude with the norm of its corresponding input activations, determined from a small calibration dataset. The method focuses on selectively pruning weights within individual outputs of linear layers, aiming for high sparsity levels without modifying unpruned weights. Wanda is computationally efficient, executable in a single forward pass.and two quantization methods:GPTQ: an algorithm designed for efficient weight quantization in large-scale models. It revises the weight quantization approach by quantizing weights in a fixed order rather than a greedy order, which shows minimal performance difference, especially in larger models. GPTQ introduced a novel method where each weight is quantized column-by-column, reducing computational complexity.AWQ: is based on the premise that not all weights are equally critical for model performance, and it identifies a small fraction of salient weights whose quantization significantly impacts model accuracy. This identification is done by analyzing activation distributions rather than weight distributions, under the rationale that weights linked to larger activation magnitudes are more crucial.C.4Machine Unlearning DetailsWe implement three unlearning techniques: RMU, GA, and GD. The success of each technique hinges on the careful selection of hyperparameters. We describe our parameter searches and selected values in the following sections.C.4.1Representation Misdirection UnlearningIntroduced in, RMU intervenes on a specific layer of the LM to perturb activations for inputs related to the unlearning target. This technique has achieved impressive performance on the WMDP dataset, with performance on the unlearning targets dropping to near-random while overall model utility (MMLU)  minimally regresses. Wikitextacts as the retain set.RMU relies on multiple hyperparameters for selecting which layer to modify and how the loss should be calculated. We found that RMU is quite sensitive to hyperparameter choice, with most combinations either leaving the model unaffected or significantly harming model utility. Table5details the results from our hyperparameters search. We otherwise use the RMU repo’s555https://github.com/centerforaisafety/wmdpdefault settings.ParameterDescriptionSearchSelectedAlphaThe weight for the retain loss[1, 10, 100, 1000, 10000]1000LayerThe layer to modify[3, 17]3Maximum BatchesTraining duration RMU in batches[100, 150, 200, 250, 300]250Table 5:RMU Hyperparameter Search.Details for the grid search over possible RMU settings. Hyperparameters were selected using grid search.C.4.2Gradient AscentGA, a simple and widely studied unlearning method, compels the model to minimize the likelihood of the correct token. The underlying intuition is that models will learn to avoid generating tokens associated with the unlearning target. We only train the final 16 layers of the model due to memory constraints.ParameterDescriptionSearchSelectedLRThe learning rate[5e-6, 1e-6, … 5e-3, 1e-3]5e-5Training SamplesTrain size balanced across Cyber/Bio[10, 25, 50, 100, … 500, 1000, 2000]50Table 6:GA Hyperparameter Search.Details for the grid search over possible GA settings. Hyperparameters were selected using grid search.C.4.3Gradient DifferenceA significant challenge when leveraging GA is balancing reducing performance on the unlearning target while retaining overall model utility. Since GA reduces the likelihood of the correct token, it is fragile to catastrophic forgetting, where the model incorrectly generalizes to forgetting how to model natural language.GDaims to address this shortcoming by adding a retain loss term to the loss function. This term aims to maximize the likelihood of the correct token on an unrelated retain dataset. We use Wikitetxas the retain set. As with GA, we only train the final 16 layers of the model.Table7reports our hyperparameter search. Unlike, which found that GD provided only a marginal improvement over GA, we found that GD can perform significantly better if we upweight the retain loss term. Upweighting is necessary since it is easy for the model to significantly minimize the correct token likelihood, which causes it to drown out the retain loss term. Assigning a weight of 400 to the retain term solved this issue. This hyperparameter combination can make GD competitive with RMU, though still less composable (Section4) .ParameterDescriptionSearchSelectedAlphaThe weight for the retain loss[1, 2, 4, 6, 8, 10, 20, 40, 60, 80, 100]40LRThe learning rate[5e-6, 1e-6, … 5e-3, 1e-3]5e-5Training SamplesTrain size balanced across Cyber/Bio[10, 25, 50, 100, … 500, 1000, 2000]50Table 7:GD Hyperparameter Search.Details for the grid search over possible GD settings. Hyperparameters were selected using grid search.Appendix DExtended ResultsWe report the task performance for all compositions below and the baseline results, where only a single intervention is applied (TableD) . Perplexity on Wikitextis also included as another overall utility metric. Average Bits (AppendixC.3.1)  measures the degree to which the composition compresses the model.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLNone0.020.020.0416.000.580.625.54\cdashline1-9

FTNone0.990.820.1116.00.570.615.57MEMITNone0.890.850.0416.00.570.615.57LoRANone1.00.710.0616.00.560.6119.25\cdashline1-9

SparseGPT (0.25) None0.020.020.0312.250.560.615.87SparseGPT (0.35) None0.020.020.0310.750.540.586.34SparseGPT (0.45) None0.020.010.039.250.510.547.43SparseGPT (0.55) None0.010.010.037.750.450.4410.43SparseGPT (0.65) None0.00.010.036.250.30.2720.83SparseGPT (0.75) None0.010.010.034.750.260.2381.84Wanda (0.25) None0.020.020.0312.250.560.615.84Wanda (0.35) None0.020.020.0410.750.540.586.31Wanda (0.45) None0.020.020.039.250.50.527.52Wanda (0.55) None0.020.010.027.750.360.3712.5Wanda (0.65) None0.020.010.026.250.260.2347.18Wanda (0.75) None0.020.030.034.750.260.23257.46GPTQ (2-Bit) None0.00.00.032.250.250.243681.23GPTQ (3-Bit) None0.010.010.033.250.450.468.6GPTQ (4-Bit) None0.020.020.034.250.560.69.97GPTQ (8-Bit) None0.020.020.048.250.580.625.54AWQ (2-Bit) None0.00.00.02.250.240.271748954.75AWQ (3-Bit) None0.010.010.033.250.520.537.47AWQ (4-Bit) None0.020.020.034.250.570.65.91AWQ (5-Bit) None0.020.030.035.250.570.625.62AWQ (6-Bit) None0.020.030.046.250.580.625.56AWQ (8-Bit) None0.020.020.048.250.580.625.54\cdashline1-9

GANone0.00.00.016.00.460.5infGDNone0.020.00.0916.00.260.524.48RMUNone0.020.020.0316.00.290.575.56Table 8:Baseline intervention Results without composition.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLFTSparseGPT (0.25) 0.960.760.1112.250.550.65.9FTSparseGPT (0.35) 0.840.660.0910.750.530.576.38FTSparseGPT (0.45) 0.640.50.079.250.50.537.48FTSparseGPT (0.55) 0.330.260.047.750.440.4410.35FTSparseGPT (0.65) 0.120.10.036.250.330.321.54FTSparseGPT (0.75) 0.030.020.034.750.260.2388.73FTWanda (0.25) 0.960.760.1112.250.560.65.88FTWanda (0.35) 0.880.680.0910.750.530.576.35FTWanda (0.45) 0.650.50.069.250.490.527.57FTWanda (0.55) 0.310.240.047.750.350.3712.48FTWanda (0.65) 0.080.070.036.250.260.2346.37FTWanda (0.75) 0.020.020.034.750.260.23344.62FTGPTQ (2-Bit) 0.00.00.022.250.250.241995.71FTGPTQ (3-Bit) 0.270.20.043.250.430.469.03FTGPTQ (4-Bit) 0.790.60.114.250.540.5912.93FTGPTQ (8-Bit) 0.990.830.118.250.570.615.57FTAWQ (2-Bit) 0.00.00.02.250.240.271739216.25FTAWQ (3-Bit) 0.70.550.073.250.50.517.56FTAWQ (4-Bit) 0.950.790.14.250.550.595.95FTAWQ (5-Bit) 0.990.840.115.250.560.615.66FTAWQ (6-Bit) 0.990.840.16.250.570.615.59FTAWQ (8-Bit) 0.990.830.118.250.570.615.57\cdashline1-8

MEMITSparseGPT (0.25) 0.870.830.0412.250.560.65.89MEMITSparseGPT (0.35) 0.810.740.0310.750.540.576.4MEMITSparseGPT (0.45) 0.620.560.049.250.510.537.43MEMITSparseGPT (0.55) 0.280.20.037.750.430.4310.37MEMITSparseGPT (0.65) 0.030.030.036.250.30.2921.02MEMITSparseGPT (0.75) 0.010.010.024.750.260.2371.62MEMITWanda (0.25) 0.70.690.0312.250.540.595.87MEMITWanda (0.35) 0.850.780.0310.750.530.576.34MEMITWanda (0.45) 0.750.670.039.250.480.517.56MEMITWanda (0.55) 0.320.260.037.750.350.3612.33MEMITWanda (0.65) 0.010.010.026.250.260.2346.98MEMITWanda (0.75) 0.020.020.034.750.260.23290.27MEMITGPTQ (2-Bit) 0.00.00.022.250.250.241663.86MEMITGPTQ (3-Bit) 0.350.290.043.250.450.468.89MEMITGPTQ (4-Bit) 0.840.80.034.250.550.5913.17MEMITGPTQ (8-Bit) 0.940.90.048.250.570.615.56MEMITAWQ (2-Bit) 0.00.00.02.250.240.271738085.5MEMITAWQ (3-Bit) 0.830.790.043.250.50.57.54MEMITAWQ (4-Bit) 0.930.890.034.250.550.595.94MEMITAWQ (5-Bit) 0.920.890.035.250.570.615.65MEMITAWQ (6-Bit) 0.930.890.046.250.570.615.59MEMITAWQ (8-Bit) 0.940.90.048.250.570.615.56\cdashline1-8

LoRASparseGPT (0.25) 0.860.520.0412.250.550.613.81LoRASparseGPT (0.35) 0.620.370.0510.750.520.5710.6LoRASparseGPT (0.45) 0.360.220.049.250.480.5216.48LoRASparseGPT (0.55) 0.170.140.047.750.430.4323.46LoRASparseGPT (0.65) 0.070.060.036.250.320.353.89LoRASparseGPT (0.75) 0.030.030.024.750.260.25186.65LoRAWanda (0.25) 0.870.530.0412.250.550.69.23LoRAWanda (0.35) 0.640.40.0510.750.520.5711.01LoRAWanda (0.45) 0.370.250.039.250.460.4912.6LoRAWanda (0.55) 0.180.140.037.750.360.3633.59LoRAWanda (0.65) 0.050.040.026.250.260.24194.26LoRAWanda (0.75) 0.020.020.034.750.250.25914.43LoRAGPTQ (2-Bit) 0.00.00.012.250.250.2410465.63LoRAGPTQ (3-Bit) 0.080.060.023.250.40.4240.02LoRAGPTQ (4-Bit) 0.580.410.054.250.540.5863.45LoRAGPTQ (8-Bit) 1.00.710.068.250.570.6118.86LoRAAWQ (2-Bit) 0.00.00.02.250.240.251764088.0LoRAAWQ (3-Bit) 0.460.310.053.250.50.5124.29LoRAAWQ (4-Bit) 0.910.60.044.250.550.5916.37LoRAAWQ (5-Bit) 0.990.70.055.250.560.616.31LoRAAWQ (6-Bit) 1.00.710.066.250.560.6115.34LoRAAWQ (8-Bit) 1.00.720.068.250.570.6115.25Table 9:Detailed results for editingcompression.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLSparseGPT (0.25) FT0.990.770.0812.250.550.65.91SparseGPT (0.35) FT0.980.750.0610.750.540.586.37SparseGPT (0.45) FT0.960.710.069.250.510.547.49SparseGPT (0.55) FT0.950.660.057.750.440.4410.19SparseGPT (0.65) FT0.890.540.046.250.330.317.67SparseGPT (0.75) FT0.580.250.044.750.260.2498.18SparseGPT (0.25) MEMIT0.910.860.0412.250.560.595.89SparseGPT (0.35) MEMIT0.890.80.0310.750.540.576.38SparseGPT (0.45) MEMIT0.620.480.039.250.510.537.49SparseGPT (0.55) MEMIT0.370.270.037.750.440.4310.62SparseGPT (0.65) MEMIT0.180.130.036.250.320.2922.83SparseGPT (0.75) MEMIT0.110.080.044.750.250.23394.59SparseGPT (0.25) LoRA0.850.450.0412.250.540.610.69SparseGPT (0.35) LoRA0.760.420.0410.750.520.579.54SparseGPT (0.45) LoRA0.640.330.049.250.480.5210.52SparseGPT (0.55) LoRA0.550.290.037.750.430.4311.5SparseGPT (0.65) LoRA0.370.170.036.250.320.2923.13SparseGPT (0.75) LoRA0.080.060.034.750.260.2589.47\cdashline1-8

Wanda (0.25) FT0.990.80.0712.250.560.65.86Wanda (0.35) FT0.970.760.0510.750.540.586.32Wanda (0.45) FT0.940.690.049.250.490.527.52Wanda (0.55) FT0.80.50.037.750.360.3811.98Wanda (0.65) FT0.50.280.036.250.260.2341.41Wanda (0.75) FT0.050.040.034.750.260.23249.29Wanda (0.25) MEMIT0.950.90.0312.250.560.595.87Wanda (0.35) MEMIT0.860.760.0310.750.540.576.34Wanda (0.45) MEMIT0.720.590.039.250.490.517.54Wanda (0.55) MEMIT0.370.260.037.750.370.3712.07Wanda (0.65) MEMIT0.040.040.026.250.260.2340.49Wanda (0.75) MEMIT0.040.040.024.750.260.23280.04Wanda (0.25) LoRA0.950.540.0312.250.550.610.65Wanda (0.35) LoRA0.860.480.0410.750.520.577.98Wanda (0.45) LoRA0.740.430.049.250.470.59.18Wanda (0.55) LoRA0.680.370.037.750.360.3715.01Wanda (0.65) LoRA0.340.190.026.250.260.2357.63Wanda (0.75) LoRA0.060.060.034.750.260.23318.06\cdashline1-8

GPTQ (2-Bit) FT0.00.00.012.250.250.24970816.25GPTQ (3-Bit) FT0.10.080.033.250.410.429.96GPTQ (4-Bit) FT0.440.240.044.250.550.586.4GPTQ (8-Bit) FT0.990.830.118.250.570.615.57GPTQ (2-Bit) MEMIT0.00.00.012.250.250.24739248.62GPTQ (3-Bit) MEMIT0.120.10.033.250.410.429.81GPTQ (4-Bit) MEMIT0.810.690.044.250.550.586.38GPTQ (8-Bit) MEMIT0.950.910.038.250.570.615.57GPTQ (2-Bit) LoRA0.00.00.012.250.250.24695958.19GPTQ (3-Bit) LoRA0.010.020.023.250.370.3811.14GPTQ (4-Bit) LoRA0.210.140.054.250.530.579.54GPTQ (8-Bit) LoRA1.00.720.068.250.560.6119.99\cdashline1-8

AWQ (2-Bit) FT0.00.00.02.250.240.2633638.44AWQ (3-Bit) FT1.00.820.083.250.50.517.57AWQ (4-Bit) FT0.990.840.154.250.540.575.98AWQ (5-Bit) FT0.990.850.125.250.550.65.68AWQ (6-Bit) FT0.990.880.136.250.560.65.61AWQ (8-Bit) FT0.990.870.138.250.560.65.6AWQ (2-Bit) MEMIT0.00.00.02.250.240.261735678.75AWQ (3-Bit) MEMIT0.920.870.033.250.490.477.57AWQ (4-Bit) MEMIT0.920.890.044.250.550.585.97AWQ (5-Bit) MEMIT0.980.960.035.250.560.65.68AWQ (6-Bit) MEMIT0.980.920.046.250.570.65.62AWQ (8-Bit) MEMIT0.980.940.048.250.570.65.59AWQ (2-Bit) LoRA0.060.030.02.250.240.26141960.91AWQ (3-Bit) LoRA1.00.710.053.250.50.5155.44AWQ (4-Bit) LoRA1.00.740.074.250.550.5829.71AWQ (5-Bit) LoRA1.00.690.055.250.560.617.32AWQ (6-Bit) LoRA1.00.690.066.250.570.6116.6AWQ (8-Bit) LoRA1.00.730.068.250.570.6121.64Table 10:Detailed results for compressionediting.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLGASparseGPT (0.25) 0.00.00.012.250.460.51infGASparseGPT (0.35) 0.00.00.010.750.440.49infGASparseGPT (0.45) 0.00.00.09.250.420.45infGASparseGPT (0.55) 0.00.00.07.750.30.32infGASparseGPT (0.65) 0.00.00.06.250.250.25infGASparseGPT (0.75) 0.00.00.04.750.250.25infGAWanda (0.25) 0.00.00.012.250.460.52infGAWanda (0.35) 0.00.00.010.750.450.49infGAWanda (0.45) 0.00.00.09.250.40.43infGAWanda (0.55) 0.00.00.07.750.30.32infGAWanda (0.65) 0.00.00.06.250.250.25infGAWanda (0.75) 0.00.00.04.750.250.26infGAGPTQ (2-Bit) 0.00.00.02.250.250.25infGAGPTQ (3-Bit) 0.00.00.03.250.330.39infGAGPTQ (4-Bit) 0.00.00.04.250.430.48infGAGPTQ (8-Bit) 0.00.00.08.250.450.49infGAAWQ (2-Bit) 0.00.00.02.250.250.251769133.88GAAWQ (3-Bit) 0.00.00.03.250.410.44infGAAWQ (4-Bit) 0.00.00.04.250.430.47infGAAWQ (5-Bit) 0.00.00.05.250.460.5infGAAWQ (6-Bit) 0.00.00.06.250.450.49infGAAWQ (8-Bit) 0.00.00.08.250.450.49inf\cdashline1-8

GDSparseGPT (0.25) 0.00.010.0212.250.270.585.07GDSparseGPT (0.35) 0.00.00.0210.750.280.555.64GDSparseGPT (0.45) 0.00.00.029.250.280.517.02GDSparseGPT (0.55) 0.00.00.017.750.270.4210.6GDSparseGPT (0.65) 0.00.00.026.250.260.2725.97GDSparseGPT (0.75) 0.010.00.014.750.240.23119.71GDWanda (0.25) 0.00.010.0312.250.350.585.34GDWanda (0.35) 0.00.00.0210.750.380.566.61GDWanda (0.45) 0.00.00.019.250.460.519.9GDWanda (0.55) 0.00.00.07.750.370.3922.22GDWanda (0.65) 0.00.00.06.250.260.23184.75GDWanda (0.75) 0.00.00.04.750.260.233298.92GDGPTQ (2-Bit) 0.00.00.02.250.240.27infGDGPTQ (3-Bit) 0.00.00.03.250.240.3infGDGPTQ (4-Bit) 0.00.00.04.250.240.325.106230513313957e+35GDGPTQ (8-Bit) 0.00.00.08.250.240.352.445586872346403e+26GDAWQ (2-Bit) 0.00.00.02.250.240.272598854.5GDAWQ (3-Bit) 0.00.00.03.250.330.33infGDAWQ (4-Bit) 0.00.00.04.250.240.352.0452847856930923e+28GDAWQ (5-Bit) 0.00.00.05.250.250.361.0052004503012292e+28GDAWQ (6-Bit) 0.00.00.06.250.250.356.499345951176105e+25GDAWQ (8-Bit) 0.00.00.08.250.250.359.430016153317283e+25\cdashline1-8

RMUSparseGPT (0.25) 0.020.020.0312.250.280.565.93RMUSparseGPT (0.35) 0.020.020.0310.750.280.536.39RMUSparseGPT (0.45) 0.020.020.039.250.270.488.14RMUSparseGPT (0.55) 0.010.010.037.750.260.3910.58RMUSparseGPT (0.65) 0.00.010.036.250.250.2635.97RMUSparseGPT (0.75) 0.010.010.034.750.250.23101.32RMUWanda (0.25) 0.020.020.0412.250.290.565.88RMUWanda (0.35) 0.020.020.0310.750.280.536.36RMUWanda (0.45) 0.020.010.039.250.290.487.71RMUWanda (0.55) 0.010.010.037.750.270.3513.73RMUWanda (0.65) 0.010.010.036.250.250.2352.67RMUWanda (0.75) 0.030.020.034.750.250.23334.45RMUGPTQ (2-Bit) 0.00.010.032.250.250.243451.67RMUGPTQ (3-Bit) 0.010.010.033.250.260.429.09RMUGPTQ (4-Bit) 0.020.020.034.250.270.5312.88RMUGPTQ (8-Bit) 0.020.020.038.250.290.575.56RMUAWQ (2-Bit) 0.00.00.02.250.240.261726409.12RMUAWQ (3-Bit) 0.010.010.033.250.270.457.55RMUAWQ (4-Bit) 0.020.020.034.250.290.565.95RMUAWQ (5-Bit) 0.020.030.045.250.280.565.68RMUAWQ (6-Bit) 0.020.020.036.250.280.565.62RMUAWQ (8-Bit) 0.020.020.038.250.290.575.56Table 11:Detailed results for unlearningcompression.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLSparseGPT (0.25) GA0.00.00.012.250.430.45infSparseGPT (0.35) GA0.00.00.010.750.340.36infSparseGPT (0.45) GA0.00.00.09.250.310.33infSparseGPT (0.55) GA0.00.00.07.750.250.25infSparseGPT (0.65) GA0.00.00.06.250.270.28infSparseGPT (0.75) GA0.010.010.034.750.260.23infSparseGPT (0.25) GD0.010.00.0212.250.50.4813.32SparseGPT (0.35) GD0.010.010.0110.750.290.252712.26SparseGPT (0.45) GD0.00.00.09.250.250.481.246813090597988e+19SparseGPT (0.55) GD0.00.00.027.750.350.369.35SparseGPT (0.65) GD0.00.00.016.250.240.26infSparseGPT (0.75) GD0.00.010.014.750.260.231437619072.0SparseGPT (0.25) RMU0.010.020.0412.250.310.575.92SparseGPT (0.35) RMU0.00.020.0210.750.310.546.35SparseGPT (0.45) RMU0.010.010.039.250.310.517.86SparseGPT (0.55) RMU0.010.010.037.750.290.4110.39SparseGPT (0.65) RMU0.00.00.026.250.260.2430.26SparseGPT (0.75) RMU0.010.010.024.750.250.2397.03\cdashline1-8

Wanda (0.25) GA0.00.00.012.250.490.54infWanda (0.35) GA0.00.00.010.750.440.51infWanda (0.45) GA0.00.00.09.250.410.43infWanda (0.55) GA0.00.00.07.750.320.34infWanda (0.65) GA0.00.00.06.250.260.26infWanda (0.75) GA0.010.010.034.750.260.238.12597183809491e+36Wanda (0.25) GD0.00.00.012.250.520.5713.26Wanda (0.35) GD0.00.00.010.750.450.4857480503296.0Wanda (0.45) GD0.00.00.09.250.50.5349.77Wanda (0.55) GD0.00.00.07.750.390.35121.07Wanda (0.65) GD0.00.00.06.250.260.2316268.01Wanda (0.75) GD0.00.00.04.750.250.2510226426.0Wanda (0.25) RMU0.020.020.0412.250.320.575.86Wanda (0.35) RMU0.020.020.0310.750.340.566.33Wanda (0.45) RMU0.020.020.039.250.390.527.55Wanda (0.55) RMU0.010.010.037.750.360.3612.58Wanda (0.65) RMU0.010.010.026.250.260.2345.53Wanda (0.75) RMU0.020.030.024.750.260.23391.87\cdashline1-8

GPTQ (2-Bit) GA0.00.00.02.250.250.25infGPTQ (3-Bit) GA0.00.00.03.250.240.27infGPTQ (4-Bit) GA0.00.00.04.250.340.38infGPTQ (8-Bit) GA0.00.00.08.250.450.5infGPTQ (2-Bit) GD0.00.00.02.250.250.25infGPTQ (3-Bit) GD0.00.00.03.250.260.24628.24GPTQ (4-Bit) GD0.010.010.014.250.270.569.38GPTQ (8-Bit) GD0.010.020.058.250.260.47infGPTQ (2-Bit) RMU0.00.00.02.250.250.241017825.94GPTQ (3-Bit) RMU0.010.010.023.250.320.3211.1GPTQ (4-Bit) RMU0.00.020.024.250.450.586.35GPTQ (8-Bit) RMU0.010.020.038.250.280.575.58\cdashline1-8

AWQ (2-Bit) GA0.00.00.02.250.240.27infAWQ (3-Bit) GA0.00.00.03.250.240.27infAWQ (4-Bit) GA0.00.00.04.250.450.45infAWQ (5-Bit) GA0.00.00.05.250.350.39infAWQ (6-Bit) GA0.00.00.06.250.250.3infAWQ (8-Bit) GA0.00.00.08.250.280.37infAWQ (2-Bit) GD0.00.00.022.250.240.274.496028624899119e+33AWQ (3-Bit) GD0.010.00.023.250.290.34166895.91AWQ (4-Bit) GD0.010.00.044.250.240.427.05AWQ (5-Bit) GD0.00.00.035.250.250.382685384.0AWQ (6-Bit) GD0.00.00.056.250.250.3127025.07AWQ (8-Bit) GD0.00.00.018.250.250.2549224794112.0AWQ (2-Bit) RMU0.00.00.02.250.240.271749321.75AWQ (3-Bit) RMU0.010.010.033.250.270.467.51AWQ (4-Bit) RMU0.020.020.034.250.270.556.06AWQ (5-Bit) RMU0.020.030.045.250.270.565.68AWQ (6-Bit) RMU0.020.020.046.250.280.565.6AWQ (8-Bit) RMU0.020.030.038.250.280.575.71Table 12:Detailed results for compressionunlearning.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLFTGA0.00.00.016.00.470.53infFTGD0.320.250.0716.00.290.412.8236466916108585e+31FTRMU0.990.820.116.00.280.565.61\cdashline1-8

MEMITGA0.00.00.016.00.40.45infMEMITGD0.530.490.0316.00.260.3627645056122880.0MEMITRMU0.960.890.0316.00.290.565.58\cdashline1-8

LoRAGA0.00.00.016.00.280.29infLoRAGD0.440.230.0516.00.30.4529.46LoRARMU1.00.680.0516.00.30.5234.56Table 13:Detailed results for editingunlearning.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLGAFT0.070.040.016.00.470.52infGAMEMIT0.480.410.016.00.450.49infGALoRA1.00.780.0316.00.340.3656.91\cdashline1-8

GDFT0.990.810.1116.00.290.594.64GDMEMIT0.930.890.0516.00.280.584.65GDLoRA1.00.710.0816.00.540.594.99\cdashline1-8

RMUFT1.00.790.1316.00.320.575.6RMUMEMIT0.970.930.0316.00.30.565.58RMULoRA1.00.710.0516.00.290.5612.83Table 14:Detailed results for unlearningediting.
body
Composable Interventions for Language Models
4Results
2407.06483
5ConclusionWe conduct an extensive study of composing test-time interventions for language models.
Such interventions are an important step towards keeping expensively trained models up-to-date over extended deployments with respect to factuality, efficiency, and social responsibility.
Our study lays the foundation for a new range of intervention methods that arecomposablewith one another, proposing two metrics for measuring the composability of interventions.
Our setup aligns with the practical need to update the same models multiple times as use cases evolve and training remains expensive.
We deploy this new framework to investigate composability between popular interventions spanning knowledge editing, machine unlearning, and model compression.
We show that these interventions interact in non-trivial ways, pointing to concrete directions for future work.
We further discover guidance for current practice: model compression harms other interventions, unlearning compressed models is especially hard, and the order in which interventions are applied changes their outcomes substantially.
Our composability metrics also pave the way for formal investigations and practical principles.
Finally, our framework is general and naturally extends to a broad range of test-time interventions.AcknowledgementsWe thank the University of Virginia Research Computing team for providing access to excellent high-performance computing resources.References[1]K. Singhal, T. Tu, J. Gottweis, R. Sayres, E. Wulczyn, L. Hou, K. Clark, S. Pfohl, H. Cole-Lewis, D. Neal,et al., “Towards expert-level medical question answering with large language models,”arXiv preprint arXiv:2305.09617, 2023.[2]S. Wu, O. Irsoy, S. Lu, V. Dabravolski, M. Dredze, S. Gehrmann, P. Kambadur, D. Rosenberg, and G. Mann, “Bloomberggpt: A large language model for finance,”arXiv preprint arXiv:2303.17564, 2023.[3]R. Taylor, M. Kardas, G. Cucurull, T. Scialom, A. Hartshorn, E. Saravia, A. Poulton, V. Kerkez, and R. Stojnic, “Galactica: A large language model for science,”arXiv preprint arXiv:2211.09085, 2022.[4]S. Zhong, Z. Huang, S. Gao, W. Wen, L. Lin, M. Zitnik, and P. Zhou, “Let’s think outside the box: Exploring leap-of-thought in large language models with creative humor generation,”arXiv preprint arXiv:2312.02439, 2023.[5]Z. Ji, N. Lee, R. Frieske, T. Yu, D. Su, Y. Xu, E. Ishii, Y. J. Bang, A. Madotto, and P. Fung, “Survey of hallucination in natural language generation,”ACM Computing Surveys, vol. 55, no. 12, pp. 1–38, 2023.[6]Y. Zhao, J. Zhang, I. Chern, S. Gao, P. Liu, J. He,et al., “Felm: Benchmarking factuality evaluation of large language models,”Advances in Neural Information Processing Systems, vol. 36, 2023.[7]J. Mendelsohn, R. L. Bras, Y. Choi, and M. Sap, “From dogwhistles to bullhorns: Unveiling coded rhetoric with language models,”arXiv preprint arXiv:2305.17174, 2023.[8]D. Jain, P. Kumar, S. Gehman, X. Zhou, T. Hartvigsen, and M. Sap, “PolygloToxicityPrompts: Multilingual evaluation of neural toxic degeneration in large language models,”arXiv preprint arXiv:2405.09373, 2024.[9]T. Hartvigsen, S. Gabriel, H. Palangi, M. Sap, D. Ray, and E. Kamar, “Toxigen: A large-scale machine-generated dataset for adversarial and implicit hate speech detection,” inAnnual Meeting of the Association for Computational Linguistics, 2022.[10]The President of the United States, “Executive order 14110 of october 30, 2023: Safe, secure, and trustworthy development and use of artificial intelligence.” Federal Register, 2023.Published November 1, 2023.[11]J. Tack, J. Kim, E. Mitchell, J. Shin, Y. W. Teh, and J. R. Schwarz, “Online adaptation of language models with a memory of amortized contexts,”arXiv preprint arXiv:2403.04317, 2024.[12]M. Grynbaum and R. Mac, “The times sues openai and microsoft over a.i. use of copyrighted work,”The New York Times, 2023.[13]J. Kaddour, J. Harris, M. Mozes, H. Bradley, R. Raileanu, and R. McHardy, “Challenges and applications of large language models,”arXiv preprint arXiv:2307.10169, 2023.[14]X. Zhu, J. Li, Y. Liu, C. Ma, and W. Wang, “A survey on model compression for large language models,”arXiv preprint arXiv:2308.07633, 2023.[15]E. Frantar, S. Ashkboos, T. Hoefler, and D. Alistarh, “GPTQ: Accurate post-training quantization for generative pre-trained transformers,” inInternational Conference on Learning Representations, 2023.[16]E. Frantar and D. Alistarh, “SparseGPT: Massive language models can be accurately pruned in one-shot,” inInternational Conference on Machine Learning, 2023.[17]V. Mazzia, A. Pedrani, A. Caciolai, K. Rottmann, and D. Bernardi, “A survey on knowledge editing of neural networks,”arXiv preprint arXiv:2310.19704, 2023.[18]T. Hartvigsen, S. Sankaranarayanan, H. Palangi, Y. Kim, and M. Ghassemi, “Aging with grace: Lifelong model editing with discrete key-value adaptors,” inAdvances in Neural Information Processing Systems, 2023.[19]K. Meng, D. Bau, A. Andonian, and Y. Belinkov, “Locating and editing factual associations in gpt,” inAdvances in Neural Information Processing Systems, 2022.[20]K. Meng, A. S. Sharma, A. Andonian, Y. Belinkov, and D. Bau, “Mass-editing memory in a transformer,” inInternational Conference on Learning Representations, 2023.[21]L. Yu, Q. Chen, J. Zhou, and L. He, “Melo: Enhancing model editing with neuron-indexed dynamic lora,” inProceedings of the AAAI Conference on Artificial Intelligence, vol. 38, pp. 19449–19457, 2024.[22]C. Tan, G. Zhang, and J. Fu, “Massive editing for large language models via meta learning,” inInternational Conference on Learning Representations, 2024.[23]J. Welbl, A. Glaese, J. Uesato, S. Dathathri, J. Mellor, L. A. Hendricks, K. Anderson, P. Kohli, B. Coppin, and P.-S. Huang, “Challenges in detoxifying language models,” inFindings of the Association for Computational Linguistics: EMNLP 2021, pp. 2447–2469, 2021.[24]C. Yu, S. Jeoung, A. Kasi, P. Yu, and H. Ji, “Unlearning bias in language models by partitioning gradients,” inFindings of the Association for Computational Linguistics: ACL 2023, pp. 6032–6048, 2023.[25]B. Wang, W. Ping, C. Xiao, P. Xu, M. Patwary, M. Shoeybi, B. Li, A. Anandkumar, and B. Catanzaro, “Exploring the limits of domain-adaptive training for detoxifying large-scale language models,”Advances in Neural Information Processing Systems, vol. 35, pp. 35811–35824, 2022.[26]X. Suau, P. Delobelle, K. Metcalf, A. Joulin, N. Apostoloff, L. Zappella, and P. Rodriguez, “Whispering experts: Neural interventions for toxicity mitigation in language models,” inForty-first International Conference on Machine Learning.[27]S. Liu, Y. Yao, J. Jia, S. Casper, N. Baracaldo, P. Hase, X. Xu, Y. Yao, H. Li, K. R. Varshney, M. Bansal, S. Koyejo, and Y. Liu, “Rethinking machine unlearning for large language models,”ArXiv, vol. abs/2402.08787, 2024.[28]A. Lynch, P. Guo, A. Ewart, S. Casper, and D. Hadfield-Menell, “Eight methods to evaluate robust unlearning in llms,”arXiv preprint arXiv:2402.16835, 2024.[29]R. Eldan and M. Russinovich, “Who’s harry potter? approximate unlearning in llms,”ArXiv, vol. abs/2310.02238, 2023.[30]C. Matzken, S. Eger, and I. Habernal, “Trade-offs between fairness and privacy in language modeling,”arXiv preprint arXiv:2305.14936, 2023.[31]Z. Xu, Z. Liu, B. Chen, Y. Tang, J. Wang, K. Zhou, X. Hu, and A. Shrivastava, “Compress, then prompt: Improving accuracy-efficiency trade-off of llm inference with transferable prompt,”arXiv preprint arXiv:2305.11186, 2023.[32]Y. Li, Y. Yu, C. Liang, P. He, N. Karampatziakis, W. Chen, and T. Zhao, “LoftQ: Lora-fine-tuning-aware quantization for large language models,” inInternational Conference on Learning Representations, 2024.[33]H. Touvron, L. Martin, K. Stone, P. Albert, A. Almahairi, Y. Babaei, N. Bashlykov, S. Batra, P. Bhargava, S. Bhosale,et al., “Llama 2: Open foundation and fine-tuned chat models,”arXiv preprint arXiv:2307.09288, 2023.[34]E. J. Hu, Y. Shen, P. Wallis, Z. Allen-Zhu, Y. Li, S. Wang, L. Wang, and W. Chen, “Lora: Low-rank adaptation of large language models,”arXiv preprint arXiv:2106.09685, 2021.[35]O. Levy, M. Seo, E. Choi, and L. Zettlemoyer, “Zero-shot relation extraction via reading comprehension,” inProceedings of the 21st Conference on Computational Natural Language Learning (CoNLL 2017) , pp. 333–342, 2017.[36]M. Sun, Z. Liu, A. Bair, and J. Z. Kolter, “A simple and effective pruning approach for large language models,”arXiv preprint arXiv:2306.11695, 2023.[37]J. Lin, J. Tang, H. Tang, S. Yang, X. Dang, and S. Han, “AWQ: Activation-aware weight quantization for llm compression and acceleration,”arXiv preprint arXiv:2306.00978, 2023.[38]J. Jang, D. Yoon, S. Yang, S. Cha, M. Lee, L. Logeswaran, and M. Seo, “Knowledge unlearning for mitigating privacy risks in language models,” inAnnual Meeting of the Association for Computational Linguistics, 2022.[39]P. Maini, Z. Feng, A. Schwarzschild, Z. C. Lipton, and J. Z. Kolter, “Tofu: A task of fictitious unlearning for llms,”ArXiv, vol. abs/2401.06121, 2024.[40]N. Li, A. Pan, A. Gopal, S. Yue, D. Berrios, A. Gatti, J. D. Li, A.-K. Dombrowski, S. Goel, L. Phan, G. Mukobi, N. Helm-Burger, R. Lababidi, L. Justen, A. B. Liu, M. Chen, I. Barrass, O. Zhang, X. Zhu, R. Tamirisa, B. Bharathi, A. Khoja, Z. Zhao, A. Herbert-Voss, C. B. Breuer, S. Marks, O. Patel, A. Zou, M. Mazeika, Z. Wang, P. Oswal, W. Liu, A. A. Hunt, J. Tienken-Harder, K. Y. Shih, K. Talley, J. Guan, R. Kaplan, I. Steneker, D. Campbell, B. Jokubaitis, A. Levinson, J. Wang, W. Qian, K. K. Karmakar, S. Basart, S. Fitz, M. Levine, P. Kumaraguru, U. Tupakula, V. Varadharajan, Y. Shoshitaishvili, J. Ba, K. M. Esvelt, A. Wang, and D. Hendrycks, “The wmdp benchmark: Measuring and reducing malicious use with unlearning,” 2024.[41]G. Gangadhar and K. Stratos, “Model editing by pure fine-tuning,” 2024.[42]C.-H. Hsueh, P. K.-M. Huang, T.-H. Lin, C.-W. Liao, H.-C. Fang, C.-W. Huang, and Y.-N. Chen, “Editing the mind of giants: An in-depth exploration of pitfalls of knowledge editing in large language models,”arXiv preprint arXiv:2406.01436, 2024.[43]W. Hua, J. Guo, M. Dong, H. Zhu, P. Ng, and Z. Wang, “Propagation and pitfalls: Reasoning-based assessment of knowledge editing through counterfactual tasks,”arXiv preprint arXiv:2401.17585, 2024.[44]Y. Yao, P. Wang, B. Tian, S. Cheng, Z. Li, S. Deng, H. Chen, and N. Zhang, “Editing large language models: Problems, methods, and opportunities,” 2023.[45]A. Karamolegkou, J. Li, L. Zhou, and A. Sogaard, “Copyright violations and large language models,”ArXiv, vol. abs/2310.13771, 2023.[46]M. Meeus, I. Shilov, M. Faysse, and Y.-A. de Montjoye, “Copyright traps for large language models,”ArXiv, vol. abs/2402.09363, 2024.[47]E. Sheng, K.-W. Chang, P. Natarajan, and N. Peng, “The woman worked as a babysitter: On biases in language generation,”ArXiv, vol. abs/1909.01326, 2019.[48]S. Gehman, S. Gururangan, M. Sap, Y. Choi, and N. A. Smith, “Realtoxicityprompts: Evaluating neural toxic degeneration in language models,” inFindings, 2020.[49]N. Carlini, C. Liu, Ú. Erlingsson, J. Kos, and D. X. Song, “The secret sharer: Evaluating and testing unintended memorization in neural networks,” inUSENIX Security Symposium, 2018.[50]D. Ippolito, F. Tramèr, M. Nasr, C. Zhang, M. Jagielski, K. Lee, C. A. Choquette-Choo, and N. Carlini, “Preventing verbatim memorization in language models gives a false sense of privacy,”ArXiv, vol. abs/2210.17546, 2022.[51]S. Biderman, U. S. Prashanth, L. Sutawika, H. Schoelkopf, Q. G. Anthony, S. Purohit, and E. Raf, “Emergent and predictable memorization in large language models,”ArXiv, vol. abs/2304.11158, 2023.[52]U. S. Prashanth, A. Deng, K. O’Brien, V. JyothirS, M. A. Khan, J. Borkar, C. A. Choquette-Choo, J. R. Fuehne, S. Biderman, T. Ke, K. Lee, and N. Saphra, “Recite, reconstruct, recollect: Memorization in lms as a multifaceted phenomenon,” 2024.[53]Y. Cao and J. Yang, “Towards making systems forget with machine unlearning,”2015 IEEE Symposium on Security and Privacy, pp. 463–480, 2015.[54]M. Dehghani, Y. Tay, A. A. Gritsenko, Z. Zhao, N. Houlsby, F. Diaz, D. Metzler, and O. Vinyals, “The benchmark lottery,”ArXiv, vol. abs/2107.07002, 2021.[55]S. Biderman, H. Schoelkopf, L. Sutawika, L. Gao, J. Tow, B. Abbasi, A. F. Aji, P. S. Ammanamanchi, S. Black, J. Clive, A. DiPofi, J. Etxaniz, B. Fattori, J. Z. Forde, C. Foster, M. Jaiswal, W. Y. Lee, H. Li, C. Lovering, N. Muennighoff, E. Pavlick, J. Phang, A. Skowron, S. Tan, X. Tang, K. A. Wang, G. I. Winata, F. Yvon, and A. Zou, “Lessons from the trenches on reproducible evaluation of language models,” 2024.[56]V. Gupta, D. Pantoja, C. Ross, A. Williams, and M. Ung, “Changing answer order can decrease mmlu accuracy,” 2024.[57]D. Hendrycks, C. Burns, S. Basart, A. Zou, M. Mazeika, D. X. Song, and J. Steinhardt, “Measuring massive multitask language understanding,”ArXiv, vol. abs/2009.03300, 2020.[58]L. Gao, J. Tow, B. Abbasi, S. Biderman, S. Black, A. DiPofi, C. Foster, L. Golding, J. Hsu, A. Le Noac’h, H. Li, K. McDonell, N. Muennighoff, C. Ociepa, J. Phang, L. Reynolds, H. Schoelkopf, A. Skowron, L. Sutawika, E. Tang, A. Thite, B. Wang, K. Wang, and A. Zou, “A framework for few-shot language model evaluation,” 12 2023.[59]AI@Meta, “Llama 3 model card,” 2024.[60]R. Zhang, L. Lin, Y. Bai, and S. Mei, “Negative preference optimization: From catastrophic collapse to effective unlearning,”ArXiv, vol. abs/2404.05868, 2024.[61]Q. Wang, B. Han, P. Yang, J. Zhu, T. Liu, and M. Sugiyama, “Unlearning with control: Assessing real-world utility for large language model unlearning,” 2024.[62]S. Hooker, A. C. Courville, G. Clark, Y. Dauphin, and A. Frome, “What do compressed deep neural networks forget,”arXiv: Learning, 2019.[63]M. Du, S. Mukherjee, Y. Cheng, M. Shokouhi, X. Hu, and A. H. Awadallah, “What do compressed large language models forget? robustness challenges in model compression,”ArXiv, vol. abs/2110.08419, 2021.[64]A. H. Azeemi, I. A. Qazi, and A. A. Raza, “Data pruning for efficient model pruning in neural machine translation,” inConference on Empirical Methods in Natural Language Processing, 2023.[65]D. N. M. Hoang, M. Cho, T. Merth, M. Rastegari, and Z. Wang, “Do compressed llms forget knowledge? an experimental study with practical implications,” 2023.[66]S. Sai, S. Namburi, M. N. Sreedhar, S. Srinivasan, and F. Sala, “The cost of compression: Investigating the impact of compression on parametric knowledge in language models,”ArXiv, vol. abs/2312.00960, 2023.[67]D. Brown, C. Godfrey, C. Nizinski, J. Tu, and H. Kvinge, “Edit at your own risk: evaluating the robustness of edited models to distribution shifts,” 2023.[68]J.-C. Gu, H. Xu, J.-Y. Ma, P. Lu, Z.-H. Ling, K.-W. Chang, and N. Peng, “Model editing harms general abilities of large language models: Regularization to the rescue,” 2024.[69]A. Zou, L. Phan, S. Chen, J. Campbell, P. Guo, R. Ren, A. Pan, X. Yin, M. Mazeika, A.-K. Dombrowski, S. Goel, N. Li, M. J. Byun, Z. Wang, A. Mallen, S. Basart, S. Koyejo, D. Song, M. Fredrikson, Z. Kolter, and D. Hendrycks, “Representation engineering: A top-down approach to ai transparency,”ArXiv, vol. abs/2310.01405, 2023.[70]S. Wang, Y. Zhu, H. Liu, Z. Zheng, C. Chen, and J. Li, “Knowledge editing for large language models: A survey,”arXiv preprint arXiv:2310.16218, 2023.[71]E. Mitchell, C. Lin, A. Bosselut, C. D. Manning, and C. Finn, “Memory-based model editing at scale,” inInternational Conference on Machine Learning, PMLR, 2022.[72]A. Sinitsin, V. Plokhotnyuk, D. Pyrkin, S. Popov, and A. Babenko, “Editable neural networks,” inInternational Conference on Learning Representations, 2020.[73]E. Mitchell, C. Lin, A. Bosselut, C. Finn, and C. D. Manning, “Fast model editing at scale,” inInternational Conference on Learning Representations, 2022.[74]Z. Zhong, Z. Wu, C. D. Manning, C. Potts, and D. Chen, “Mquake: Assessing knowledge editing in language models via multi-hop questions,” inProceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, 2023.[75]R. Cohen, E. Biran, O. Yoran, A. Globerson, and M. Geva, “Evaluating the ripple effects of knowledge editing in language models,”Transactions of the Association for Computational Linguistics, vol. 12, pp. 283–298, 2024.[76]N. De Cao, W. Aziz, and I. Titov, “Editing factual knowledge in language models,” inProceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pp. 6491–6506, 2021.[77]Z. Huang, Y. Shen, X. Zhang, J. Zhou, W. Rong, and Z. Xiong, “Transformer-patcher: One mistake worth one neuron,” inInternational Conference on Learning Representations, 2023.[78]J.-C. Gu, H.-X. Xu, J.-Y. Ma, P. Lu, Z.-H. Ling, K.-W. Chang, and N. Peng, “Model editing can hurt general abilities of large language models,”arXiv preprint arXiv:2401.04700, 2024.[79]J. Hoelscher-Obermaier, J. Persson, E. Kran, I. Konstas, and F. Barez, “Detecting edit failures in large language models: An improved specificity benchmark,”arXiv preprint arXiv:2305.17553, 2023.[80]Z. Li, N. Zhang, Y. Yao, M. Wang, X. Chen, and H. Chen, “Unveiling the pitfalls of knowledge editing for large language models,”arXiv preprint arXiv:2310.02129, 2023.[81]R. Hazra, S. Layek, S. Banerjee, and S. Poria, “Sowing the wind, reaping the whirlwind: The impact of editing language models,”arXiv preprint arXiv:2401.10647, 2024.[82]D. Brown, C. Godfrey, C. Nizinski, J. Tu, and H. Kvinge, “Robustness of edited neural networks,”arXiv preprint arXiv:2303.00046, 2023.[83]P. Hase, M. Bansal, B. Kim, and A. Ghandeharioun, “Does localization inform editing? surprising differences in causality-based localization vs. knowledge editing in language models,” inAdvances in Neural Information Processing Systems, 2023.[84]D. Powell, W. Gerych, and T. Hartvigsen, “TAXI: Evaluating categorical knowledge editing for language models,” inFindings of the Association for Computational Linguistics: ACL 2024, 2024.[85]T. B. Shaik, X. Tao, H. Xie, L. Li, X. Zhu, and Q. Li, “Exploring the landscape of machine unlearning: A comprehensive survey and taxonomy,”ArXiv, vol. abs/2305.06360, 2023.[86]J. Zhao, Z. Deng, D. Madras, J. Zou, and M. Ren, “Learning and forgetting unsafe examples in large language models,”ArXiv, vol. abs/2312.12736, 2023.[87]Z. Liu, G. Dou, Z. Tan, Y. Tian, and M. Jiang, “Towards safer large language models through machine unlearning,”ArXiv, vol. abs/2402.10058, 2024.[88]J. Jia, J. Liu, P. Ram, Y. Yao, G. Liu, Y. Liu, P. Sharma, and S. Liu, “Model sparsity can simplify machine unlearning,” inNeural Information Processing Systems, 2023.[89]G. Xiao, J. Lin, M. Seznec, H. Wu, J. Demouth, and S. Han, “Smoothquant: Accurate and efficient post-training quantization for large language models,” inInternational Conference on Machine Learning, pp. 38087–38099, PMLR, 2023.[90]L. Yin, Y. Wu, Z. Zhang, C.-Y. Hsieh, Y. Wang, Y. Jia, M. Pechenizkiy, Y. Liang, Z. Wang, and S. Liu, “Outlier weighed layerwise sparsity (owl) : A missing secret sauce for pruning llms to high sparsity,”arXiv preprint arXiv:2310.05175, 2023.[91]X. Ma, G. Fang, and X. Wang, “Llm-pruner: On the structural pruning of large language models,”arXiv preprint arXiv:2305.11627, 2023.[92]P. Wang, Z. Wang, Z. Li, Y. Gao, B. Yin, and X. Ren, “Scott: Self-consistent chain-of-thought distillation,”arXiv preprint arXiv:2305.01879, 2023.[93]S. Saha, P. Hase, and M. Bansal, “Can language models teach weaker agents? teacher explanations improve students via theory of mind,”arXiv preprint arXiv:2306.09299, 2023.[94]C.-Y. Hsieh, C.-L. Li, C.-K. Yeh, H. Nakhost, Y. Fujii, A. Ratner, R. Krishna, C.-Y. Lee, and T. Pfister, “Distilling step-by-step! outperforming larger language models with less training data and smaller model sizes,”arXiv preprint arXiv:2305.02301, 2023.[95]M. Xu, Y. L. Xu, and D. P. Mandic, “Tensorgpt: Efficient compression of the embedding layer in llms based on the tensor-train decomposition,”arXiv preprint arXiv:2307.00526, 2023.[96]P. Sharma, J. T. Ash, and D. Misra, “The truth is in there: Improving reasoning in language models with layer-selective rank reduction,”arXiv preprint arXiv:2312.13558, 2023.[97]Z. Yao, R. Yazdani Aminabadi, M. Zhang, X. Wu, C. Li, and Y. He, “Zeroquant: Efficient and affordable post-training quantization for large-scale transformers,”Advances in Neural Information Processing Systems, vol. 35, pp. 27168–27183, 2022.[98]T. Dettmers, M. Lewis, Y. Belkada, and L. Zettlemoyer, “Llm. int8 () : 8-bit matrix multiplication for transformers at scale,”arXiv preprint arXiv:2208.07339, 2022.[99]P. Wang, N. Zhang, X. Xie, Y. Yao, B. Tian, M. Wang, Z. Xi, S. Cheng, K. Liu, G. Zheng,et al., “Easyedit: An easy-to-use knowledge editing framework for large language models,”arXiv preprint arXiv:2308.07269, 2023.[100]S. Merity, C. Xiong, J. Bradbury, and R. Socher, “Pointer sentinel mixture models,” inInternational Conference on Learning Representations, 2016.[101]A. Golatkar, A. Achille, and S. Soatto, “Eternal sunshine of the spotless net: Selective forgetting in deep networks,”2020 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) , pp. 9301–9309, 2019.[102]J. Yao, E. Chien, M. Du, X. Niu, T. Wang, Z. Cheng, and X. Yue, “Machine unlearning of pre-trained large language models,”ArXiv, vol. abs/2402.15159, 2024.AppendixAppendix ALimitationsWhile our work provides the first look at how state-of-the-art methods compose, there are far more possible combinations of interventions, methods, models, and datasets than we can consider in one work.
So some popular methods are omitted.
We also prioritize broad experiments with many different intervention methods.
We hope future works will closely examine individual methods to study, improve, and extend beyond the mechanisms that drive their composability.
Lastly, we only experiment with Llama3-8B.
While our code is extendable and Llama3-8B is popular, it is unknown whether our findings generalize between language models.
We hope our codebase empowers others to explore more model sizes and architectures in future research.Appendix BBackground on InterventionsDesigning interventions for language models is a highly-active research area.
We focus on three key types of intervention.Knowledge Editing.
Knowledge editors aim to address factuality decay in LLMs: As the world changes, some facts an LLM learned during training become inaccurate.
For example, a model trained before 2020 would still predictThe latest pandemic in the US is“Swine Flu” until updated to predict “COVID”.
There are four general approaches to model editing: 1)  Memory-based methods that cache knowledge, 2)  Locate-then-Edit methods that selectively finetune parameter subsets, 3)  Hypernetwork methods that predict new model weights, and 4)  Prompt editors that add facts to prompts.
Most editors update singular facts, though recent works apply many simultaneously.
Others works embrace the practical need forsequentialedits.
Recent works have also begun investigating unintended impacts of editing on pre-trained models, while others are exploring multi-hop editing.
However, these works study impacts on model performance, not interactions with other types of interventions.Machine Unlearning.Machine Unlearning (MU)  refers to a broad set of techniques for modifying a model post-training to remove the influence of specific training examples or other undesirable properties444Most MU techniques for LMs are considered approximate unlearning in that they aim to modify the model to approximate the behavior of it not being trained on the unlearning target without retraining from scratch.. These undesirable properties are commonly referred to as the unlearning target. Crucially, MU must excel at causing the model to forget the unlearning target without adversely affecting overall utility by reducing performance on other domains. Another requirement is that MU must cause models to deeply forget knowledge such that the unlearning target can not be elicited by adversaries or probing. With the rise of generative models, recent work has explored MU for forgetting potentially copyrighted work, information about individuals, and potentially dangerous knowledge. Existing works have studied which types of examples models forget when compressed, whether compression hinders unlearning, and whether compression in itself can be used as an unlearning technique. While related to knowledge editing, the two areas have largely developed independently.Model Compression.
To mitigate the huge computational and memory demands of LLMs, plenty of classical techniques in model compression have been explored, such as quantization, network pruning, knowledge distillation, and low-rank factorization. We focus on quantization and network pruning,
which both make in-place updates to pre-trained model weights.
Quantization involves lowering the bit-precision of model parameters, effectively shrinking model size and expediting inference. Early works, such as ZeroQuantand LLM.int8, focused on fine-tuning quantization granularity, showing promising results predominantly at higher bit-widths, like 8-bit. More recent innovations like GPTQand AWQhave successfully pushed the boundaries by reducing bit-width to as low as 3 or 4 bits per weight, with negligible performance degradation.
Pruning is another compression methods, which aims to eliminate redundant components like weights and channels.
Techniques like SparseGPTand Wandasuccessfully maintain much of an LLM’s performance, even when discarding approximately 50% of its weights. OWLachieves higher levels of sparsity by reallocating sparsities across layers.Appendix CImplementation DetailsIn this section we outline the set up and hyperparameters used for knowledge editing, unlearning, and compression.C.1Knowledge Editing DetailsC.1.1Knowledge Editing MetricsWe adopt three widely-used editing metrics and three compression metrics to measure the effectiveness of these interventions. The editing metrics are:Edit Success:Assesses the post-edit model’s ability to provide the correct, expected answer. An issue arises here with the different levels of verbosity for the models. An unedited model might return the correct token, but use a differently phrased answer or arrange the correct tokens differently. We therefore adopt and F1-style measure used in recent workfor this.First, we determine the common tokens between the generated response and the ground truth, which are represented by the sets(generated response tokens without special tokens)  and(target tokens without padding) , respectively. The common tokens are given by.Ifis empty is empty, the F1 score is set to 0, as there are no common elements to compare.Otherwise, we calculate precision and recall. Precision is the ratio of the number of common tokens to the number of tokens in the generated response, and recall is the ratio of the number of common tokens to the number of target tokens. They are defined as:PrecisionRecallThe F1 score, which is the harmonic mean of precision and recall, is then calculated as:with the condition that if the denominatoris zero, the F1 score is set to 0.The final F1 score is the average of the F1 scores for all items. If there are no F1 scores (i.e., the list of scores is empty) , the average F1 score is set to 0.Edit Generalization:Measures the model’s capability to genuinely update knowledge, rather than just repeating keywords. This is tested by evaluating edit success on paraphrased prompts given by the respective dataset.Edit Locality:Examines the precision of edits, ensuring that only relevant, in-distribution knowledge is altered while out-of-distribution facts remain unaffected. In this setting, the F1 score is critical as testing on the pre-trained knowledge can result in varied answer phrasings.C.1.2Knowledge Editing DatasetZsRE: This dataset focuses on context-free question-answering. Given a question related to a subject and a relation, the model’s task is to provide the correct object as the answer.follow the procedure outlined in, to improve the locality sets used. As the initial locality sets were solely based on Natural Question annotations, we use the locality sets provided by.In all three datasets, original prompt rephrasings and locality sets are used to derive the respective metrics.C.2Model Editing DetailsC.2.1MEMIT DetailsWe use the state-of-the-art MEMITmodel editor, which applies batches of edits simultaneously.
The editing process was applied to layers 4 through 8 of the model, with a clamp normalization factor set at 4. The learning parameters adhered closely to the original implementation:v_num_grad_stepswas set to 25, accompanied by a learning rate (lr)  of 0.5, and using the last layer for loss calculation. Additionally, a weight decay (weight_decay)  of 0.001 was employed. The KL divergence contribution to the overall loss was controlled by aof 0.0625. Moreover, a second momentum adjustment was enabled, with an update weight of, to fine-tune the optimization process. The model generated a maximum length of 40 tokens and a batch size of 50, matching the number of edits being made. 10 repeats were made for each edit and the results averaged.C.2.2LoRA DetailsWe also employed LoRA as a model editor, specifically using the adalora variant for our experiments. The editing process utilized 70 gradient steps, with a learning rate of 0.005. The rank for the low-rank adaptation was set at 8, with a scaling factor of 32. There was no dropout applied to the adaptation matrices, and the normalization constraint was disabled. The target modules for the edits consisted of query and value projection layers. Consistent with the MEMIT method, the model generated sequences with a maximum length of 40 tokens and utilized a batch size of 50, matching the number of edits per batch. Each edit was repeated 10 times, and the results were averaged to ensure reliability. The data type for the computations was set totorch.float.C.2.3Fine-Tuning (FT)  DetailsFor fine-tuning (FT) , we focused on layers 4 through 8 of the model. The specific modules targeted for editing included the MLP down-projection (model.layers.{}.mlp.down_proj) , the general layer module (model.layers.{}) , the MLP module (model.layers.{}.mlp) , the self-attention module (model.layers.{}.self_attn) , the final layer normalization (model.norm) , and the language model head (lm_head) .The fine-tuning process was conducted with 25 optimization steps, using a learning rate of. No weight decay was applied. The normalization constraint was disabled. The optimization objective was set to target the new data and the data type for the computations was set to bfloat16.As with the MEMIT and LoRA methods, the model generated sequences with a maximum length of 40 tokens and used a batch size of 50, matching the number of edits per batch. Each edit was repeated 10 times, and the results were averaged for consistency and reliability.C.3Model Compression DetailsC.3.1Compression MetricsThe compression metrics include:Sparsity Ratio.The fraction of parameters in the model that are zero, indicating the extent to which the model is sparse.Average Bits.The mean number of bits utilized to represent each parameter in a quantized or sparsified neural network, reflecting the level of compression.C.3.2Compressor detailsWe use four state-of-the-art compression methods including two pruning methods:SparseGPT: an efficient one-shot pruning method tailored for large models. It converts the pruning process into solving large-scale sparse regression problems using an approximate solver. This approach enables rapid pruning on a single GPU with minimal accuracy loss, achieving 50-60% on large models.Wanda: is another popular method for pruning large language models that relies on a pruning metric that combines a weight’s magnitude with the norm of its corresponding input activations, determined from a small calibration dataset. The method focuses on selectively pruning weights within individual outputs of linear layers, aiming for high sparsity levels without modifying unpruned weights. Wanda is computationally efficient, executable in a single forward pass.and two quantization methods:GPTQ: an algorithm designed for efficient weight quantization in large-scale models. It revises the weight quantization approach by quantizing weights in a fixed order rather than a greedy order, which shows minimal performance difference, especially in larger models. GPTQ introduced a novel method where each weight is quantized column-by-column, reducing computational complexity.AWQ: is based on the premise that not all weights are equally critical for model performance, and it identifies a small fraction of salient weights whose quantization significantly impacts model accuracy. This identification is done by analyzing activation distributions rather than weight distributions, under the rationale that weights linked to larger activation magnitudes are more crucial.C.4Machine Unlearning DetailsWe implement three unlearning techniques: RMU, GA, and GD. The success of each technique hinges on the careful selection of hyperparameters. We describe our parameter searches and selected values in the following sections.C.4.1Representation Misdirection UnlearningIntroduced in, RMU intervenes on a specific layer of the LM to perturb activations for inputs related to the unlearning target. This technique has achieved impressive performance on the WMDP dataset, with performance on the unlearning targets dropping to near-random while overall model utility (MMLU)  minimally regresses. Wikitextacts as the retain set.RMU relies on multiple hyperparameters for selecting which layer to modify and how the loss should be calculated. We found that RMU is quite sensitive to hyperparameter choice, with most combinations either leaving the model unaffected or significantly harming model utility. Table5details the results from our hyperparameters search. We otherwise use the RMU repo’s555https://github.com/centerforaisafety/wmdpdefault settings.ParameterDescriptionSearchSelectedAlphaThe weight for the retain loss[1, 10, 100, 1000, 10000]1000LayerThe layer to modify[3, 17]3Maximum BatchesTraining duration RMU in batches[100, 150, 200, 250, 300]250Table 5:RMU Hyperparameter Search.Details for the grid search over possible RMU settings. Hyperparameters were selected using grid search.C.4.2Gradient AscentGA, a simple and widely studied unlearning method, compels the model to minimize the likelihood of the correct token. The underlying intuition is that models will learn to avoid generating tokens associated with the unlearning target. We only train the final 16 layers of the model due to memory constraints.ParameterDescriptionSearchSelectedLRThe learning rate[5e-6, 1e-6, … 5e-3, 1e-3]5e-5Training SamplesTrain size balanced across Cyber/Bio[10, 25, 50, 100, … 500, 1000, 2000]50Table 6:GA Hyperparameter Search.Details for the grid search over possible GA settings. Hyperparameters were selected using grid search.C.4.3Gradient DifferenceA significant challenge when leveraging GA is balancing reducing performance on the unlearning target while retaining overall model utility. Since GA reduces the likelihood of the correct token, it is fragile to catastrophic forgetting, where the model incorrectly generalizes to forgetting how to model natural language.GDaims to address this shortcoming by adding a retain loss term to the loss function. This term aims to maximize the likelihood of the correct token on an unrelated retain dataset. We use Wikitetxas the retain set. As with GA, we only train the final 16 layers of the model.Table7reports our hyperparameter search. Unlike, which found that GD provided only a marginal improvement over GA, we found that GD can perform significantly better if we upweight the retain loss term. Upweighting is necessary since it is easy for the model to significantly minimize the correct token likelihood, which causes it to drown out the retain loss term. Assigning a weight of 400 to the retain term solved this issue. This hyperparameter combination can make GD competitive with RMU, though still less composable (Section4) .ParameterDescriptionSearchSelectedAlphaThe weight for the retain loss[1, 2, 4, 6, 8, 10, 20, 40, 60, 80, 100]40LRThe learning rate[5e-6, 1e-6, … 5e-3, 1e-3]5e-5Training SamplesTrain size balanced across Cyber/Bio[10, 25, 50, 100, … 500, 1000, 2000]50Table 7:GD Hyperparameter Search.Details for the grid search over possible GD settings. Hyperparameters were selected using grid search.Appendix DExtended ResultsWe report the task performance for all compositions below and the baseline results, where only a single intervention is applied (TableD) . Perplexity on Wikitextis also included as another overall utility metric. Average Bits (AppendixC.3.1)  measures the degree to which the composition compresses the model.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLNone0.020.020.0416.000.580.625.54\cdashline1-9

FTNone0.990.820.1116.00.570.615.57MEMITNone0.890.850.0416.00.570.615.57LoRANone1.00.710.0616.00.560.6119.25\cdashline1-9

SparseGPT (0.25) None0.020.020.0312.250.560.615.87SparseGPT (0.35) None0.020.020.0310.750.540.586.34SparseGPT (0.45) None0.020.010.039.250.510.547.43SparseGPT (0.55) None0.010.010.037.750.450.4410.43SparseGPT (0.65) None0.00.010.036.250.30.2720.83SparseGPT (0.75) None0.010.010.034.750.260.2381.84Wanda (0.25) None0.020.020.0312.250.560.615.84Wanda (0.35) None0.020.020.0410.750.540.586.31Wanda (0.45) None0.020.020.039.250.50.527.52Wanda (0.55) None0.020.010.027.750.360.3712.5Wanda (0.65) None0.020.010.026.250.260.2347.18Wanda (0.75) None0.020.030.034.750.260.23257.46GPTQ (2-Bit) None0.00.00.032.250.250.243681.23GPTQ (3-Bit) None0.010.010.033.250.450.468.6GPTQ (4-Bit) None0.020.020.034.250.560.69.97GPTQ (8-Bit) None0.020.020.048.250.580.625.54AWQ (2-Bit) None0.00.00.02.250.240.271748954.75AWQ (3-Bit) None0.010.010.033.250.520.537.47AWQ (4-Bit) None0.020.020.034.250.570.65.91AWQ (5-Bit) None0.020.030.035.250.570.625.62AWQ (6-Bit) None0.020.030.046.250.580.625.56AWQ (8-Bit) None0.020.020.048.250.580.625.54\cdashline1-9

GANone0.00.00.016.00.460.5infGDNone0.020.00.0916.00.260.524.48RMUNone0.020.020.0316.00.290.575.56Table 8:Baseline intervention Results without composition.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLFTSparseGPT (0.25) 0.960.760.1112.250.550.65.9FTSparseGPT (0.35) 0.840.660.0910.750.530.576.38FTSparseGPT (0.45) 0.640.50.079.250.50.537.48FTSparseGPT (0.55) 0.330.260.047.750.440.4410.35FTSparseGPT (0.65) 0.120.10.036.250.330.321.54FTSparseGPT (0.75) 0.030.020.034.750.260.2388.73FTWanda (0.25) 0.960.760.1112.250.560.65.88FTWanda (0.35) 0.880.680.0910.750.530.576.35FTWanda (0.45) 0.650.50.069.250.490.527.57FTWanda (0.55) 0.310.240.047.750.350.3712.48FTWanda (0.65) 0.080.070.036.250.260.2346.37FTWanda (0.75) 0.020.020.034.750.260.23344.62FTGPTQ (2-Bit) 0.00.00.022.250.250.241995.71FTGPTQ (3-Bit) 0.270.20.043.250.430.469.03FTGPTQ (4-Bit) 0.790.60.114.250.540.5912.93FTGPTQ (8-Bit) 0.990.830.118.250.570.615.57FTAWQ (2-Bit) 0.00.00.02.250.240.271739216.25FTAWQ (3-Bit) 0.70.550.073.250.50.517.56FTAWQ (4-Bit) 0.950.790.14.250.550.595.95FTAWQ (5-Bit) 0.990.840.115.250.560.615.66FTAWQ (6-Bit) 0.990.840.16.250.570.615.59FTAWQ (8-Bit) 0.990.830.118.250.570.615.57\cdashline1-8

MEMITSparseGPT (0.25) 0.870.830.0412.250.560.65.89MEMITSparseGPT (0.35) 0.810.740.0310.750.540.576.4MEMITSparseGPT (0.45) 0.620.560.049.250.510.537.43MEMITSparseGPT (0.55) 0.280.20.037.750.430.4310.37MEMITSparseGPT (0.65) 0.030.030.036.250.30.2921.02MEMITSparseGPT (0.75) 0.010.010.024.750.260.2371.62MEMITWanda (0.25) 0.70.690.0312.250.540.595.87MEMITWanda (0.35) 0.850.780.0310.750.530.576.34MEMITWanda (0.45) 0.750.670.039.250.480.517.56MEMITWanda (0.55) 0.320.260.037.750.350.3612.33MEMITWanda (0.65) 0.010.010.026.250.260.2346.98MEMITWanda (0.75) 0.020.020.034.750.260.23290.27MEMITGPTQ (2-Bit) 0.00.00.022.250.250.241663.86MEMITGPTQ (3-Bit) 0.350.290.043.250.450.468.89MEMITGPTQ (4-Bit) 0.840.80.034.250.550.5913.17MEMITGPTQ (8-Bit) 0.940.90.048.250.570.615.56MEMITAWQ (2-Bit) 0.00.00.02.250.240.271738085.5MEMITAWQ (3-Bit) 0.830.790.043.250.50.57.54MEMITAWQ (4-Bit) 0.930.890.034.250.550.595.94MEMITAWQ (5-Bit) 0.920.890.035.250.570.615.65MEMITAWQ (6-Bit) 0.930.890.046.250.570.615.59MEMITAWQ (8-Bit) 0.940.90.048.250.570.615.56\cdashline1-8

LoRASparseGPT (0.25) 0.860.520.0412.250.550.613.81LoRASparseGPT (0.35) 0.620.370.0510.750.520.5710.6LoRASparseGPT (0.45) 0.360.220.049.250.480.5216.48LoRASparseGPT (0.55) 0.170.140.047.750.430.4323.46LoRASparseGPT (0.65) 0.070.060.036.250.320.353.89LoRASparseGPT (0.75) 0.030.030.024.750.260.25186.65LoRAWanda (0.25) 0.870.530.0412.250.550.69.23LoRAWanda (0.35) 0.640.40.0510.750.520.5711.01LoRAWanda (0.45) 0.370.250.039.250.460.4912.6LoRAWanda (0.55) 0.180.140.037.750.360.3633.59LoRAWanda (0.65) 0.050.040.026.250.260.24194.26LoRAWanda (0.75) 0.020.020.034.750.250.25914.43LoRAGPTQ (2-Bit) 0.00.00.012.250.250.2410465.63LoRAGPTQ (3-Bit) 0.080.060.023.250.40.4240.02LoRAGPTQ (4-Bit) 0.580.410.054.250.540.5863.45LoRAGPTQ (8-Bit) 1.00.710.068.250.570.6118.86LoRAAWQ (2-Bit) 0.00.00.02.250.240.251764088.0LoRAAWQ (3-Bit) 0.460.310.053.250.50.5124.29LoRAAWQ (4-Bit) 0.910.60.044.250.550.5916.37LoRAAWQ (5-Bit) 0.990.70.055.250.560.616.31LoRAAWQ (6-Bit) 1.00.710.066.250.560.6115.34LoRAAWQ (8-Bit) 1.00.720.068.250.570.6115.25Table 9:Detailed results for editingcompression.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLSparseGPT (0.25) FT0.990.770.0812.250.550.65.91SparseGPT (0.35) FT0.980.750.0610.750.540.586.37SparseGPT (0.45) FT0.960.710.069.250.510.547.49SparseGPT (0.55) FT0.950.660.057.750.440.4410.19SparseGPT (0.65) FT0.890.540.046.250.330.317.67SparseGPT (0.75) FT0.580.250.044.750.260.2498.18SparseGPT (0.25) MEMIT0.910.860.0412.250.560.595.89SparseGPT (0.35) MEMIT0.890.80.0310.750.540.576.38SparseGPT (0.45) MEMIT0.620.480.039.250.510.537.49SparseGPT (0.55) MEMIT0.370.270.037.750.440.4310.62SparseGPT (0.65) MEMIT0.180.130.036.250.320.2922.83SparseGPT (0.75) MEMIT0.110.080.044.750.250.23394.59SparseGPT (0.25) LoRA0.850.450.0412.250.540.610.69SparseGPT (0.35) LoRA0.760.420.0410.750.520.579.54SparseGPT (0.45) LoRA0.640.330.049.250.480.5210.52SparseGPT (0.55) LoRA0.550.290.037.750.430.4311.5SparseGPT (0.65) LoRA0.370.170.036.250.320.2923.13SparseGPT (0.75) LoRA0.080.060.034.750.260.2589.47\cdashline1-8

Wanda (0.25) FT0.990.80.0712.250.560.65.86Wanda (0.35) FT0.970.760.0510.750.540.586.32Wanda (0.45) FT0.940.690.049.250.490.527.52Wanda (0.55) FT0.80.50.037.750.360.3811.98Wanda (0.65) FT0.50.280.036.250.260.2341.41Wanda (0.75) FT0.050.040.034.750.260.23249.29Wanda (0.25) MEMIT0.950.90.0312.250.560.595.87Wanda (0.35) MEMIT0.860.760.0310.750.540.576.34Wanda (0.45) MEMIT0.720.590.039.250.490.517.54Wanda (0.55) MEMIT0.370.260.037.750.370.3712.07Wanda (0.65) MEMIT0.040.040.026.250.260.2340.49Wanda (0.75) MEMIT0.040.040.024.750.260.23280.04Wanda (0.25) LoRA0.950.540.0312.250.550.610.65Wanda (0.35) LoRA0.860.480.0410.750.520.577.98Wanda (0.45) LoRA0.740.430.049.250.470.59.18Wanda (0.55) LoRA0.680.370.037.750.360.3715.01Wanda (0.65) LoRA0.340.190.026.250.260.2357.63Wanda (0.75) LoRA0.060.060.034.750.260.23318.06\cdashline1-8

GPTQ (2-Bit) FT0.00.00.012.250.250.24970816.25GPTQ (3-Bit) FT0.10.080.033.250.410.429.96GPTQ (4-Bit) FT0.440.240.044.250.550.586.4GPTQ (8-Bit) FT0.990.830.118.250.570.615.57GPTQ (2-Bit) MEMIT0.00.00.012.250.250.24739248.62GPTQ (3-Bit) MEMIT0.120.10.033.250.410.429.81GPTQ (4-Bit) MEMIT0.810.690.044.250.550.586.38GPTQ (8-Bit) MEMIT0.950.910.038.250.570.615.57GPTQ (2-Bit) LoRA0.00.00.012.250.250.24695958.19GPTQ (3-Bit) LoRA0.010.020.023.250.370.3811.14GPTQ (4-Bit) LoRA0.210.140.054.250.530.579.54GPTQ (8-Bit) LoRA1.00.720.068.250.560.6119.99\cdashline1-8

AWQ (2-Bit) FT0.00.00.02.250.240.2633638.44AWQ (3-Bit) FT1.00.820.083.250.50.517.57AWQ (4-Bit) FT0.990.840.154.250.540.575.98AWQ (5-Bit) FT0.990.850.125.250.550.65.68AWQ (6-Bit) FT0.990.880.136.250.560.65.61AWQ (8-Bit) FT0.990.870.138.250.560.65.6AWQ (2-Bit) MEMIT0.00.00.02.250.240.261735678.75AWQ (3-Bit) MEMIT0.920.870.033.250.490.477.57AWQ (4-Bit) MEMIT0.920.890.044.250.550.585.97AWQ (5-Bit) MEMIT0.980.960.035.250.560.65.68AWQ (6-Bit) MEMIT0.980.920.046.250.570.65.62AWQ (8-Bit) MEMIT0.980.940.048.250.570.65.59AWQ (2-Bit) LoRA0.060.030.02.250.240.26141960.91AWQ (3-Bit) LoRA1.00.710.053.250.50.5155.44AWQ (4-Bit) LoRA1.00.740.074.250.550.5829.71AWQ (5-Bit) LoRA1.00.690.055.250.560.617.32AWQ (6-Bit) LoRA1.00.690.066.250.570.6116.6AWQ (8-Bit) LoRA1.00.730.068.250.570.6121.64Table 10:Detailed results for compressionediting.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLGASparseGPT (0.25) 0.00.00.012.250.460.51infGASparseGPT (0.35) 0.00.00.010.750.440.49infGASparseGPT (0.45) 0.00.00.09.250.420.45infGASparseGPT (0.55) 0.00.00.07.750.30.32infGASparseGPT (0.65) 0.00.00.06.250.250.25infGASparseGPT (0.75) 0.00.00.04.750.250.25infGAWanda (0.25) 0.00.00.012.250.460.52infGAWanda (0.35) 0.00.00.010.750.450.49infGAWanda (0.45) 0.00.00.09.250.40.43infGAWanda (0.55) 0.00.00.07.750.30.32infGAWanda (0.65) 0.00.00.06.250.250.25infGAWanda (0.75) 0.00.00.04.750.250.26infGAGPTQ (2-Bit) 0.00.00.02.250.250.25infGAGPTQ (3-Bit) 0.00.00.03.250.330.39infGAGPTQ (4-Bit) 0.00.00.04.250.430.48infGAGPTQ (8-Bit) 0.00.00.08.250.450.49infGAAWQ (2-Bit) 0.00.00.02.250.250.251769133.88GAAWQ (3-Bit) 0.00.00.03.250.410.44infGAAWQ (4-Bit) 0.00.00.04.250.430.47infGAAWQ (5-Bit) 0.00.00.05.250.460.5infGAAWQ (6-Bit) 0.00.00.06.250.450.49infGAAWQ (8-Bit) 0.00.00.08.250.450.49inf\cdashline1-8

GDSparseGPT (0.25) 0.00.010.0212.250.270.585.07GDSparseGPT (0.35) 0.00.00.0210.750.280.555.64GDSparseGPT (0.45) 0.00.00.029.250.280.517.02GDSparseGPT (0.55) 0.00.00.017.750.270.4210.6GDSparseGPT (0.65) 0.00.00.026.250.260.2725.97GDSparseGPT (0.75) 0.010.00.014.750.240.23119.71GDWanda (0.25) 0.00.010.0312.250.350.585.34GDWanda (0.35) 0.00.00.0210.750.380.566.61GDWanda (0.45) 0.00.00.019.250.460.519.9GDWanda (0.55) 0.00.00.07.750.370.3922.22GDWanda (0.65) 0.00.00.06.250.260.23184.75GDWanda (0.75) 0.00.00.04.750.260.233298.92GDGPTQ (2-Bit) 0.00.00.02.250.240.27infGDGPTQ (3-Bit) 0.00.00.03.250.240.3infGDGPTQ (4-Bit) 0.00.00.04.250.240.325.106230513313957e+35GDGPTQ (8-Bit) 0.00.00.08.250.240.352.445586872346403e+26GDAWQ (2-Bit) 0.00.00.02.250.240.272598854.5GDAWQ (3-Bit) 0.00.00.03.250.330.33infGDAWQ (4-Bit) 0.00.00.04.250.240.352.0452847856930923e+28GDAWQ (5-Bit) 0.00.00.05.250.250.361.0052004503012292e+28GDAWQ (6-Bit) 0.00.00.06.250.250.356.499345951176105e+25GDAWQ (8-Bit) 0.00.00.08.250.250.359.430016153317283e+25\cdashline1-8

RMUSparseGPT (0.25) 0.020.020.0312.250.280.565.93RMUSparseGPT (0.35) 0.020.020.0310.750.280.536.39RMUSparseGPT (0.45) 0.020.020.039.250.270.488.14RMUSparseGPT (0.55) 0.010.010.037.750.260.3910.58RMUSparseGPT (0.65) 0.00.010.036.250.250.2635.97RMUSparseGPT (0.75) 0.010.010.034.750.250.23101.32RMUWanda (0.25) 0.020.020.0412.250.290.565.88RMUWanda (0.35) 0.020.020.0310.750.280.536.36RMUWanda (0.45) 0.020.010.039.250.290.487.71RMUWanda (0.55) 0.010.010.037.750.270.3513.73RMUWanda (0.65) 0.010.010.036.250.250.2352.67RMUWanda (0.75) 0.030.020.034.750.250.23334.45RMUGPTQ (2-Bit) 0.00.010.032.250.250.243451.67RMUGPTQ (3-Bit) 0.010.010.033.250.260.429.09RMUGPTQ (4-Bit) 0.020.020.034.250.270.5312.88RMUGPTQ (8-Bit) 0.020.020.038.250.290.575.56RMUAWQ (2-Bit) 0.00.00.02.250.240.261726409.12RMUAWQ (3-Bit) 0.010.010.033.250.270.457.55RMUAWQ (4-Bit) 0.020.020.034.250.290.565.95RMUAWQ (5-Bit) 0.020.030.045.250.280.565.68RMUAWQ (6-Bit) 0.020.020.036.250.280.565.62RMUAWQ (8-Bit) 0.020.020.038.250.290.575.56Table 11:Detailed results for unlearningcompression.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLSparseGPT (0.25) GA0.00.00.012.250.430.45infSparseGPT (0.35) GA0.00.00.010.750.340.36infSparseGPT (0.45) GA0.00.00.09.250.310.33infSparseGPT (0.55) GA0.00.00.07.750.250.25infSparseGPT (0.65) GA0.00.00.06.250.270.28infSparseGPT (0.75) GA0.010.010.034.750.260.23infSparseGPT (0.25) GD0.010.00.0212.250.50.4813.32SparseGPT (0.35) GD0.010.010.0110.750.290.252712.26SparseGPT (0.45) GD0.00.00.09.250.250.481.246813090597988e+19SparseGPT (0.55) GD0.00.00.027.750.350.369.35SparseGPT (0.65) GD0.00.00.016.250.240.26infSparseGPT (0.75) GD0.00.010.014.750.260.231437619072.0SparseGPT (0.25) RMU0.010.020.0412.250.310.575.92SparseGPT (0.35) RMU0.00.020.0210.750.310.546.35SparseGPT (0.45) RMU0.010.010.039.250.310.517.86SparseGPT (0.55) RMU0.010.010.037.750.290.4110.39SparseGPT (0.65) RMU0.00.00.026.250.260.2430.26SparseGPT (0.75) RMU0.010.010.024.750.250.2397.03\cdashline1-8

Wanda (0.25) GA0.00.00.012.250.490.54infWanda (0.35) GA0.00.00.010.750.440.51infWanda (0.45) GA0.00.00.09.250.410.43infWanda (0.55) GA0.00.00.07.750.320.34infWanda (0.65) GA0.00.00.06.250.260.26infWanda (0.75) GA0.010.010.034.750.260.238.12597183809491e+36Wanda (0.25) GD0.00.00.012.250.520.5713.26Wanda (0.35) GD0.00.00.010.750.450.4857480503296.0Wanda (0.45) GD0.00.00.09.250.50.5349.77Wanda (0.55) GD0.00.00.07.750.390.35121.07Wanda (0.65) GD0.00.00.06.250.260.2316268.01Wanda (0.75) GD0.00.00.04.750.250.2510226426.0Wanda (0.25) RMU0.020.020.0412.250.320.575.86Wanda (0.35) RMU0.020.020.0310.750.340.566.33Wanda (0.45) RMU0.020.020.039.250.390.527.55Wanda (0.55) RMU0.010.010.037.750.360.3612.58Wanda (0.65) RMU0.010.010.026.250.260.2345.53Wanda (0.75) RMU0.020.030.024.750.260.23391.87\cdashline1-8

GPTQ (2-Bit) GA0.00.00.02.250.250.25infGPTQ (3-Bit) GA0.00.00.03.250.240.27infGPTQ (4-Bit) GA0.00.00.04.250.340.38infGPTQ (8-Bit) GA0.00.00.08.250.450.5infGPTQ (2-Bit) GD0.00.00.02.250.250.25infGPTQ (3-Bit) GD0.00.00.03.250.260.24628.24GPTQ (4-Bit) GD0.010.010.014.250.270.569.38GPTQ (8-Bit) GD0.010.020.058.250.260.47infGPTQ (2-Bit) RMU0.00.00.02.250.250.241017825.94GPTQ (3-Bit) RMU0.010.010.023.250.320.3211.1GPTQ (4-Bit) RMU0.00.020.024.250.450.586.35GPTQ (8-Bit) RMU0.010.020.038.250.280.575.58\cdashline1-8

AWQ (2-Bit) GA0.00.00.02.250.240.27infAWQ (3-Bit) GA0.00.00.03.250.240.27infAWQ (4-Bit) GA0.00.00.04.250.450.45infAWQ (5-Bit) GA0.00.00.05.250.350.39infAWQ (6-Bit) GA0.00.00.06.250.250.3infAWQ (8-Bit) GA0.00.00.08.250.280.37infAWQ (2-Bit) GD0.00.00.022.250.240.274.496028624899119e+33AWQ (3-Bit) GD0.010.00.023.250.290.34166895.91AWQ (4-Bit) GD0.010.00.044.250.240.427.05AWQ (5-Bit) GD0.00.00.035.250.250.382685384.0AWQ (6-Bit) GD0.00.00.056.250.250.3127025.07AWQ (8-Bit) GD0.00.00.018.250.250.2549224794112.0AWQ (2-Bit) RMU0.00.00.02.250.240.271749321.75AWQ (3-Bit) RMU0.010.010.033.250.270.467.51AWQ (4-Bit) RMU0.020.020.034.250.270.556.06AWQ (5-Bit) RMU0.020.030.045.250.270.565.68AWQ (6-Bit) RMU0.020.020.046.250.280.565.6AWQ (8-Bit) RMU0.020.030.038.250.280.575.71Table 12:Detailed results for compressionunlearning.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLFTGA0.00.00.016.00.470.53infFTGD0.320.250.0716.00.290.412.8236466916108585e+31FTRMU0.990.820.116.00.280.565.61\cdashline1-8

MEMITGA0.00.00.016.00.40.45infMEMITGD0.530.490.0316.00.260.3627645056122880.0MEMITRMU0.960.890.0316.00.290.565.58\cdashline1-8

LoRAGA0.00.00.016.00.280.29infLoRAGD0.440.230.0516.00.30.4529.46LoRARMU1.00.680.0516.00.30.5234.56Table 13:Detailed results for editingunlearning.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLGAFT0.070.040.016.00.470.52infGAMEMIT0.480.410.016.00.450.49infGALoRA1.00.780.0316.00.340.3656.91\cdashline1-8

GDFT0.990.810.1116.00.290.594.64GDMEMIT0.930.890.0516.00.280.584.65GDLoRA1.00.710.0816.00.540.594.99\cdashline1-8

RMUFT1.00.790.1316.00.320.575.6RMUMEMIT0.970.930.0316.00.30.565.58RMULoRA1.00.710.0516.00.290.5612.83Table 14:Detailed results for unlearningediting.
body
Composable Interventions for Language Models
5Conclusion
2407.06483
AcknowledgementsWe thank the University of Virginia Research Computing team for providing access to excellent high-performance computing resources.References[1]K. Singhal, T. Tu, J. Gottweis, R. Sayres, E. Wulczyn, L. Hou, K. Clark, S. Pfohl, H. Cole-Lewis, D. Neal,et al., “Towards expert-level medical question answering with large language models,”arXiv preprint arXiv:2305.09617, 2023.[2]S. Wu, O. Irsoy, S. Lu, V. Dabravolski, M. Dredze, S. Gehrmann, P. Kambadur, D. Rosenberg, and G. Mann, “Bloomberggpt: A large language model for finance,”arXiv preprint arXiv:2303.17564, 2023.[3]R. Taylor, M. Kardas, G. Cucurull, T. Scialom, A. Hartshorn, E. Saravia, A. Poulton, V. Kerkez, and R. Stojnic, “Galactica: A large language model for science,”arXiv preprint arXiv:2211.09085, 2022.[4]S. Zhong, Z. Huang, S. Gao, W. Wen, L. Lin, M. Zitnik, and P. Zhou, “Let’s think outside the box: Exploring leap-of-thought in large language models with creative humor generation,”arXiv preprint arXiv:2312.02439, 2023.[5]Z. Ji, N. Lee, R. Frieske, T. Yu, D. Su, Y. Xu, E. Ishii, Y. J. Bang, A. Madotto, and P. Fung, “Survey of hallucination in natural language generation,”ACM Computing Surveys, vol. 55, no. 12, pp. 1–38, 2023.[6]Y. Zhao, J. Zhang, I. Chern, S. Gao, P. Liu, J. He,et al., “Felm: Benchmarking factuality evaluation of large language models,”Advances in Neural Information Processing Systems, vol. 36, 2023.[7]J. Mendelsohn, R. L. Bras, Y. Choi, and M. Sap, “From dogwhistles to bullhorns: Unveiling coded rhetoric with language models,”arXiv preprint arXiv:2305.17174, 2023.[8]D. Jain, P. Kumar, S. Gehman, X. Zhou, T. Hartvigsen, and M. Sap, “PolygloToxicityPrompts: Multilingual evaluation of neural toxic degeneration in large language models,”arXiv preprint arXiv:2405.09373, 2024.[9]T. Hartvigsen, S. Gabriel, H. Palangi, M. Sap, D. Ray, and E. Kamar, “Toxigen: A large-scale machine-generated dataset for adversarial and implicit hate speech detection,” inAnnual Meeting of the Association for Computational Linguistics, 2022.[10]The President of the United States, “Executive order 14110 of october 30, 2023: Safe, secure, and trustworthy development and use of artificial intelligence.” Federal Register, 2023.Published November 1, 2023.[11]J. Tack, J. Kim, E. Mitchell, J. Shin, Y. W. Teh, and J. R. Schwarz, “Online adaptation of language models with a memory of amortized contexts,”arXiv preprint arXiv:2403.04317, 2024.[12]M. Grynbaum and R. Mac, “The times sues openai and microsoft over a.i. use of copyrighted work,”The New York Times, 2023.[13]J. Kaddour, J. Harris, M. Mozes, H. Bradley, R. Raileanu, and R. McHardy, “Challenges and applications of large language models,”arXiv preprint arXiv:2307.10169, 2023.[14]X. Zhu, J. Li, Y. Liu, C. Ma, and W. Wang, “A survey on model compression for large language models,”arXiv preprint arXiv:2308.07633, 2023.[15]E. Frantar, S. Ashkboos, T. Hoefler, and D. Alistarh, “GPTQ: Accurate post-training quantization for generative pre-trained transformers,” inInternational Conference on Learning Representations, 2023.[16]E. Frantar and D. Alistarh, “SparseGPT: Massive language models can be accurately pruned in one-shot,” inInternational Conference on Machine Learning, 2023.[17]V. Mazzia, A. Pedrani, A. Caciolai, K. Rottmann, and D. Bernardi, “A survey on knowledge editing of neural networks,”arXiv preprint arXiv:2310.19704, 2023.[18]T. Hartvigsen, S. Sankaranarayanan, H. Palangi, Y. Kim, and M. Ghassemi, “Aging with grace: Lifelong model editing with discrete key-value adaptors,” inAdvances in Neural Information Processing Systems, 2023.[19]K. Meng, D. Bau, A. Andonian, and Y. Belinkov, “Locating and editing factual associations in gpt,” inAdvances in Neural Information Processing Systems, 2022.[20]K. Meng, A. S. Sharma, A. Andonian, Y. Belinkov, and D. Bau, “Mass-editing memory in a transformer,” inInternational Conference on Learning Representations, 2023.[21]L. Yu, Q. Chen, J. Zhou, and L. He, “Melo: Enhancing model editing with neuron-indexed dynamic lora,” inProceedings of the AAAI Conference on Artificial Intelligence, vol. 38, pp. 19449–19457, 2024.[22]C. Tan, G. Zhang, and J. Fu, “Massive editing for large language models via meta learning,” inInternational Conference on Learning Representations, 2024.[23]J. Welbl, A. Glaese, J. Uesato, S. Dathathri, J. Mellor, L. A. Hendricks, K. Anderson, P. Kohli, B. Coppin, and P.-S. Huang, “Challenges in detoxifying language models,” inFindings of the Association for Computational Linguistics: EMNLP 2021, pp. 2447–2469, 2021.[24]C. Yu, S. Jeoung, A. Kasi, P. Yu, and H. Ji, “Unlearning bias in language models by partitioning gradients,” inFindings of the Association for Computational Linguistics: ACL 2023, pp. 6032–6048, 2023.[25]B. Wang, W. Ping, C. Xiao, P. Xu, M. Patwary, M. Shoeybi, B. Li, A. Anandkumar, and B. Catanzaro, “Exploring the limits of domain-adaptive training for detoxifying large-scale language models,”Advances in Neural Information Processing Systems, vol. 35, pp. 35811–35824, 2022.[26]X. Suau, P. Delobelle, K. Metcalf, A. Joulin, N. Apostoloff, L. Zappella, and P. Rodriguez, “Whispering experts: Neural interventions for toxicity mitigation in language models,” inForty-first International Conference on Machine Learning.[27]S. Liu, Y. Yao, J. Jia, S. Casper, N. Baracaldo, P. Hase, X. Xu, Y. Yao, H. Li, K. R. Varshney, M. Bansal, S. Koyejo, and Y. Liu, “Rethinking machine unlearning for large language models,”ArXiv, vol. abs/2402.08787, 2024.[28]A. Lynch, P. Guo, A. Ewart, S. Casper, and D. Hadfield-Menell, “Eight methods to evaluate robust unlearning in llms,”arXiv preprint arXiv:2402.16835, 2024.[29]R. Eldan and M. Russinovich, “Who’s harry potter? approximate unlearning in llms,”ArXiv, vol. abs/2310.02238, 2023.[30]C. Matzken, S. Eger, and I. Habernal, “Trade-offs between fairness and privacy in language modeling,”arXiv preprint arXiv:2305.14936, 2023.[31]Z. Xu, Z. Liu, B. Chen, Y. Tang, J. Wang, K. Zhou, X. Hu, and A. Shrivastava, “Compress, then prompt: Improving accuracy-efficiency trade-off of llm inference with transferable prompt,”arXiv preprint arXiv:2305.11186, 2023.[32]Y. Li, Y. Yu, C. Liang, P. He, N. Karampatziakis, W. Chen, and T. Zhao, “LoftQ: Lora-fine-tuning-aware quantization for large language models,” inInternational Conference on Learning Representations, 2024.[33]H. Touvron, L. Martin, K. Stone, P. Albert, A. Almahairi, Y. Babaei, N. Bashlykov, S. Batra, P. Bhargava, S. Bhosale,et al., “Llama 2: Open foundation and fine-tuned chat models,”arXiv preprint arXiv:2307.09288, 2023.[34]E. J. Hu, Y. Shen, P. Wallis, Z. Allen-Zhu, Y. Li, S. Wang, L. Wang, and W. Chen, “Lora: Low-rank adaptation of large language models,”arXiv preprint arXiv:2106.09685, 2021.[35]O. Levy, M. Seo, E. Choi, and L. Zettlemoyer, “Zero-shot relation extraction via reading comprehension,” inProceedings of the 21st Conference on Computational Natural Language Learning (CoNLL 2017) , pp. 333–342, 2017.[36]M. Sun, Z. Liu, A. Bair, and J. Z. Kolter, “A simple and effective pruning approach for large language models,”arXiv preprint arXiv:2306.11695, 2023.[37]J. Lin, J. Tang, H. Tang, S. Yang, X. Dang, and S. Han, “AWQ: Activation-aware weight quantization for llm compression and acceleration,”arXiv preprint arXiv:2306.00978, 2023.[38]J. Jang, D. Yoon, S. Yang, S. Cha, M. Lee, L. Logeswaran, and M. Seo, “Knowledge unlearning for mitigating privacy risks in language models,” inAnnual Meeting of the Association for Computational Linguistics, 2022.[39]P. Maini, Z. Feng, A. Schwarzschild, Z. C. Lipton, and J. Z. Kolter, “Tofu: A task of fictitious unlearning for llms,”ArXiv, vol. abs/2401.06121, 2024.[40]N. Li, A. Pan, A. Gopal, S. Yue, D. Berrios, A. Gatti, J. D. Li, A.-K. Dombrowski, S. Goel, L. Phan, G. Mukobi, N. Helm-Burger, R. Lababidi, L. Justen, A. B. Liu, M. Chen, I. Barrass, O. Zhang, X. Zhu, R. Tamirisa, B. Bharathi, A. Khoja, Z. Zhao, A. Herbert-Voss, C. B. Breuer, S. Marks, O. Patel, A. Zou, M. Mazeika, Z. Wang, P. Oswal, W. Liu, A. A. Hunt, J. Tienken-Harder, K. Y. Shih, K. Talley, J. Guan, R. Kaplan, I. Steneker, D. Campbell, B. Jokubaitis, A. Levinson, J. Wang, W. Qian, K. K. Karmakar, S. Basart, S. Fitz, M. Levine, P. Kumaraguru, U. Tupakula, V. Varadharajan, Y. Shoshitaishvili, J. Ba, K. M. Esvelt, A. Wang, and D. Hendrycks, “The wmdp benchmark: Measuring and reducing malicious use with unlearning,” 2024.[41]G. Gangadhar and K. Stratos, “Model editing by pure fine-tuning,” 2024.[42]C.-H. Hsueh, P. K.-M. Huang, T.-H. Lin, C.-W. Liao, H.-C. Fang, C.-W. Huang, and Y.-N. Chen, “Editing the mind of giants: An in-depth exploration of pitfalls of knowledge editing in large language models,”arXiv preprint arXiv:2406.01436, 2024.[43]W. Hua, J. Guo, M. Dong, H. Zhu, P. Ng, and Z. Wang, “Propagation and pitfalls: Reasoning-based assessment of knowledge editing through counterfactual tasks,”arXiv preprint arXiv:2401.17585, 2024.[44]Y. Yao, P. Wang, B. Tian, S. Cheng, Z. Li, S. Deng, H. Chen, and N. Zhang, “Editing large language models: Problems, methods, and opportunities,” 2023.[45]A. Karamolegkou, J. Li, L. Zhou, and A. Sogaard, “Copyright violations and large language models,”ArXiv, vol. abs/2310.13771, 2023.[46]M. Meeus, I. Shilov, M. Faysse, and Y.-A. de Montjoye, “Copyright traps for large language models,”ArXiv, vol. abs/2402.09363, 2024.[47]E. Sheng, K.-W. Chang, P. Natarajan, and N. Peng, “The woman worked as a babysitter: On biases in language generation,”ArXiv, vol. abs/1909.01326, 2019.[48]S. Gehman, S. Gururangan, M. Sap, Y. Choi, and N. A. Smith, “Realtoxicityprompts: Evaluating neural toxic degeneration in language models,” inFindings, 2020.[49]N. Carlini, C. Liu, Ú. Erlingsson, J. Kos, and D. X. Song, “The secret sharer: Evaluating and testing unintended memorization in neural networks,” inUSENIX Security Symposium, 2018.[50]D. Ippolito, F. Tramèr, M. Nasr, C. Zhang, M. Jagielski, K. Lee, C. A. Choquette-Choo, and N. Carlini, “Preventing verbatim memorization in language models gives a false sense of privacy,”ArXiv, vol. abs/2210.17546, 2022.[51]S. Biderman, U. S. Prashanth, L. Sutawika, H. Schoelkopf, Q. G. Anthony, S. Purohit, and E. Raf, “Emergent and predictable memorization in large language models,”ArXiv, vol. abs/2304.11158, 2023.[52]U. S. Prashanth, A. Deng, K. O’Brien, V. JyothirS, M. A. Khan, J. Borkar, C. A. Choquette-Choo, J. R. Fuehne, S. Biderman, T. Ke, K. Lee, and N. Saphra, “Recite, reconstruct, recollect: Memorization in lms as a multifaceted phenomenon,” 2024.[53]Y. Cao and J. Yang, “Towards making systems forget with machine unlearning,”2015 IEEE Symposium on Security and Privacy, pp. 463–480, 2015.[54]M. Dehghani, Y. Tay, A. A. Gritsenko, Z. Zhao, N. Houlsby, F. Diaz, D. Metzler, and O. Vinyals, “The benchmark lottery,”ArXiv, vol. abs/2107.07002, 2021.[55]S. Biderman, H. Schoelkopf, L. Sutawika, L. Gao, J. Tow, B. Abbasi, A. F. Aji, P. S. Ammanamanchi, S. Black, J. Clive, A. DiPofi, J. Etxaniz, B. Fattori, J. Z. Forde, C. Foster, M. Jaiswal, W. Y. Lee, H. Li, C. Lovering, N. Muennighoff, E. Pavlick, J. Phang, A. Skowron, S. Tan, X. Tang, K. A. Wang, G. I. Winata, F. Yvon, and A. Zou, “Lessons from the trenches on reproducible evaluation of language models,” 2024.[56]V. Gupta, D. Pantoja, C. Ross, A. Williams, and M. Ung, “Changing answer order can decrease mmlu accuracy,” 2024.[57]D. Hendrycks, C. Burns, S. Basart, A. Zou, M. Mazeika, D. X. Song, and J. Steinhardt, “Measuring massive multitask language understanding,”ArXiv, vol. abs/2009.03300, 2020.[58]L. Gao, J. Tow, B. Abbasi, S. Biderman, S. Black, A. DiPofi, C. Foster, L. Golding, J. Hsu, A. Le Noac’h, H. Li, K. McDonell, N. Muennighoff, C. Ociepa, J. Phang, L. Reynolds, H. Schoelkopf, A. Skowron, L. Sutawika, E. Tang, A. Thite, B. Wang, K. Wang, and A. Zou, “A framework for few-shot language model evaluation,” 12 2023.[59]AI@Meta, “Llama 3 model card,” 2024.[60]R. Zhang, L. Lin, Y. Bai, and S. Mei, “Negative preference optimization: From catastrophic collapse to effective unlearning,”ArXiv, vol. abs/2404.05868, 2024.[61]Q. Wang, B. Han, P. Yang, J. Zhu, T. Liu, and M. Sugiyama, “Unlearning with control: Assessing real-world utility for large language model unlearning,” 2024.[62]S. Hooker, A. C. Courville, G. Clark, Y. Dauphin, and A. Frome, “What do compressed deep neural networks forget,”arXiv: Learning, 2019.[63]M. Du, S. Mukherjee, Y. Cheng, M. Shokouhi, X. Hu, and A. H. Awadallah, “What do compressed large language models forget? robustness challenges in model compression,”ArXiv, vol. abs/2110.08419, 2021.[64]A. H. Azeemi, I. A. Qazi, and A. A. Raza, “Data pruning for efficient model pruning in neural machine translation,” inConference on Empirical Methods in Natural Language Processing, 2023.[65]D. N. M. Hoang, M. Cho, T. Merth, M. Rastegari, and Z. Wang, “Do compressed llms forget knowledge? an experimental study with practical implications,” 2023.[66]S. Sai, S. Namburi, M. N. Sreedhar, S. Srinivasan, and F. Sala, “The cost of compression: Investigating the impact of compression on parametric knowledge in language models,”ArXiv, vol. abs/2312.00960, 2023.[67]D. Brown, C. Godfrey, C. Nizinski, J. Tu, and H. Kvinge, “Edit at your own risk: evaluating the robustness of edited models to distribution shifts,” 2023.[68]J.-C. Gu, H. Xu, J.-Y. Ma, P. Lu, Z.-H. Ling, K.-W. Chang, and N. Peng, “Model editing harms general abilities of large language models: Regularization to the rescue,” 2024.[69]A. Zou, L. Phan, S. Chen, J. Campbell, P. Guo, R. Ren, A. Pan, X. Yin, M. Mazeika, A.-K. Dombrowski, S. Goel, N. Li, M. J. Byun, Z. Wang, A. Mallen, S. Basart, S. Koyejo, D. Song, M. Fredrikson, Z. Kolter, and D. Hendrycks, “Representation engineering: A top-down approach to ai transparency,”ArXiv, vol. abs/2310.01405, 2023.[70]S. Wang, Y. Zhu, H. Liu, Z. Zheng, C. Chen, and J. Li, “Knowledge editing for large language models: A survey,”arXiv preprint arXiv:2310.16218, 2023.[71]E. Mitchell, C. Lin, A. Bosselut, C. D. Manning, and C. Finn, “Memory-based model editing at scale,” inInternational Conference on Machine Learning, PMLR, 2022.[72]A. Sinitsin, V. Plokhotnyuk, D. Pyrkin, S. Popov, and A. Babenko, “Editable neural networks,” inInternational Conference on Learning Representations, 2020.[73]E. Mitchell, C. Lin, A. Bosselut, C. Finn, and C. D. Manning, “Fast model editing at scale,” inInternational Conference on Learning Representations, 2022.[74]Z. Zhong, Z. Wu, C. D. Manning, C. Potts, and D. Chen, “Mquake: Assessing knowledge editing in language models via multi-hop questions,” inProceedings of the 2023 Conference on Empirical Methods in Natural Language Processing, 2023.[75]R. Cohen, E. Biran, O. Yoran, A. Globerson, and M. Geva, “Evaluating the ripple effects of knowledge editing in language models,”Transactions of the Association for Computational Linguistics, vol. 12, pp. 283–298, 2024.[76]N. De Cao, W. Aziz, and I. Titov, “Editing factual knowledge in language models,” inProceedings of the 2021 Conference on Empirical Methods in Natural Language Processing, pp. 6491–6506, 2021.[77]Z. Huang, Y. Shen, X. Zhang, J. Zhou, W. Rong, and Z. Xiong, “Transformer-patcher: One mistake worth one neuron,” inInternational Conference on Learning Representations, 2023.[78]J.-C. Gu, H.-X. Xu, J.-Y. Ma, P. Lu, Z.-H. Ling, K.-W. Chang, and N. Peng, “Model editing can hurt general abilities of large language models,”arXiv preprint arXiv:2401.04700, 2024.[79]J. Hoelscher-Obermaier, J. Persson, E. Kran, I. Konstas, and F. Barez, “Detecting edit failures in large language models: An improved specificity benchmark,”arXiv preprint arXiv:2305.17553, 2023.[80]Z. Li, N. Zhang, Y. Yao, M. Wang, X. Chen, and H. Chen, “Unveiling the pitfalls of knowledge editing for large language models,”arXiv preprint arXiv:2310.02129, 2023.[81]R. Hazra, S. Layek, S. Banerjee, and S. Poria, “Sowing the wind, reaping the whirlwind: The impact of editing language models,”arXiv preprint arXiv:2401.10647, 2024.[82]D. Brown, C. Godfrey, C. Nizinski, J. Tu, and H. Kvinge, “Robustness of edited neural networks,”arXiv preprint arXiv:2303.00046, 2023.[83]P. Hase, M. Bansal, B. Kim, and A. Ghandeharioun, “Does localization inform editing? surprising differences in causality-based localization vs. knowledge editing in language models,” inAdvances in Neural Information Processing Systems, 2023.[84]D. Powell, W. Gerych, and T. Hartvigsen, “TAXI: Evaluating categorical knowledge editing for language models,” inFindings of the Association for Computational Linguistics: ACL 2024, 2024.[85]T. B. Shaik, X. Tao, H. Xie, L. Li, X. Zhu, and Q. Li, “Exploring the landscape of machine unlearning: A comprehensive survey and taxonomy,”ArXiv, vol. abs/2305.06360, 2023.[86]J. Zhao, Z. Deng, D. Madras, J. Zou, and M. Ren, “Learning and forgetting unsafe examples in large language models,”ArXiv, vol. abs/2312.12736, 2023.[87]Z. Liu, G. Dou, Z. Tan, Y. Tian, and M. Jiang, “Towards safer large language models through machine unlearning,”ArXiv, vol. abs/2402.10058, 2024.[88]J. Jia, J. Liu, P. Ram, Y. Yao, G. Liu, Y. Liu, P. Sharma, and S. Liu, “Model sparsity can simplify machine unlearning,” inNeural Information Processing Systems, 2023.[89]G. Xiao, J. Lin, M. Seznec, H. Wu, J. Demouth, and S. Han, “Smoothquant: Accurate and efficient post-training quantization for large language models,” inInternational Conference on Machine Learning, pp. 38087–38099, PMLR, 2023.[90]L. Yin, Y. Wu, Z. Zhang, C.-Y. Hsieh, Y. Wang, Y. Jia, M. Pechenizkiy, Y. Liang, Z. Wang, and S. Liu, “Outlier weighed layerwise sparsity (owl) : A missing secret sauce for pruning llms to high sparsity,”arXiv preprint arXiv:2310.05175, 2023.[91]X. Ma, G. Fang, and X. Wang, “Llm-pruner: On the structural pruning of large language models,”arXiv preprint arXiv:2305.11627, 2023.[92]P. Wang, Z. Wang, Z. Li, Y. Gao, B. Yin, and X. Ren, “Scott: Self-consistent chain-of-thought distillation,”arXiv preprint arXiv:2305.01879, 2023.[93]S. Saha, P. Hase, and M. Bansal, “Can language models teach weaker agents? teacher explanations improve students via theory of mind,”arXiv preprint arXiv:2306.09299, 2023.[94]C.-Y. Hsieh, C.-L. Li, C.-K. Yeh, H. Nakhost, Y. Fujii, A. Ratner, R. Krishna, C.-Y. Lee, and T. Pfister, “Distilling step-by-step! outperforming larger language models with less training data and smaller model sizes,”arXiv preprint arXiv:2305.02301, 2023.[95]M. Xu, Y. L. Xu, and D. P. Mandic, “Tensorgpt: Efficient compression of the embedding layer in llms based on the tensor-train decomposition,”arXiv preprint arXiv:2307.00526, 2023.[96]P. Sharma, J. T. Ash, and D. Misra, “The truth is in there: Improving reasoning in language models with layer-selective rank reduction,”arXiv preprint arXiv:2312.13558, 2023.[97]Z. Yao, R. Yazdani Aminabadi, M. Zhang, X. Wu, C. Li, and Y. He, “Zeroquant: Efficient and affordable post-training quantization for large-scale transformers,”Advances in Neural Information Processing Systems, vol. 35, pp. 27168–27183, 2022.[98]T. Dettmers, M. Lewis, Y. Belkada, and L. Zettlemoyer, “Llm. int8 () : 8-bit matrix multiplication for transformers at scale,”arXiv preprint arXiv:2208.07339, 2022.[99]P. Wang, N. Zhang, X. Xie, Y. Yao, B. Tian, M. Wang, Z. Xi, S. Cheng, K. Liu, G. Zheng,et al., “Easyedit: An easy-to-use knowledge editing framework for large language models,”arXiv preprint arXiv:2308.07269, 2023.[100]S. Merity, C. Xiong, J. Bradbury, and R. Socher, “Pointer sentinel mixture models,” inInternational Conference on Learning Representations, 2016.[101]A. Golatkar, A. Achille, and S. Soatto, “Eternal sunshine of the spotless net: Selective forgetting in deep networks,”2020 IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR) , pp. 9301–9309, 2019.[102]J. Yao, E. Chien, M. Du, X. Niu, T. Wang, Z. Cheng, and X. Yue, “Machine unlearning of pre-trained large language models,”ArXiv, vol. abs/2402.15159, 2024.AppendixAppendix ALimitationsWhile our work provides the first look at how state-of-the-art methods compose, there are far more possible combinations of interventions, methods, models, and datasets than we can consider in one work.
So some popular methods are omitted.
We also prioritize broad experiments with many different intervention methods.
We hope future works will closely examine individual methods to study, improve, and extend beyond the mechanisms that drive their composability.
Lastly, we only experiment with Llama3-8B.
While our code is extendable and Llama3-8B is popular, it is unknown whether our findings generalize between language models.
We hope our codebase empowers others to explore more model sizes and architectures in future research.Appendix BBackground on InterventionsDesigning interventions for language models is a highly-active research area.
We focus on three key types of intervention.Knowledge Editing.
Knowledge editors aim to address factuality decay in LLMs: As the world changes, some facts an LLM learned during training become inaccurate.
For example, a model trained before 2020 would still predictThe latest pandemic in the US is“Swine Flu” until updated to predict “COVID”.
There are four general approaches to model editing: 1)  Memory-based methods that cache knowledge, 2)  Locate-then-Edit methods that selectively finetune parameter subsets, 3)  Hypernetwork methods that predict new model weights, and 4)  Prompt editors that add facts to prompts.
Most editors update singular facts, though recent works apply many simultaneously.
Others works embrace the practical need forsequentialedits.
Recent works have also begun investigating unintended impacts of editing on pre-trained models, while others are exploring multi-hop editing.
However, these works study impacts on model performance, not interactions with other types of interventions.Machine Unlearning.Machine Unlearning (MU)  refers to a broad set of techniques for modifying a model post-training to remove the influence of specific training examples or other undesirable properties444Most MU techniques for LMs are considered approximate unlearning in that they aim to modify the model to approximate the behavior of it not being trained on the unlearning target without retraining from scratch.. These undesirable properties are commonly referred to as the unlearning target. Crucially, MU must excel at causing the model to forget the unlearning target without adversely affecting overall utility by reducing performance on other domains. Another requirement is that MU must cause models to deeply forget knowledge such that the unlearning target can not be elicited by adversaries or probing. With the rise of generative models, recent work has explored MU for forgetting potentially copyrighted work, information about individuals, and potentially dangerous knowledge. Existing works have studied which types of examples models forget when compressed, whether compression hinders unlearning, and whether compression in itself can be used as an unlearning technique. While related to knowledge editing, the two areas have largely developed independently.Model Compression.
To mitigate the huge computational and memory demands of LLMs, plenty of classical techniques in model compression have been explored, such as quantization, network pruning, knowledge distillation, and low-rank factorization. We focus on quantization and network pruning,
which both make in-place updates to pre-trained model weights.
Quantization involves lowering the bit-precision of model parameters, effectively shrinking model size and expediting inference. Early works, such as ZeroQuantand LLM.int8, focused on fine-tuning quantization granularity, showing promising results predominantly at higher bit-widths, like 8-bit. More recent innovations like GPTQand AWQhave successfully pushed the boundaries by reducing bit-width to as low as 3 or 4 bits per weight, with negligible performance degradation.
Pruning is another compression methods, which aims to eliminate redundant components like weights and channels.
Techniques like SparseGPTand Wandasuccessfully maintain much of an LLM’s performance, even when discarding approximately 50% of its weights. OWLachieves higher levels of sparsity by reallocating sparsities across layers.Appendix CImplementation DetailsIn this section we outline the set up and hyperparameters used for knowledge editing, unlearning, and compression.C.1Knowledge Editing DetailsC.1.1Knowledge Editing MetricsWe adopt three widely-used editing metrics and three compression metrics to measure the effectiveness of these interventions. The editing metrics are:Edit Success:Assesses the post-edit model’s ability to provide the correct, expected answer. An issue arises here with the different levels of verbosity for the models. An unedited model might return the correct token, but use a differently phrased answer or arrange the correct tokens differently. We therefore adopt and F1-style measure used in recent workfor this.First, we determine the common tokens between the generated response and the ground truth, which are represented by the sets(generated response tokens without special tokens)  and(target tokens without padding) , respectively. The common tokens are given by.Ifis empty is empty, the F1 score is set to 0, as there are no common elements to compare.Otherwise, we calculate precision and recall. Precision is the ratio of the number of common tokens to the number of tokens in the generated response, and recall is the ratio of the number of common tokens to the number of target tokens. They are defined as:PrecisionRecallThe F1 score, which is the harmonic mean of precision and recall, is then calculated as:with the condition that if the denominatoris zero, the F1 score is set to 0.The final F1 score is the average of the F1 scores for all items. If there are no F1 scores (i.e., the list of scores is empty) , the average F1 score is set to 0.Edit Generalization:Measures the model’s capability to genuinely update knowledge, rather than just repeating keywords. This is tested by evaluating edit success on paraphrased prompts given by the respective dataset.Edit Locality:Examines the precision of edits, ensuring that only relevant, in-distribution knowledge is altered while out-of-distribution facts remain unaffected. In this setting, the F1 score is critical as testing on the pre-trained knowledge can result in varied answer phrasings.C.1.2Knowledge Editing DatasetZsRE: This dataset focuses on context-free question-answering. Given a question related to a subject and a relation, the model’s task is to provide the correct object as the answer.follow the procedure outlined in, to improve the locality sets used. As the initial locality sets were solely based on Natural Question annotations, we use the locality sets provided by.In all three datasets, original prompt rephrasings and locality sets are used to derive the respective metrics.C.2Model Editing DetailsC.2.1MEMIT DetailsWe use the state-of-the-art MEMITmodel editor, which applies batches of edits simultaneously.
The editing process was applied to layers 4 through 8 of the model, with a clamp normalization factor set at 4. The learning parameters adhered closely to the original implementation:v_num_grad_stepswas set to 25, accompanied by a learning rate (lr)  of 0.5, and using the last layer for loss calculation. Additionally, a weight decay (weight_decay)  of 0.001 was employed. The KL divergence contribution to the overall loss was controlled by aof 0.0625. Moreover, a second momentum adjustment was enabled, with an update weight of, to fine-tune the optimization process. The model generated a maximum length of 40 tokens and a batch size of 50, matching the number of edits being made. 10 repeats were made for each edit and the results averaged.C.2.2LoRA DetailsWe also employed LoRA as a model editor, specifically using the adalora variant for our experiments. The editing process utilized 70 gradient steps, with a learning rate of 0.005. The rank for the low-rank adaptation was set at 8, with a scaling factor of 32. There was no dropout applied to the adaptation matrices, and the normalization constraint was disabled. The target modules for the edits consisted of query and value projection layers. Consistent with the MEMIT method, the model generated sequences with a maximum length of 40 tokens and utilized a batch size of 50, matching the number of edits per batch. Each edit was repeated 10 times, and the results were averaged to ensure reliability. The data type for the computations was set totorch.float.C.2.3Fine-Tuning (FT)  DetailsFor fine-tuning (FT) , we focused on layers 4 through 8 of the model. The specific modules targeted for editing included the MLP down-projection (model.layers.{}.mlp.down_proj) , the general layer module (model.layers.{}) , the MLP module (model.layers.{}.mlp) , the self-attention module (model.layers.{}.self_attn) , the final layer normalization (model.norm) , and the language model head (lm_head) .The fine-tuning process was conducted with 25 optimization steps, using a learning rate of. No weight decay was applied. The normalization constraint was disabled. The optimization objective was set to target the new data and the data type for the computations was set to bfloat16.As with the MEMIT and LoRA methods, the model generated sequences with a maximum length of 40 tokens and used a batch size of 50, matching the number of edits per batch. Each edit was repeated 10 times, and the results were averaged for consistency and reliability.C.3Model Compression DetailsC.3.1Compression MetricsThe compression metrics include:Sparsity Ratio.The fraction of parameters in the model that are zero, indicating the extent to which the model is sparse.Average Bits.The mean number of bits utilized to represent each parameter in a quantized or sparsified neural network, reflecting the level of compression.C.3.2Compressor detailsWe use four state-of-the-art compression methods including two pruning methods:SparseGPT: an efficient one-shot pruning method tailored for large models. It converts the pruning process into solving large-scale sparse regression problems using an approximate solver. This approach enables rapid pruning on a single GPU with minimal accuracy loss, achieving 50-60% on large models.Wanda: is another popular method for pruning large language models that relies on a pruning metric that combines a weight’s magnitude with the norm of its corresponding input activations, determined from a small calibration dataset. The method focuses on selectively pruning weights within individual outputs of linear layers, aiming for high sparsity levels without modifying unpruned weights. Wanda is computationally efficient, executable in a single forward pass.and two quantization methods:GPTQ: an algorithm designed for efficient weight quantization in large-scale models. It revises the weight quantization approach by quantizing weights in a fixed order rather than a greedy order, which shows minimal performance difference, especially in larger models. GPTQ introduced a novel method where each weight is quantized column-by-column, reducing computational complexity.AWQ: is based on the premise that not all weights are equally critical for model performance, and it identifies a small fraction of salient weights whose quantization significantly impacts model accuracy. This identification is done by analyzing activation distributions rather than weight distributions, under the rationale that weights linked to larger activation magnitudes are more crucial.C.4Machine Unlearning DetailsWe implement three unlearning techniques: RMU, GA, and GD. The success of each technique hinges on the careful selection of hyperparameters. We describe our parameter searches and selected values in the following sections.C.4.1Representation Misdirection UnlearningIntroduced in, RMU intervenes on a specific layer of the LM to perturb activations for inputs related to the unlearning target. This technique has achieved impressive performance on the WMDP dataset, with performance on the unlearning targets dropping to near-random while overall model utility (MMLU)  minimally regresses. Wikitextacts as the retain set.RMU relies on multiple hyperparameters for selecting which layer to modify and how the loss should be calculated. We found that RMU is quite sensitive to hyperparameter choice, with most combinations either leaving the model unaffected or significantly harming model utility. Table5details the results from our hyperparameters search. We otherwise use the RMU repo’s555https://github.com/centerforaisafety/wmdpdefault settings.ParameterDescriptionSearchSelectedAlphaThe weight for the retain loss[1, 10, 100, 1000, 10000]1000LayerThe layer to modify[3, 17]3Maximum BatchesTraining duration RMU in batches[100, 150, 200, 250, 300]250Table 5:RMU Hyperparameter Search.Details for the grid search over possible RMU settings. Hyperparameters were selected using grid search.C.4.2Gradient AscentGA, a simple and widely studied unlearning method, compels the model to minimize the likelihood of the correct token. The underlying intuition is that models will learn to avoid generating tokens associated with the unlearning target. We only train the final 16 layers of the model due to memory constraints.ParameterDescriptionSearchSelectedLRThe learning rate[5e-6, 1e-6, … 5e-3, 1e-3]5e-5Training SamplesTrain size balanced across Cyber/Bio[10, 25, 50, 100, … 500, 1000, 2000]50Table 6:GA Hyperparameter Search.Details for the grid search over possible GA settings. Hyperparameters were selected using grid search.C.4.3Gradient DifferenceA significant challenge when leveraging GA is balancing reducing performance on the unlearning target while retaining overall model utility. Since GA reduces the likelihood of the correct token, it is fragile to catastrophic forgetting, where the model incorrectly generalizes to forgetting how to model natural language.GDaims to address this shortcoming by adding a retain loss term to the loss function. This term aims to maximize the likelihood of the correct token on an unrelated retain dataset. We use Wikitetxas the retain set. As with GA, we only train the final 16 layers of the model.Table7reports our hyperparameter search. Unlike, which found that GD provided only a marginal improvement over GA, we found that GD can perform significantly better if we upweight the retain loss term. Upweighting is necessary since it is easy for the model to significantly minimize the correct token likelihood, which causes it to drown out the retain loss term. Assigning a weight of 400 to the retain term solved this issue. This hyperparameter combination can make GD competitive with RMU, though still less composable (Section4) .ParameterDescriptionSearchSelectedAlphaThe weight for the retain loss[1, 2, 4, 6, 8, 10, 20, 40, 60, 80, 100]40LRThe learning rate[5e-6, 1e-6, … 5e-3, 1e-3]5e-5Training SamplesTrain size balanced across Cyber/Bio[10, 25, 50, 100, … 500, 1000, 2000]50Table 7:GD Hyperparameter Search.Details for the grid search over possible GD settings. Hyperparameters were selected using grid search.Appendix DExtended ResultsWe report the task performance for all compositions below and the baseline results, where only a single intervention is applied (TableD) . Perplexity on Wikitextis also included as another overall utility metric. Average Bits (AppendixC.3.1)  measures the degree to which the composition compresses the model.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLNone0.020.020.0416.000.580.625.54\cdashline1-9

FTNone0.990.820.1116.00.570.615.57MEMITNone0.890.850.0416.00.570.615.57LoRANone1.00.710.0616.00.560.6119.25\cdashline1-9

SparseGPT (0.25) None0.020.020.0312.250.560.615.87SparseGPT (0.35) None0.020.020.0310.750.540.586.34SparseGPT (0.45) None0.020.010.039.250.510.547.43SparseGPT (0.55) None0.010.010.037.750.450.4410.43SparseGPT (0.65) None0.00.010.036.250.30.2720.83SparseGPT (0.75) None0.010.010.034.750.260.2381.84Wanda (0.25) None0.020.020.0312.250.560.615.84Wanda (0.35) None0.020.020.0410.750.540.586.31Wanda (0.45) None0.020.020.039.250.50.527.52Wanda (0.55) None0.020.010.027.750.360.3712.5Wanda (0.65) None0.020.010.026.250.260.2347.18Wanda (0.75) None0.020.030.034.750.260.23257.46GPTQ (2-Bit) None0.00.00.032.250.250.243681.23GPTQ (3-Bit) None0.010.010.033.250.450.468.6GPTQ (4-Bit) None0.020.020.034.250.560.69.97GPTQ (8-Bit) None0.020.020.048.250.580.625.54AWQ (2-Bit) None0.00.00.02.250.240.271748954.75AWQ (3-Bit) None0.010.010.033.250.520.537.47AWQ (4-Bit) None0.020.020.034.250.570.65.91AWQ (5-Bit) None0.020.030.035.250.570.625.62AWQ (6-Bit) None0.020.030.046.250.580.625.56AWQ (8-Bit) None0.020.020.048.250.580.625.54\cdashline1-9

GANone0.00.00.016.00.460.5infGDNone0.020.00.0916.00.260.524.48RMUNone0.020.020.0316.00.290.575.56Table 8:Baseline intervention Results without composition.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLFTSparseGPT (0.25) 0.960.760.1112.250.550.65.9FTSparseGPT (0.35) 0.840.660.0910.750.530.576.38FTSparseGPT (0.45) 0.640.50.079.250.50.537.48FTSparseGPT (0.55) 0.330.260.047.750.440.4410.35FTSparseGPT (0.65) 0.120.10.036.250.330.321.54FTSparseGPT (0.75) 0.030.020.034.750.260.2388.73FTWanda (0.25) 0.960.760.1112.250.560.65.88FTWanda (0.35) 0.880.680.0910.750.530.576.35FTWanda (0.45) 0.650.50.069.250.490.527.57FTWanda (0.55) 0.310.240.047.750.350.3712.48FTWanda (0.65) 0.080.070.036.250.260.2346.37FTWanda (0.75) 0.020.020.034.750.260.23344.62FTGPTQ (2-Bit) 0.00.00.022.250.250.241995.71FTGPTQ (3-Bit) 0.270.20.043.250.430.469.03FTGPTQ (4-Bit) 0.790.60.114.250.540.5912.93FTGPTQ (8-Bit) 0.990.830.118.250.570.615.57FTAWQ (2-Bit) 0.00.00.02.250.240.271739216.25FTAWQ (3-Bit) 0.70.550.073.250.50.517.56FTAWQ (4-Bit) 0.950.790.14.250.550.595.95FTAWQ (5-Bit) 0.990.840.115.250.560.615.66FTAWQ (6-Bit) 0.990.840.16.250.570.615.59FTAWQ (8-Bit) 0.990.830.118.250.570.615.57\cdashline1-8

MEMITSparseGPT (0.25) 0.870.830.0412.250.560.65.89MEMITSparseGPT (0.35) 0.810.740.0310.750.540.576.4MEMITSparseGPT (0.45) 0.620.560.049.250.510.537.43MEMITSparseGPT (0.55) 0.280.20.037.750.430.4310.37MEMITSparseGPT (0.65) 0.030.030.036.250.30.2921.02MEMITSparseGPT (0.75) 0.010.010.024.750.260.2371.62MEMITWanda (0.25) 0.70.690.0312.250.540.595.87MEMITWanda (0.35) 0.850.780.0310.750.530.576.34MEMITWanda (0.45) 0.750.670.039.250.480.517.56MEMITWanda (0.55) 0.320.260.037.750.350.3612.33MEMITWanda (0.65) 0.010.010.026.250.260.2346.98MEMITWanda (0.75) 0.020.020.034.750.260.23290.27MEMITGPTQ (2-Bit) 0.00.00.022.250.250.241663.86MEMITGPTQ (3-Bit) 0.350.290.043.250.450.468.89MEMITGPTQ (4-Bit) 0.840.80.034.250.550.5913.17MEMITGPTQ (8-Bit) 0.940.90.048.250.570.615.56MEMITAWQ (2-Bit) 0.00.00.02.250.240.271738085.5MEMITAWQ (3-Bit) 0.830.790.043.250.50.57.54MEMITAWQ (4-Bit) 0.930.890.034.250.550.595.94MEMITAWQ (5-Bit) 0.920.890.035.250.570.615.65MEMITAWQ (6-Bit) 0.930.890.046.250.570.615.59MEMITAWQ (8-Bit) 0.940.90.048.250.570.615.56\cdashline1-8

LoRASparseGPT (0.25) 0.860.520.0412.250.550.613.81LoRASparseGPT (0.35) 0.620.370.0510.750.520.5710.6LoRASparseGPT (0.45) 0.360.220.049.250.480.5216.48LoRASparseGPT (0.55) 0.170.140.047.750.430.4323.46LoRASparseGPT (0.65) 0.070.060.036.250.320.353.89LoRASparseGPT (0.75) 0.030.030.024.750.260.25186.65LoRAWanda (0.25) 0.870.530.0412.250.550.69.23LoRAWanda (0.35) 0.640.40.0510.750.520.5711.01LoRAWanda (0.45) 0.370.250.039.250.460.4912.6LoRAWanda (0.55) 0.180.140.037.750.360.3633.59LoRAWanda (0.65) 0.050.040.026.250.260.24194.26LoRAWanda (0.75) 0.020.020.034.750.250.25914.43LoRAGPTQ (2-Bit) 0.00.00.012.250.250.2410465.63LoRAGPTQ (3-Bit) 0.080.060.023.250.40.4240.02LoRAGPTQ (4-Bit) 0.580.410.054.250.540.5863.45LoRAGPTQ (8-Bit) 1.00.710.068.250.570.6118.86LoRAAWQ (2-Bit) 0.00.00.02.250.240.251764088.0LoRAAWQ (3-Bit) 0.460.310.053.250.50.5124.29LoRAAWQ (4-Bit) 0.910.60.044.250.550.5916.37LoRAAWQ (5-Bit) 0.990.70.055.250.560.616.31LoRAAWQ (6-Bit) 1.00.710.066.250.560.6115.34LoRAAWQ (8-Bit) 1.00.720.068.250.570.6115.25Table 9:Detailed results for editingcompression.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLSparseGPT (0.25) FT0.990.770.0812.250.550.65.91SparseGPT (0.35) FT0.980.750.0610.750.540.586.37SparseGPT (0.45) FT0.960.710.069.250.510.547.49SparseGPT (0.55) FT0.950.660.057.750.440.4410.19SparseGPT (0.65) FT0.890.540.046.250.330.317.67SparseGPT (0.75) FT0.580.250.044.750.260.2498.18SparseGPT (0.25) MEMIT0.910.860.0412.250.560.595.89SparseGPT (0.35) MEMIT0.890.80.0310.750.540.576.38SparseGPT (0.45) MEMIT0.620.480.039.250.510.537.49SparseGPT (0.55) MEMIT0.370.270.037.750.440.4310.62SparseGPT (0.65) MEMIT0.180.130.036.250.320.2922.83SparseGPT (0.75) MEMIT0.110.080.044.750.250.23394.59SparseGPT (0.25) LoRA0.850.450.0412.250.540.610.69SparseGPT (0.35) LoRA0.760.420.0410.750.520.579.54SparseGPT (0.45) LoRA0.640.330.049.250.480.5210.52SparseGPT (0.55) LoRA0.550.290.037.750.430.4311.5SparseGPT (0.65) LoRA0.370.170.036.250.320.2923.13SparseGPT (0.75) LoRA0.080.060.034.750.260.2589.47\cdashline1-8

Wanda (0.25) FT0.990.80.0712.250.560.65.86Wanda (0.35) FT0.970.760.0510.750.540.586.32Wanda (0.45) FT0.940.690.049.250.490.527.52Wanda (0.55) FT0.80.50.037.750.360.3811.98Wanda (0.65) FT0.50.280.036.250.260.2341.41Wanda (0.75) FT0.050.040.034.750.260.23249.29Wanda (0.25) MEMIT0.950.90.0312.250.560.595.87Wanda (0.35) MEMIT0.860.760.0310.750.540.576.34Wanda (0.45) MEMIT0.720.590.039.250.490.517.54Wanda (0.55) MEMIT0.370.260.037.750.370.3712.07Wanda (0.65) MEMIT0.040.040.026.250.260.2340.49Wanda (0.75) MEMIT0.040.040.024.750.260.23280.04Wanda (0.25) LoRA0.950.540.0312.250.550.610.65Wanda (0.35) LoRA0.860.480.0410.750.520.577.98Wanda (0.45) LoRA0.740.430.049.250.470.59.18Wanda (0.55) LoRA0.680.370.037.750.360.3715.01Wanda (0.65) LoRA0.340.190.026.250.260.2357.63Wanda (0.75) LoRA0.060.060.034.750.260.23318.06\cdashline1-8

GPTQ (2-Bit) FT0.00.00.012.250.250.24970816.25GPTQ (3-Bit) FT0.10.080.033.250.410.429.96GPTQ (4-Bit) FT0.440.240.044.250.550.586.4GPTQ (8-Bit) FT0.990.830.118.250.570.615.57GPTQ (2-Bit) MEMIT0.00.00.012.250.250.24739248.62GPTQ (3-Bit) MEMIT0.120.10.033.250.410.429.81GPTQ (4-Bit) MEMIT0.810.690.044.250.550.586.38GPTQ (8-Bit) MEMIT0.950.910.038.250.570.615.57GPTQ (2-Bit) LoRA0.00.00.012.250.250.24695958.19GPTQ (3-Bit) LoRA0.010.020.023.250.370.3811.14GPTQ (4-Bit) LoRA0.210.140.054.250.530.579.54GPTQ (8-Bit) LoRA1.00.720.068.250.560.6119.99\cdashline1-8

AWQ (2-Bit) FT0.00.00.02.250.240.2633638.44AWQ (3-Bit) FT1.00.820.083.250.50.517.57AWQ (4-Bit) FT0.990.840.154.250.540.575.98AWQ (5-Bit) FT0.990.850.125.250.550.65.68AWQ (6-Bit) FT0.990.880.136.250.560.65.61AWQ (8-Bit) FT0.990.870.138.250.560.65.6AWQ (2-Bit) MEMIT0.00.00.02.250.240.261735678.75AWQ (3-Bit) MEMIT0.920.870.033.250.490.477.57AWQ (4-Bit) MEMIT0.920.890.044.250.550.585.97AWQ (5-Bit) MEMIT0.980.960.035.250.560.65.68AWQ (6-Bit) MEMIT0.980.920.046.250.570.65.62AWQ (8-Bit) MEMIT0.980.940.048.250.570.65.59AWQ (2-Bit) LoRA0.060.030.02.250.240.26141960.91AWQ (3-Bit) LoRA1.00.710.053.250.50.5155.44AWQ (4-Bit) LoRA1.00.740.074.250.550.5829.71AWQ (5-Bit) LoRA1.00.690.055.250.560.617.32AWQ (6-Bit) LoRA1.00.690.066.250.570.6116.6AWQ (8-Bit) LoRA1.00.730.068.250.570.6121.64Table 10:Detailed results for compressionediting.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLGASparseGPT (0.25) 0.00.00.012.250.460.51infGASparseGPT (0.35) 0.00.00.010.750.440.49infGASparseGPT (0.45) 0.00.00.09.250.420.45infGASparseGPT (0.55) 0.00.00.07.750.30.32infGASparseGPT (0.65) 0.00.00.06.250.250.25infGASparseGPT (0.75) 0.00.00.04.750.250.25infGAWanda (0.25) 0.00.00.012.250.460.52infGAWanda (0.35) 0.00.00.010.750.450.49infGAWanda (0.45) 0.00.00.09.250.40.43infGAWanda (0.55) 0.00.00.07.750.30.32infGAWanda (0.65) 0.00.00.06.250.250.25infGAWanda (0.75) 0.00.00.04.750.250.26infGAGPTQ (2-Bit) 0.00.00.02.250.250.25infGAGPTQ (3-Bit) 0.00.00.03.250.330.39infGAGPTQ (4-Bit) 0.00.00.04.250.430.48infGAGPTQ (8-Bit) 0.00.00.08.250.450.49infGAAWQ (2-Bit) 0.00.00.02.250.250.251769133.88GAAWQ (3-Bit) 0.00.00.03.250.410.44infGAAWQ (4-Bit) 0.00.00.04.250.430.47infGAAWQ (5-Bit) 0.00.00.05.250.460.5infGAAWQ (6-Bit) 0.00.00.06.250.450.49infGAAWQ (8-Bit) 0.00.00.08.250.450.49inf\cdashline1-8

GDSparseGPT (0.25) 0.00.010.0212.250.270.585.07GDSparseGPT (0.35) 0.00.00.0210.750.280.555.64GDSparseGPT (0.45) 0.00.00.029.250.280.517.02GDSparseGPT (0.55) 0.00.00.017.750.270.4210.6GDSparseGPT (0.65) 0.00.00.026.250.260.2725.97GDSparseGPT (0.75) 0.010.00.014.750.240.23119.71GDWanda (0.25) 0.00.010.0312.250.350.585.34GDWanda (0.35) 0.00.00.0210.750.380.566.61GDWanda (0.45) 0.00.00.019.250.460.519.9GDWanda (0.55) 0.00.00.07.750.370.3922.22GDWanda (0.65) 0.00.00.06.250.260.23184.75GDWanda (0.75) 0.00.00.04.750.260.233298.92GDGPTQ (2-Bit) 0.00.00.02.250.240.27infGDGPTQ (3-Bit) 0.00.00.03.250.240.3infGDGPTQ (4-Bit) 0.00.00.04.250.240.325.106230513313957e+35GDGPTQ (8-Bit) 0.00.00.08.250.240.352.445586872346403e+26GDAWQ (2-Bit) 0.00.00.02.250.240.272598854.5GDAWQ (3-Bit) 0.00.00.03.250.330.33infGDAWQ (4-Bit) 0.00.00.04.250.240.352.0452847856930923e+28GDAWQ (5-Bit) 0.00.00.05.250.250.361.0052004503012292e+28GDAWQ (6-Bit) 0.00.00.06.250.250.356.499345951176105e+25GDAWQ (8-Bit) 0.00.00.08.250.250.359.430016153317283e+25\cdashline1-8

RMUSparseGPT (0.25) 0.020.020.0312.250.280.565.93RMUSparseGPT (0.35) 0.020.020.0310.750.280.536.39RMUSparseGPT (0.45) 0.020.020.039.250.270.488.14RMUSparseGPT (0.55) 0.010.010.037.750.260.3910.58RMUSparseGPT (0.65) 0.00.010.036.250.250.2635.97RMUSparseGPT (0.75) 0.010.010.034.750.250.23101.32RMUWanda (0.25) 0.020.020.0412.250.290.565.88RMUWanda (0.35) 0.020.020.0310.750.280.536.36RMUWanda (0.45) 0.020.010.039.250.290.487.71RMUWanda (0.55) 0.010.010.037.750.270.3513.73RMUWanda (0.65) 0.010.010.036.250.250.2352.67RMUWanda (0.75) 0.030.020.034.750.250.23334.45RMUGPTQ (2-Bit) 0.00.010.032.250.250.243451.67RMUGPTQ (3-Bit) 0.010.010.033.250.260.429.09RMUGPTQ (4-Bit) 0.020.020.034.250.270.5312.88RMUGPTQ (8-Bit) 0.020.020.038.250.290.575.56RMUAWQ (2-Bit) 0.00.00.02.250.240.261726409.12RMUAWQ (3-Bit) 0.010.010.033.250.270.457.55RMUAWQ (4-Bit) 0.020.020.034.250.290.565.95RMUAWQ (5-Bit) 0.020.030.045.250.280.565.68RMUAWQ (6-Bit) 0.020.020.036.250.280.565.62RMUAWQ (8-Bit) 0.020.020.038.250.290.575.56Table 11:Detailed results for unlearningcompression.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLSparseGPT (0.25) GA0.00.00.012.250.430.45infSparseGPT (0.35) GA0.00.00.010.750.340.36infSparseGPT (0.45) GA0.00.00.09.250.310.33infSparseGPT (0.55) GA0.00.00.07.750.250.25infSparseGPT (0.65) GA0.00.00.06.250.270.28infSparseGPT (0.75) GA0.010.010.034.750.260.23infSparseGPT (0.25) GD0.010.00.0212.250.50.4813.32SparseGPT (0.35) GD0.010.010.0110.750.290.252712.26SparseGPT (0.45) GD0.00.00.09.250.250.481.246813090597988e+19SparseGPT (0.55) GD0.00.00.027.750.350.369.35SparseGPT (0.65) GD0.00.00.016.250.240.26infSparseGPT (0.75) GD0.00.010.014.750.260.231437619072.0SparseGPT (0.25) RMU0.010.020.0412.250.310.575.92SparseGPT (0.35) RMU0.00.020.0210.750.310.546.35SparseGPT (0.45) RMU0.010.010.039.250.310.517.86SparseGPT (0.55) RMU0.010.010.037.750.290.4110.39SparseGPT (0.65) RMU0.00.00.026.250.260.2430.26SparseGPT (0.75) RMU0.010.010.024.750.250.2397.03\cdashline1-8

Wanda (0.25) GA0.00.00.012.250.490.54infWanda (0.35) GA0.00.00.010.750.440.51infWanda (0.45) GA0.00.00.09.250.410.43infWanda (0.55) GA0.00.00.07.750.320.34infWanda (0.65) GA0.00.00.06.250.260.26infWanda (0.75) GA0.010.010.034.750.260.238.12597183809491e+36Wanda (0.25) GD0.00.00.012.250.520.5713.26Wanda (0.35) GD0.00.00.010.750.450.4857480503296.0Wanda (0.45) GD0.00.00.09.250.50.5349.77Wanda (0.55) GD0.00.00.07.750.390.35121.07Wanda (0.65) GD0.00.00.06.250.260.2316268.01Wanda (0.75) GD0.00.00.04.750.250.2510226426.0Wanda (0.25) RMU0.020.020.0412.250.320.575.86Wanda (0.35) RMU0.020.020.0310.750.340.566.33Wanda (0.45) RMU0.020.020.039.250.390.527.55Wanda (0.55) RMU0.010.010.037.750.360.3612.58Wanda (0.65) RMU0.010.010.026.250.260.2345.53Wanda (0.75) RMU0.020.030.024.750.260.23391.87\cdashline1-8

GPTQ (2-Bit) GA0.00.00.02.250.250.25infGPTQ (3-Bit) GA0.00.00.03.250.240.27infGPTQ (4-Bit) GA0.00.00.04.250.340.38infGPTQ (8-Bit) GA0.00.00.08.250.450.5infGPTQ (2-Bit) GD0.00.00.02.250.250.25infGPTQ (3-Bit) GD0.00.00.03.250.260.24628.24GPTQ (4-Bit) GD0.010.010.014.250.270.569.38GPTQ (8-Bit) GD0.010.020.058.250.260.47infGPTQ (2-Bit) RMU0.00.00.02.250.250.241017825.94GPTQ (3-Bit) RMU0.010.010.023.250.320.3211.1GPTQ (4-Bit) RMU0.00.020.024.250.450.586.35GPTQ (8-Bit) RMU0.010.020.038.250.280.575.58\cdashline1-8

AWQ (2-Bit) GA0.00.00.02.250.240.27infAWQ (3-Bit) GA0.00.00.03.250.240.27infAWQ (4-Bit) GA0.00.00.04.250.450.45infAWQ (5-Bit) GA0.00.00.05.250.350.39infAWQ (6-Bit) GA0.00.00.06.250.250.3infAWQ (8-Bit) GA0.00.00.08.250.280.37infAWQ (2-Bit) GD0.00.00.022.250.240.274.496028624899119e+33AWQ (3-Bit) GD0.010.00.023.250.290.34166895.91AWQ (4-Bit) GD0.010.00.044.250.240.427.05AWQ (5-Bit) GD0.00.00.035.250.250.382685384.0AWQ (6-Bit) GD0.00.00.056.250.250.3127025.07AWQ (8-Bit) GD0.00.00.018.250.250.2549224794112.0AWQ (2-Bit) RMU0.00.00.02.250.240.271749321.75AWQ (3-Bit) RMU0.010.010.033.250.270.467.51AWQ (4-Bit) RMU0.020.020.034.250.270.556.06AWQ (5-Bit) RMU0.020.030.045.250.270.565.68AWQ (6-Bit) RMU0.020.020.046.250.280.565.6AWQ (8-Bit) RMU0.020.030.038.250.280.575.71Table 12:Detailed results for compressionunlearning.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLFTGA0.00.00.016.00.470.53infFTGD0.320.250.0716.00.290.412.8236466916108585e+31FTRMU0.990.820.116.00.280.565.61\cdashline1-8

MEMITGA0.00.00.016.00.40.45infMEMITGD0.530.490.0316.00.260.3627645056122880.0MEMITRMU0.960.890.0316.00.290.565.58\cdashline1-8

LoRAGA0.00.00.016.00.280.29infLoRAGD0.440.230.0516.00.30.4529.46LoRARMU1.00.680.0516.00.30.5234.56Table 13:Detailed results for editingunlearning.EditingCompressionUnlearningUtilityEdit SuccessGeneralizationLocalityAvg. BitsAvg. WMDPMMLUWikiText PPLGAFT0.070.040.016.00.470.52infGAMEMIT0.480.410.016.00.450.49infGALoRA1.00.780.0316.00.340.3656.91\cdashline1-8

GDFT0.990.810.1116.00.290.594.64GDMEMIT0.930.890.0516.00.280.584.65GDLoRA1.00.710.0816.00.540.594.99\cdashline1-8

RMUFT1.00.790.1316.00.320.575.6RMUMEMIT0.970.930.0316.00.30.565.58RMULoRA1.00.710.0516.00.290.5612.83Table 14:Detailed results for unlearningediting.
conclusion
Composable Interventions for Language Models
Acknowledgements