text: <html>
          <head> <meta charset="UTF-8">
          <style>
          table, th, td {
              border: 1px solid black;
              font-size: 10px;
          }
          </style> </head>
          <body>
          <table frame="hsides" rules="groups" width="100%%">
              [html]<thead><tr><td></td><td></td><td>Compression</td><td>Performance</td><td>Speedup</td><td>Model</td><td>Evaluation</td></tr></thead><tbody><tr><td></td><td>BERT - base ( Devlin et al., 2019 )</td><td>0</td><td>100 %</td><td>× 1</td><td>BERT12</td><td>All GLUE tasks, SQuAD</td></tr><tr><td></td><td>BERT - small</td><td>× 3.8</td><td>91 %</td><td>-</td><td>BERT4 †</td><td>All GLUE tasks</td></tr><tr><td></td><td>DistilBERT ( Sanh et al., 2019a )</td><td>× 1.5</td><td>90 %§</td><td>× 1.6</td><td>BERT6</td><td>All GLUE tasks, SQuAD</td></tr><tr><td></td><td>BERT6 - PKD ( Sun et al., 2019a )</td><td>× 1.6</td><td>98 %</td><td>× 1.9</td><td>BERT6</td><td>No WNLI, CoLA, STS - B ; RACE</td></tr><tr><td></td><td>BERT3 - PKD ( Sun et al., 2019a )</td><td>× 2.4</td><td>92 %</td><td>× 3.7</td><td>BERT3</td><td>No WNLI, CoLA, STS - B ; RACE</td></tr><tr><td></td><td>Aguilar et al. ( 2019 ), Exp. 3</td><td>× 1.6</td><td>93 %</td><td>-</td><td>BERT6</td><td>CoLA, MRPC, QQP, RTE</td></tr><tr><td></td><td>BERT - 48 ( Zhao et al., 2019 )</td><td>× 62</td><td>87 %</td><td>× 77</td><td>BERT12 *†</td><td>MNLI, MRPC, SST - 2</td></tr><tr><td></td><td>BERT - 192 ( Zhao et al., 2019 )</td><td>× 5.7</td><td>93 %</td><td>× 22</td><td>BERT12 ∗†</td><td>MNLI, MRPC, SST - 2</td></tr><tr><td></td><td>TinyBERT ( Jiao et al., 2019 )</td><td>× 7.5</td><td>96 %</td><td>× 9.4</td><td>BERT4 †</td><td>No WNLI ; SQuAD</td></tr><tr><td></td><td>MobileBERT ( Sun et al., 2020 )</td><td>× 4.3</td><td>100 %</td><td>× 4</td><td>BERT24 †</td><td>No WNLI ; SQ0AD</td></tr><tr><td></td><td>PD ( Turc et al., 2019 )</td><td>× 1.6</td><td>98 %</td><td>× 2.5 ‡</td><td>BERT6b</td><td>No WNLI, CoLA and STS - B</td></tr><tr><td></td><td>WalLDORf ( Tian et al., 2019 )</td><td>× 4.4</td><td>93 %</td><td>× 9</td><td>BERT8 †||</td><td>SQuAD</td></tr><tr><td></td><td>MiniLM ( Wang et al., 2020b )</td><td>× 1.65</td><td>99 %</td><td>× 2</td><td>BERT6</td><td>No WNLI, STS - B, MNLImm ; SQuAD</td></tr><tr><td></td><td>MiniBERT ( Tsai et al., 2019 )</td><td>× 6 **</td><td>98 %</td><td>× 27 **</td><td>mBERT3 †</td><td>CoNLL - 18 POS and morphology</td></tr><tr><td></td><td>BiLSTM - soft ( Tang et al., 2019 )</td><td>≥ 110</td><td>91 %</td><td>× 434 ‡</td><td>BiLSTM1</td><td>MNLI, QQP, SST - 2</td></tr><tr><td></td><td>Q - BERT - MP ( Shen et al., 2019 )</td><td>× 13</td><td>98 %¶</td><td>-</td><td>BERT12</td><td>MNLI, SST - 2, CoNLL - 03, SQuAD</td></tr><tr><td></td><td>BERT - QAT ( Zafrir et al., 2019 )</td><td>× 4</td><td>99 %</td><td>-</td><td>BERT12</td><td>No WNLI, MNLI ; SQuAD</td></tr><tr><td></td><td>GOBO ( Zadeh and Moshovos, 2020 )</td><td>× 9.8</td><td>99 %</td><td>-</td><td>BERT12</td><td>MNLI</td></tr><tr><td></td><td>McCarley et al. ( 2020 ), ff2</td><td>× 2.2 ‡</td><td>98 %‡</td><td>× 1.9 ‡</td><td>BERT24</td><td>SQuAD, Natural Questions</td></tr><tr><td></td><td>RPP ( Guo et al., 2019 )</td><td>× 1.7 ‡</td><td>99 %‡</td><td>-</td><td>BERT24</td><td>No WNLI, STS - B ; SQHAD</td></tr><tr><td></td><td>Soft MvP ( Sanh et al., 2020 )</td><td>× 33</td><td>94 %¶</td><td>-</td><td>BERT12</td><td>MNLI, QQP, SQuAD</td></tr><tr><td></td><td>IMP ( Chen et al., 2020 ), rewind 50 %</td><td>× 1.4 – 2.5</td><td>94 – 100 %</td><td>-</td><td>BERT12</td><td>No MNLI - mm ; SQuAD</td></tr><tr><td></td><td>ALBERT - base ( Lan et al., 2020b )</td><td>× 9</td><td>97 %</td><td>-</td><td>BERT12 †</td><td>MNLI, SST - 2</td></tr><tr><td></td><td>ALBERT - xxlarge ( Lan et al., 2020b )</td><td>× 0.47</td><td>107 %</td><td>-</td><td>BERT12 †</td><td>MNLI, SST - 2</td></tr><tr><td></td><td>BERT - of - Theseus ( Xu et al., 2020 )</td><td>× 1.6</td><td>98 %</td><td>× 1.9</td><td>BERT6</td><td>No WNLI</td></tr><tr><td></td><td>PoWER - BERT ( Goyal et al., 2020 )</td><td>N / A</td><td>99 %</td><td>× 2 – 4.5</td><td>BERT12</td><td>No WNLI ; RACE</td></tr></tbody>
          </table> </body> </html>