| { | |
| "STSBenchmark": { | |
| "train": { | |
| "pearson": [ | |
| 0.700217766857111, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.694596243514894, | |
| 0.0 | |
| ], | |
| "nsamples": 5749 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7188725565702827, | |
| 7.642460243707613e-239 | |
| ], | |
| "spearman": [ | |
| 0.7430861317995934, | |
| 1.3229042265692801e-263 | |
| ], | |
| "nsamples": 1500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.6697149917661123, | |
| 3.5510311886663796e-180 | |
| ], | |
| "spearman": [ | |
| 0.7028355722100523, | |
| 6.396511507534624e-206 | |
| ], | |
| "nsamples": 1379 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7003515412165919, | |
| "mean": 0.6962684383978353, | |
| "wmean": 0.698585738312752 | |
| }, | |
| "spearman": { | |
| "all": 0.708944371429387, | |
| "mean": 0.7135059825081799, | |
| "wmean": 0.7043432146203266 | |
| } | |
| } | |
| }, | |
| "SICKRelatedness": { | |
| "train": { | |
| "pearson": [ | |
| 0.7900787687078513, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7162798220982042, | |
| 0.0 | |
| ], | |
| "nsamples": 4500 | |
| }, | |
| "dev": { | |
| "pearson": [ | |
| 0.7941456305052318, | |
| 8.704175570070149e-110 | |
| ], | |
| "spearman": [ | |
| 0.7323591663810709, | |
| 3.70700656086251e-85 | |
| ], | |
| "nsamples": 500 | |
| }, | |
| "test": { | |
| "pearson": [ | |
| 0.7827901659487767, | |
| 0.0 | |
| ], | |
| "spearman": [ | |
| 0.7066436772359563, | |
| 0.0 | |
| ], | |
| "nsamples": 4927 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7866207722959027, | |
| "mean": 0.7890048550539532, | |
| "wmean": 0.7866661047715896 | |
| }, | |
| "spearman": { | |
| "all": 0.7122486034563766, | |
| "mean": 0.7184275552384105, | |
| "wmean": 0.7123070595722788 | |
| } | |
| } | |
| }, | |
| "MR": { | |
| "devacc": 80.75, | |
| "acc": 80.52, | |
| "ndev": 10662, | |
| "ntest": 10662 | |
| }, | |
| "CR": { | |
| "devacc": 87.02, | |
| "acc": 86.41, | |
| "ndev": 3775, | |
| "ntest": 3775 | |
| }, | |
| "SUBJ": { | |
| "devacc": 95.02, | |
| "acc": 94.49, | |
| "ndev": 10000, | |
| "ntest": 10000 | |
| }, | |
| "MPQA": { | |
| "devacc": 88.67, | |
| "acc": 88.53, | |
| "ndev": 10606, | |
| "ntest": 10606 | |
| }, | |
| "SST2": { | |
| "devacc": 82.8, | |
| "acc": 85.89, | |
| "ndev": 872, | |
| "ntest": 1821 | |
| }, | |
| "TREC": { | |
| "devacc": 78.63, | |
| "acc": 88.6, | |
| "ndev": 5452, | |
| "ntest": 500 | |
| }, | |
| "MRPC": { | |
| "devacc": 70.17, | |
| "acc": 62.55, | |
| "f1": 68.4, | |
| "ndev": 4076, | |
| "ntest": 1725 | |
| }, | |
| "STS12": { | |
| "MSRpar": { | |
| "pearson": [ | |
| 0.3279591431665635, | |
| 2.8934198828902823e-20 | |
| ], | |
| "spearman": [ | |
| 0.37151182527291027, | |
| 5.848988168730171e-26 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "MSRvid": { | |
| "pearson": [ | |
| 0.8600914351071427, | |
| 7.615361092408985e-221 | |
| ], | |
| "spearman": [ | |
| 0.8710479889820992, | |
| 3.841468226701095e-233 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "SMTeuroparl": { | |
| "pearson": [ | |
| 0.49106075103459146, | |
| 3.1227262696721795e-29 | |
| ], | |
| "spearman": [ | |
| 0.5976234583132141, | |
| 8.832799021137483e-46 | |
| ], | |
| "nsamples": 459 | |
| }, | |
| "surprise.OnWN": { | |
| "pearson": [ | |
| 0.6903453132676263, | |
| 2.993644573897565e-107 | |
| ], | |
| "spearman": [ | |
| 0.6522049690471122, | |
| 4.576466079694001e-92 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "surprise.SMTnews": { | |
| "pearson": [ | |
| 0.5747570138816134, | |
| 1.8705693938405458e-36 | |
| ], | |
| "spearman": [ | |
| 0.6161895683302498, | |
| 4.402482065232814e-43 | |
| ], | |
| "nsamples": 399 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6532360047609772, | |
| "mean": 0.5888427312915075, | |
| "wmean": 0.5995887554438998 | |
| }, | |
| "spearman": { | |
| "all": 0.6520942930121344, | |
| "mean": 0.6217155619891171, | |
| "wmean": 0.6245953644163854 | |
| } | |
| } | |
| }, | |
| "STS13": { | |
| "FNWN": { | |
| "pearson": [ | |
| 0.5455691661756268, | |
| 4.746052411411139e-16 | |
| ], | |
| "spearman": [ | |
| 0.5552285672079286, | |
| 1.1207604310221007e-16 | |
| ], | |
| "nsamples": 189 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7376364893975698, | |
| 1.0257488464328715e-129 | |
| ], | |
| "spearman": [ | |
| 0.7584758247313932, | |
| 3.1099800660504727e-141 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8062599488981587, | |
| 1.4758922804276925e-129 | |
| ], | |
| "spearman": [ | |
| 0.791620979725446, | |
| 1.0741113505197788e-121 | |
| ], | |
| "nsamples": 561 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7161952323138701, | |
| "mean": 0.6964885348237851, | |
| "wmean": 0.7391011805248252 | |
| }, | |
| "spearman": { | |
| "all": 0.7489808796663362, | |
| "mean": 0.701775123888256, | |
| "wmean": 0.7452629582512124 | |
| } | |
| } | |
| }, | |
| "STS14": { | |
| "deft-forum": { | |
| "pearson": [ | |
| 0.4539905158613596, | |
| 2.8973755391032382e-24 | |
| ], | |
| "spearman": [ | |
| 0.4539149626420418, | |
| 2.954477747813041e-24 | |
| ], | |
| "nsamples": 450 | |
| }, | |
| "deft-news": { | |
| "pearson": [ | |
| 0.6909565157190697, | |
| 6.731301106701071e-44 | |
| ], | |
| "spearman": [ | |
| 0.654918374815774, | |
| 3.9304042018460903e-38 | |
| ], | |
| "nsamples": 300 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7163249206179311, | |
| 5.064372905060631e-119 | |
| ], | |
| "spearman": [ | |
| 0.6890244755927888, | |
| 1.1000588945875164e-106 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.8341976013585205, | |
| 1.581737521468821e-195 | |
| ], | |
| "spearman": [ | |
| 0.8009087778380325, | |
| 9.127029827171149e-169 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "OnWN": { | |
| "pearson": [ | |
| 0.8416372752913062, | |
| 2.488984870540694e-202 | |
| ], | |
| "spearman": [ | |
| 0.835891574935076, | |
| 4.7883053518051054e-197 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "tweet-news": { | |
| "pearson": [ | |
| 0.5893351545919092, | |
| 2.4679056644216343e-71 | |
| ], | |
| "spearman": [ | |
| 0.5523429250762462, | |
| 4.020802928899476e-61 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.6458742493245161, | |
| "mean": 0.6877403305733494, | |
| "wmean": 0.7060543735328222 | |
| }, | |
| "spearman": { | |
| "all": 0.6214541243526541, | |
| "mean": 0.6645001818166599, | |
| "wmean": 0.6824968161907357 | |
| } | |
| } | |
| }, | |
| "STS15": { | |
| "answers-forums": { | |
| "pearson": [ | |
| 0.5843871869461419, | |
| 1.0151032193286744e-35 | |
| ], | |
| "spearman": [ | |
| 0.6084659043667563, | |
| 2.3820444508668996e-39 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "answers-students": { | |
| "pearson": [ | |
| 0.6152132265657764, | |
| 2.6617778013870566e-79 | |
| ], | |
| "spearman": [ | |
| 0.6302561398409405, | |
| 2.773969605891675e-84 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "belief": { | |
| "pearson": [ | |
| 0.6392710473707196, | |
| 1.799179528686185e-44 | |
| ], | |
| "spearman": [ | |
| 0.686875976718544, | |
| 1.2015551469859737e-53 | |
| ], | |
| "nsamples": 375 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7734487150038147, | |
| 2.927357789667028e-150 | |
| ], | |
| "spearman": [ | |
| 0.7896018531226366, | |
| 8.23498473346381e-161 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "images": { | |
| "pearson": [ | |
| 0.864344989852659, | |
| 1.7210495814828955e-225 | |
| ], | |
| "spearman": [ | |
| 0.8824596984841369, | |
| 3.2828816211446655e-247 | |
| ], | |
| "nsamples": 750 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7203379074291151, | |
| "mean": 0.6953330331478222, | |
| "wmean": 0.7162090121451702 | |
| }, | |
| "spearman": { | |
| "all": 0.7497940183329284, | |
| "mean": 0.7195319145066028, | |
| "wmean": 0.7374971579975911 | |
| } | |
| } | |
| }, | |
| "STS16": { | |
| "answer-answer": { | |
| "pearson": [ | |
| 0.6368696070010881, | |
| 2.678858858103159e-30 | |
| ], | |
| "spearman": [ | |
| 0.6921912704278468, | |
| 1.4681538672067537e-37 | |
| ], | |
| "nsamples": 254 | |
| }, | |
| "headlines": { | |
| "pearson": [ | |
| 0.7289776103151235, | |
| 1.5277023512508603e-42 | |
| ], | |
| "spearman": [ | |
| 0.7621038994295888, | |
| 1.551008215103711e-48 | |
| ], | |
| "nsamples": 249 | |
| }, | |
| "plagiarism": { | |
| "pearson": [ | |
| 0.7702063765670754, | |
| 2.003814813962993e-46 | |
| ], | |
| "spearman": [ | |
| 0.7911510680474892, | |
| 1.3837761303567582e-50 | |
| ], | |
| "nsamples": 230 | |
| }, | |
| "postediting": { | |
| "pearson": [ | |
| 0.719837823485291, | |
| 3.0286294416646686e-40 | |
| ], | |
| "spearman": [ | |
| 0.7566461459669797, | |
| 1.4825051882045699e-46 | |
| ], | |
| "nsamples": 244 | |
| }, | |
| "question-question": { | |
| "pearson": [ | |
| 0.7625392869659763, | |
| 5.114455019237254e-41 | |
| ], | |
| "spearman": [ | |
| 0.7800407128371691, | |
| 5.033994633121257e-44 | |
| ], | |
| "nsamples": 209 | |
| }, | |
| "all": { | |
| "pearson": { | |
| "all": 0.7140924962133864, | |
| "mean": 0.7236861408669109, | |
| "wmean": 0.721280701233954 | |
| }, | |
| "spearman": { | |
| "all": 0.7490972681198437, | |
| "mean": 0.7564266193418148, | |
| "wmean": 0.7548021651740933 | |
| } | |
| } | |
| }, | |
| "eval_senteval-STS12": 0.6520942930121344, | |
| "eval_senteval-STS13": 0.7489808796663362, | |
| "eval_senteval-STS14": 0.6214541243526541, | |
| "eval_senteval-STS15": 0.7497940183329284, | |
| "eval_senteval-STS16": 0.7490972681198437, | |
| "eval_senteval-STSBenchmark": 0.708944371429387, | |
| "eval_senteval-SICKRelatedness": 0.7122486034563766, | |
| "eval_senteval-avg_sts_7": 0.7060876511956659, | |
| "eval_senteval-MR": 80.75, | |
| "eval_senteval-CR": 87.02, | |
| "eval_senteval-SUBJ": 95.02, | |
| "eval_senteval-MPQA": 88.67, | |
| "eval_senteval-SST2": 82.8, | |
| "eval_senteval-TREC": 78.63, | |
| "eval_senteval-MRPC": 70.17, | |
| "eval_senteval-avg_transfer": 83.2942857142857 | |
| } | |