1351 lines
39 KiB
JSON
1351 lines
39 KiB
JSON
{
|
|
"legacy": {
|
|
"distance": {
|
|
"base": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 5.7840811612328125,
|
|
"median": 8.93182211792415,
|
|
"p90": 11.67265834734814,
|
|
"mean": 8.821002725510846
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.3615050725770508,
|
|
"median": 0.5582388823702594,
|
|
"p90": 0.7295411467092587,
|
|
"mean": 0.5513126703444279
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.02471497387658298,
|
|
"median": 0.05275652115976448,
|
|
"p90": 0.11774752758475547,
|
|
"mean": 0.06432512729180001
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.1457405612874147,
|
|
"median": 0.21746353579124,
|
|
"p90": 0.34538740233748155,
|
|
"mean": 0.23503983543721835
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.46438300527426274,
|
|
"median": 0.5649686672386036,
|
|
"p90": 0.6741052273899216,
|
|
"mean": 0.5682829621251091
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.04389082141962781,
|
|
"median": 0.06803373425718254,
|
|
"p90": 0.10318453074095452,
|
|
"mean": 0.07107512763171646
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 10.0,
|
|
"median": 11.0,
|
|
"p90": 13.0,
|
|
"mean": 11.273125
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.22150190912652762,
|
|
"median": 0.4924028478562832,
|
|
"p90": 0.7786339074373245,
|
|
"mean": 0.4935038255766646
|
|
}
|
|
},
|
|
"finetuned": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 5.776575675851443,
|
|
"median": 8.918300551653985,
|
|
"p90": 11.683939551627272,
|
|
"mean": 8.807297020017815
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.36103597974071516,
|
|
"median": 0.5573937844783741,
|
|
"p90": 0.7302462219767045,
|
|
"mean": 0.5504560637511134
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.024626743482520943,
|
|
"median": 0.05293758540105196,
|
|
"p90": 0.11798713654455906,
|
|
"mean": 0.06466812007727665
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.14612856743114083,
|
|
"median": 0.21806944835017916,
|
|
"p90": 0.3463747132982079,
|
|
"mean": 0.23575051686039453
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.46398691517338536,
|
|
"median": 0.5656402664666489,
|
|
"p90": 0.6751455273905578,
|
|
"mean": 0.5687308911331159
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.04375078049578534,
|
|
"median": 0.06814265463530741,
|
|
"p90": 0.10351322156021461,
|
|
"mean": 0.07115813787142872
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 10.0,
|
|
"median": 11.0,
|
|
"p90": 13.0,
|
|
"mean": 11.275
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.22458011865383015,
|
|
"median": 0.4925860376097262,
|
|
"p90": 0.7803986554034055,
|
|
"mean": 0.49494720498192085
|
|
}
|
|
},
|
|
"paired": {
|
|
"effective_heads_delta": {
|
|
"count": 4800,
|
|
"p10": -0.12155525346733853,
|
|
"median": -0.007827372994309911,
|
|
"p90": 0.08552963831211874,
|
|
"mean": -0.013705705493032733
|
|
},
|
|
"increased_fraction": 0.45625,
|
|
"decreased_fraction": 0.54375,
|
|
"winner_agreement": 0.981875
|
|
},
|
|
"per_text": {
|
|
"fresh-zh-1": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 5.724006719505733,
|
|
"median": 8.447270312666955,
|
|
"p90": 11.400422886121929,
|
|
"mean": 8.52580403089048
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 5.692180523473624,
|
|
"median": 8.439940185642019,
|
|
"p90": 11.383613339828717,
|
|
"mean": 8.519406557532788
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.09676644962819134,
|
|
"median": -0.0003564206512196577,
|
|
"p90": 0.07296984397018705,
|
|
"mean": -0.0063974733576900285
|
|
}
|
|
},
|
|
"fresh-zh-2": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 6.0045436576758515,
|
|
"median": 9.280419095307465,
|
|
"p90": 11.738746210854123,
|
|
"mean": 9.057065620159456
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 5.968510727111234,
|
|
"median": 9.252540842866502,
|
|
"p90": 11.720895166646851,
|
|
"mean": 9.043651292717824
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.122961714917267,
|
|
"median": -0.012195150149180378,
|
|
"p90": 0.09760457550073502,
|
|
"mean": -0.01341432744163121
|
|
}
|
|
},
|
|
"fresh-en-1": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 5.729701873062357,
|
|
"median": 8.743397719211597,
|
|
"p90": 11.683026866465143,
|
|
"mean": 8.773217379604489
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 5.7281595085239125,
|
|
"median": 8.723519639693787,
|
|
"p90": 11.708927676002391,
|
|
"mean": 8.7605887879079
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.11081218619648876,
|
|
"median": -0.007005952507822322,
|
|
"p90": 0.07766016926945382,
|
|
"mean": -0.012628591696590086
|
|
}
|
|
},
|
|
"fresh-en-2": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 5.73972334577513,
|
|
"median": 9.101710336434357,
|
|
"p90": 11.821849799673606,
|
|
"mean": 8.927923871388966
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 5.750476017284456,
|
|
"median": 9.082570331629224,
|
|
"p90": 11.853544794055829,
|
|
"mean": 8.905541441912746
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.16178742240734909,
|
|
"median": -0.013896515617046745,
|
|
"p90": 0.1005384062632225,
|
|
"mean": -0.022382429476219604
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"divergence": {
|
|
"base": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 1.8775771328772404,
|
|
"median": 3.901782998659897,
|
|
"p90": 6.9489972185767925,
|
|
"mean": 4.1958991748153185
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.11734857080482752,
|
|
"median": 0.24386143741624355,
|
|
"p90": 0.43431232616104953,
|
|
"mean": 0.2622436984259574
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.08683274490451195,
|
|
"median": 0.20671262373235755,
|
|
"p90": 0.5014413860763198,
|
|
"mean": 0.25655510469396303
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.24280447987972617,
|
|
"median": 0.4230606060703387,
|
|
"p90": 0.7145513508216859,
|
|
"mean": 0.45466951196884586
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.6646066107988152,
|
|
"median": 0.8229818456850793,
|
|
"p90": 0.9326550608746287,
|
|
"mean": 0.808919570864033
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.0034597576965423498,
|
|
"median": 0.01107327468889098,
|
|
"p90": 0.03202965476590303,
|
|
"mean": 0.014963302718693967
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 3.0,
|
|
"median": 6.0,
|
|
"p90": 9.0,
|
|
"mean": 6.310208333333334
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.006019792613687969,
|
|
"median": 0.026595803305099253,
|
|
"p90": 0.07109484240791066,
|
|
"mean": 0.03422935252324174
|
|
}
|
|
},
|
|
"finetuned": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 1.876908117382701,
|
|
"median": 3.871619853837708,
|
|
"p90": 6.96989498987909,
|
|
"mean": 4.184291975347008
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.11730675733641881,
|
|
"median": 0.24197624086485675,
|
|
"p90": 0.4356184368674431,
|
|
"mean": 0.261518248459188
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.08637246380916241,
|
|
"median": 0.20884247610187728,
|
|
"p90": 0.5016439635373461,
|
|
"mean": 0.25814451030972974
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.2422939636733382,
|
|
"median": 0.42624464148928093,
|
|
"p90": 0.7155069102079873,
|
|
"mean": 0.45622446760844526
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.665414679548231,
|
|
"median": 0.8240052643141178,
|
|
"p90": 0.9339388132525362,
|
|
"mean": 0.8095483126047844
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.003446853632797573,
|
|
"median": 0.011054384074575818,
|
|
"p90": 0.03206920533547495,
|
|
"mean": 0.014988975788690944
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 3.0,
|
|
"median": 6.0,
|
|
"p90": 9.0,
|
|
"mean": 6.302083333333333
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.006202359683584291,
|
|
"median": 0.026746092644316377,
|
|
"p90": 0.07105643208897167,
|
|
"mean": 0.03444768158607483
|
|
}
|
|
},
|
|
"paired": {
|
|
"effective_heads_delta": {
|
|
"count": 4800,
|
|
"p10": -0.12236077231273206,
|
|
"median": -0.006747826895777909,
|
|
"p90": 0.09207081416677103,
|
|
"mean": -0.011607199468310615
|
|
},
|
|
"increased_fraction": 0.44125,
|
|
"decreased_fraction": 0.55875,
|
|
"winner_agreement": 0.981875
|
|
},
|
|
"per_text": {
|
|
"fresh-zh-1": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 1.86102042169738,
|
|
"median": 3.674821925930784,
|
|
"p90": 6.496464962062872,
|
|
"mean": 3.9738265052500115
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 1.8695193592812367,
|
|
"median": 3.653253784859141,
|
|
"p90": 6.540801874780079,
|
|
"mean": 3.96916619071483
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.0904291820005377,
|
|
"median": -0.0021758020710173076,
|
|
"p90": 0.07911341590862223,
|
|
"mean": -0.004660314535179818
|
|
}
|
|
},
|
|
"fresh-zh-2": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 2.020523053994744,
|
|
"median": 4.232903043923887,
|
|
"p90": 7.155352410694885,
|
|
"mean": 4.422540511588559
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 1.989322683403461,
|
|
"median": 4.1946239719499125,
|
|
"p90": 7.156199883412831,
|
|
"mean": 4.412144595717297
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.12116194167201452,
|
|
"median": -0.010746777226108173,
|
|
"p90": 0.10483640341883402,
|
|
"mean": -0.010395915871261636
|
|
}
|
|
},
|
|
"fresh-en-1": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 1.8198022208743365,
|
|
"median": 3.866727985636466,
|
|
"p90": 7.147920033260447,
|
|
"mean": 4.154808100201001
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 1.8146582042199053,
|
|
"median": 3.84232558313389,
|
|
"p90": 7.11390324618346,
|
|
"mean": 4.146511070253586
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.10807865550731473,
|
|
"median": -0.0034545124285838558,
|
|
"p90": 0.08858334435120963,
|
|
"mean": -0.008297029947415074
|
|
}
|
|
},
|
|
"fresh-en-2": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 1.8322151674116371,
|
|
"median": 3.8933109706193667,
|
|
"p90": 7.112082581969417,
|
|
"mean": 4.2324215822217015
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 1.8021689266802543,
|
|
"median": 3.8609554823402976,
|
|
"p90": 7.099615165719617,
|
|
"mean": 4.209346044702316
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.16516077036721016,
|
|
"median": -0.014694033457486322,
|
|
"p90": 0.10418054058914197,
|
|
"mean": -0.02307553751938593
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"QKV": {
|
|
"base": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 7.475883895098561,
|
|
"median": 10.319912213892923,
|
|
"p90": 12.712833271824215,
|
|
"mean": 10.202852380581907
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.46724274344366007,
|
|
"median": 0.6449945133683077,
|
|
"p90": 0.7945520794890134,
|
|
"mean": 0.6376782737863692
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.01723804948414177,
|
|
"median": 0.03669338569017259,
|
|
"p90": 0.07601434362115887,
|
|
"mean": 0.042570735285422205
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.13321341470482867,
|
|
"median": 0.18747321888747073,
|
|
"p90": 0.28328865301359296,
|
|
"mean": 0.19983216483858976
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.42474317220679464,
|
|
"median": 0.510360546343396,
|
|
"p90": 0.5986294325410266,
|
|
"mean": 0.5111861898360973
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.04393379865513463,
|
|
"median": 0.08040363020124161,
|
|
"p90": 0.12289210604930895,
|
|
"mean": 0.08222739743141799
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 10.0,
|
|
"median": 12.0,
|
|
"p90": 13.0,
|
|
"mean": 11.84
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.3599545699544251,
|
|
"median": 0.8310977094806731,
|
|
"p90": 1.3057362334919163,
|
|
"mean": 0.832024075460137
|
|
}
|
|
},
|
|
"finetuned": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 7.415670339257133,
|
|
"median": 10.324447147382044,
|
|
"p90": 12.701222754575259,
|
|
"mean": 10.189892766418488
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.4634793962035708,
|
|
"median": 0.6452779467113777,
|
|
"p90": 0.7938264221609537,
|
|
"mean": 0.6368682979011555
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.017314748942665027,
|
|
"median": 0.03664798557992152,
|
|
"p90": 0.07717288111557072,
|
|
"mean": 0.04288492534802562
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.13294356494233925,
|
|
"median": 0.18756509230157237,
|
|
"p90": 0.2852351774363626,
|
|
"mean": 0.20058383379684908
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.4247718681503831,
|
|
"median": 0.5097655989756011,
|
|
"p90": 0.5995470748644118,
|
|
"mean": 0.5113628895373682
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.04420502297887021,
|
|
"median": 0.0806159855691112,
|
|
"p90": 0.12315113314407859,
|
|
"mean": 0.08238857534024695
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 10.0,
|
|
"median": 12.0,
|
|
"p90": 13.0,
|
|
"mean": 11.84875
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.36229494377039373,
|
|
"median": 0.8324168699327856,
|
|
"p90": 1.3076765275676736,
|
|
"mean": 0.8336380656609678
|
|
}
|
|
},
|
|
"paired": {
|
|
"effective_heads_delta": {
|
|
"count": 4800,
|
|
"p10": -0.09236974599448597,
|
|
"median": -0.004347649130346021,
|
|
"p90": 0.05748593739061878,
|
|
"mean": -0.012959614163417956
|
|
},
|
|
"increased_fraction": 0.45875,
|
|
"decreased_fraction": 0.54125,
|
|
"winner_agreement": 0.9885416666666667
|
|
},
|
|
"per_text": {
|
|
"fresh-zh-1": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 7.355600359875638,
|
|
"median": 9.903824666297758,
|
|
"p90": 12.397500772986202,
|
|
"mean": 9.861890605751142
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 7.264880226041378,
|
|
"median": 9.898966374569639,
|
|
"p90": 12.389470316075128,
|
|
"mean": 9.843565948927717
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.0807099828273353,
|
|
"median": -0.01318702061738719,
|
|
"p90": 0.03943376541940449,
|
|
"mean": -0.01832465682342548
|
|
}
|
|
},
|
|
"fresh-zh-2": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 7.355118430184284,
|
|
"median": 10.04562554273904,
|
|
"p90": 12.489044777205017,
|
|
"mean": 10.026567005234467
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 7.290105841531202,
|
|
"median": 10.026991319027882,
|
|
"p90": 12.491937373336743,
|
|
"mean": 10.021018915906389
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.08841258219596346,
|
|
"median": -0.0007642725752314838,
|
|
"p90": 0.07047669026190546,
|
|
"mean": -0.005548089328078445
|
|
}
|
|
},
|
|
"fresh-en-1": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 7.612360292624114,
|
|
"median": 10.418042154347027,
|
|
"p90": 12.785435912927255,
|
|
"mean": 10.306669554290066
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 7.578728433573872,
|
|
"median": 10.410359543867354,
|
|
"p90": 12.756459217588692,
|
|
"mean": 10.291179961128332
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.10092656919316453,
|
|
"median": -0.004362167048531873,
|
|
"p90": 0.05181777535002789,
|
|
"mean": -0.01548959316173382
|
|
}
|
|
},
|
|
"fresh-en-2": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 7.701260794945824,
|
|
"median": 10.84810403910145,
|
|
"p90": 12.86245594691649,
|
|
"mean": 10.616282357051954
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 7.649559166610384,
|
|
"median": 10.86194071558867,
|
|
"p90": 12.880097710531432,
|
|
"mean": 10.603806239711519
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.11578163334072719,
|
|
"median": 0.0025086418786655074,
|
|
"p90": 0.07020528538982156,
|
|
"mean": -0.012476117340434077
|
|
}
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"gsm8k": {
|
|
"distance": {
|
|
"base": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 6.013503917962642,
|
|
"median": 8.906046247944275,
|
|
"p90": 11.80188183284152,
|
|
"mean": 8.9017778676544
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.3758439948726651,
|
|
"median": 0.5566278904965172,
|
|
"p90": 0.737617614552595,
|
|
"mean": 0.5563611167284
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.023714400011049176,
|
|
"median": 0.053102157069302644,
|
|
"p90": 0.11071189502052788,
|
|
"mean": 0.06226195303507482
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.1451591630194614,
|
|
"median": 0.21821378474655584,
|
|
"p90": 0.3395296691096961,
|
|
"mean": 0.23231998812868285
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.46122887875718366,
|
|
"median": 0.5651361494965037,
|
|
"p90": 0.6673818130476021,
|
|
"mean": 0.5653113780132732
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.04589732432631252,
|
|
"median": 0.0717420375207896,
|
|
"p90": 0.1049109177088932,
|
|
"mean": 0.07365966028923801
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 10.0,
|
|
"median": 11.0,
|
|
"p90": 13.0,
|
|
"mean": 11.383333333333333
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.2638525074347854,
|
|
"median": 0.5107462792657316,
|
|
"p90": 0.8229535373859109,
|
|
"mean": 0.5267872787490235
|
|
}
|
|
},
|
|
"finetuned": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 5.982592644976027,
|
|
"median": 8.904116953413451,
|
|
"p90": 11.818114629158611,
|
|
"mean": 8.894597740069383
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.37391204031100167,
|
|
"median": 0.5565073095883407,
|
|
"p90": 0.7386321643224132,
|
|
"mean": 0.5559123587543364
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.023590256743970684,
|
|
"median": 0.05312810605871639,
|
|
"p90": 0.11162838720929225,
|
|
"mean": 0.06248960088996427
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.1452759810059557,
|
|
"median": 0.21887845306221598,
|
|
"p90": 0.3411258772849672,
|
|
"mean": 0.23272433431827313
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.46133812096720794,
|
|
"median": 0.5656479653815689,
|
|
"p90": 0.6690157446461986,
|
|
"mean": 0.5657888012738282
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.046235325509099874,
|
|
"median": 0.0717223750715486,
|
|
"p90": 0.10515117365332462,
|
|
"mean": 0.07383858903445469
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 10.0,
|
|
"median": 11.0,
|
|
"p90": 13.0,
|
|
"mean": 11.387708333333334
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.2662267918465659,
|
|
"median": 0.5119005234446377,
|
|
"p90": 0.8266748990863564,
|
|
"mean": 0.5289106785868004
|
|
}
|
|
},
|
|
"paired": {
|
|
"effective_heads_delta": {
|
|
"count": 4800,
|
|
"p10": -0.1591509935536532,
|
|
"median": -0.006747152142687618,
|
|
"p90": 0.13966946329318328,
|
|
"mean": -0.007180127585017427
|
|
},
|
|
"increased_fraction": 0.4725,
|
|
"decreased_fraction": 0.5275,
|
|
"winner_agreement": 0.970625
|
|
},
|
|
"per_text": {
|
|
"gsm8k-test-0000": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 6.029266196155418,
|
|
"median": 8.851735983796345,
|
|
"p90": 11.718544652857876,
|
|
"mean": 8.853039265973237
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 6.0185025710306865,
|
|
"median": 8.817902858881267,
|
|
"p90": 11.702200456136074,
|
|
"mean": 8.830522479570654
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.16698674746796555,
|
|
"median": -0.013576864730019622,
|
|
"p90": 0.10594365593799096,
|
|
"mean": -0.022516786402582326
|
|
}
|
|
},
|
|
"gsm8k-test-0002": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 5.902382519453832,
|
|
"median": 8.814916405222583,
|
|
"p90": 11.684873874581289,
|
|
"mean": 8.77475561713854
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 5.8262902839172686,
|
|
"median": 8.775601442382403,
|
|
"p90": 11.697385346787549,
|
|
"mean": 8.757974314967932
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.17076082866715633,
|
|
"median": -0.01759333235505922,
|
|
"p90": 0.1360708722019009,
|
|
"mean": -0.01678130217060961
|
|
}
|
|
},
|
|
"gsm8k-test-0003": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 6.009438240780767,
|
|
"median": 8.777000895915759,
|
|
"p90": 11.955121550413566,
|
|
"mean": 8.882677112308222
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 5.917225855217569,
|
|
"median": 8.809163836018584,
|
|
"p90": 11.993881916855772,
|
|
"mean": 8.891379637659044
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.15862776051872815,
|
|
"median": 0.004761271301123582,
|
|
"p90": 0.17976640500609026,
|
|
"mean": 0.008702525350821659
|
|
}
|
|
},
|
|
"gsm8k-test-0004": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 6.202833613253674,
|
|
"median": 9.134206344734961,
|
|
"p90": 11.802513491297152,
|
|
"mean": 9.096639475197604
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 6.200667188305828,
|
|
"median": 9.152546172625186,
|
|
"p90": 11.8442730026282,
|
|
"mean": 9.098514528079903
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.13349573481076807,
|
|
"median": -6.686357594176684e-05,
|
|
"p90": 0.135442900421867,
|
|
"mean": 0.0018750528823005652
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"divergence": {
|
|
"base": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 1.875457108203813,
|
|
"median": 3.8759650883020083,
|
|
"p90": 7.058800921305472,
|
|
"mean": 4.23391325232939
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.11721606926273831,
|
|
"median": 0.24224781801887552,
|
|
"p90": 0.441175057581592,
|
|
"mean": 0.26461957827058685
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.08444493087745249,
|
|
"median": 0.2085336117901452,
|
|
"p90": 0.5020835674830071,
|
|
"mean": 0.253319472363892
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.24145052956037794,
|
|
"median": 0.42031807846570973,
|
|
"p90": 0.7179985045869327,
|
|
"mean": 0.45173368725639157
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.6594778658704781,
|
|
"median": 0.8245290381303709,
|
|
"p90": 0.9284529929105115,
|
|
"mean": 0.8070190913253547
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.0038457043002112366,
|
|
"median": 0.011943733868154319,
|
|
"p90": 0.033109714130431796,
|
|
"mean": 0.015698414569523155
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 4.0,
|
|
"median": 6.0,
|
|
"p90": 9.0,
|
|
"mean": 6.355625
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.008542956798601153,
|
|
"median": 0.02885237614827929,
|
|
"p90": 0.07744390014122474,
|
|
"mean": 0.03753269894324168
|
|
}
|
|
},
|
|
"finetuned": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 1.8724184389765561,
|
|
"median": 3.866394622099988,
|
|
"p90": 7.027904627662534,
|
|
"mean": 4.227338292664017
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.11702615243603476,
|
|
"median": 0.24164966388124925,
|
|
"p90": 0.43924403922890837,
|
|
"mean": 0.26420864329150107
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.08510925478091586,
|
|
"median": 0.20921482882495243,
|
|
"p90": 0.5030065669595358,
|
|
"mean": 0.25406259246563967
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.2428326290560375,
|
|
"median": 0.4203880973993296,
|
|
"p90": 0.7175810034689571,
|
|
"mean": 0.45244932037391994
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.6583606497231608,
|
|
"median": 0.8246663787782131,
|
|
"p90": 0.9290253485256015,
|
|
"mean": 0.8076610638985393
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.0038046385599517733,
|
|
"median": 0.011966184570507691,
|
|
"p90": 0.033259522931901334,
|
|
"mean": 0.015794616049737992
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 4.0,
|
|
"median": 6.0,
|
|
"p90": 9.0,
|
|
"mean": 6.350208333333334
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.008829576032712793,
|
|
"median": 0.029070507331198314,
|
|
"p90": 0.0773094397965906,
|
|
"mean": 0.03776397203446997
|
|
}
|
|
},
|
|
"paired": {
|
|
"effective_heads_delta": {
|
|
"count": 4800,
|
|
"p10": -0.16764575951403446,
|
|
"median": -0.005232527666755082,
|
|
"p90": 0.147146854618983,
|
|
"mean": -0.006574959665373348
|
|
},
|
|
"increased_fraction": 0.4708333333333333,
|
|
"decreased_fraction": 0.5291666666666667,
|
|
"winner_agreement": 0.970625
|
|
},
|
|
"per_text": {
|
|
"gsm8k-test-0000": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 1.8431839972639215,
|
|
"median": 3.780522276551954,
|
|
"p90": 7.099145196184062,
|
|
"mean": 4.201646544541655
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 1.8178500400391873,
|
|
"median": 3.763288357122958,
|
|
"p90": 7.07356609840848,
|
|
"mean": 4.1789342231976345
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.17308023435001257,
|
|
"median": -0.010169449696791233,
|
|
"p90": 0.10548838266201924,
|
|
"mean": -0.022712321344020476
|
|
}
|
|
},
|
|
"gsm8k-test-0002": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 1.8908285003131953,
|
|
"median": 3.833913849764531,
|
|
"p90": 6.885469709475229,
|
|
"mean": 4.147627032640852
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 1.882329448871885,
|
|
"median": 3.8167847379661057,
|
|
"p90": 6.858497363265328,
|
|
"mean": 4.134744784084032
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.1809234445013397,
|
|
"median": -0.008109501325470725,
|
|
"p90": 0.14323401996659646,
|
|
"mean": -0.012882248556819976
|
|
}
|
|
},
|
|
"gsm8k-test-0003": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 1.8375673511907138,
|
|
"median": 3.71674632002963,
|
|
"p90": 7.103448318892024,
|
|
"mean": 4.206191412664545
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 1.8527289652676626,
|
|
"median": 3.706038772453823,
|
|
"p90": 7.179711007377434,
|
|
"mean": 4.214699774936081
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.16685274354222296,
|
|
"median": -0.0006958865930237756,
|
|
"p90": 0.1928958134565214,
|
|
"mean": 0.008508362271537395
|
|
}
|
|
},
|
|
"gsm8k-test-0004": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 1.9721590562878277,
|
|
"median": 4.12739017447695,
|
|
"p90": 7.14039115383894,
|
|
"mean": 4.380188019470508
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 1.9503733783669992,
|
|
"median": 4.112801742052209,
|
|
"p90": 7.108045377865233,
|
|
"mean": 4.380974388438318
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.15822896650350238,
|
|
"median": -0.00089962387563014,
|
|
"p90": 0.14725001676604996,
|
|
"mean": 0.0007863689678096667
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"QKV": {
|
|
"base": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 7.625732433101848,
|
|
"median": 10.387572837790255,
|
|
"p90": 12.80026994261185,
|
|
"mean": 10.270078901238158
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.4766082770688655,
|
|
"median": 0.649223302361891,
|
|
"p90": 0.8000168714132406,
|
|
"mean": 0.6418799313273849
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.01666490928122742,
|
|
"median": 0.03602013846457927,
|
|
"p90": 0.07321060761354471,
|
|
"mean": 0.04147005556576124
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.12950340585115858,
|
|
"median": 0.18680523070086608,
|
|
"p90": 0.2757132983869958,
|
|
"mean": 0.19655336416070232
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.42480706934875345,
|
|
"median": 0.5080353266733936,
|
|
"p90": 0.5975627753401668,
|
|
"mean": 0.5104096028921625
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.04665453901718664,
|
|
"median": 0.08203003363299655,
|
|
"p90": 0.12238641633949394,
|
|
"mean": 0.08348550233229636
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 11.0,
|
|
"median": 12.0,
|
|
"p90": 13.0,
|
|
"mean": 11.844375
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.4073887151665986,
|
|
"median": 0.8801846825517714,
|
|
"p90": 1.4507116448599853,
|
|
"mean": 0.9032721801065388
|
|
}
|
|
},
|
|
"finetuned": {
|
|
"effective_heads": {
|
|
"count": 4800,
|
|
"p10": 7.576624449214384,
|
|
"median": 10.374178847146444,
|
|
"p90": 12.821177123485356,
|
|
"mean": 10.264366715676784
|
|
},
|
|
"effective_fraction": {
|
|
"count": 4800,
|
|
"p10": 0.473539028075899,
|
|
"median": 0.6483861779466528,
|
|
"p90": 0.8013235702178347,
|
|
"mean": 0.641522919729799
|
|
},
|
|
"concentration": {
|
|
"count": 4800,
|
|
"p10": 0.01652902257201789,
|
|
"median": 0.03615271705068857,
|
|
"p90": 0.07411723202733041,
|
|
"mean": 0.04167572867201576
|
|
},
|
|
"top1": {
|
|
"count": 4800,
|
|
"p10": 0.12935968415437357,
|
|
"median": 0.18691265969017154,
|
|
"p90": 0.27822878056333283,
|
|
"mean": 0.19706959048869968
|
|
},
|
|
"top25": {
|
|
"count": 4800,
|
|
"p10": 0.4240190586277136,
|
|
"median": 0.5083111063316925,
|
|
"p90": 0.5988581244801308,
|
|
"mean": 0.5105091226254993
|
|
},
|
|
"bottom25": {
|
|
"count": 4800,
|
|
"p10": 0.0467784088506235,
|
|
"median": 0.08242644414525736,
|
|
"p90": 0.12263218106221838,
|
|
"mean": 0.08378531660409673
|
|
},
|
|
"heads90": {
|
|
"count": 4800,
|
|
"p10": 11.0,
|
|
"median": 12.0,
|
|
"p90": 13.0,
|
|
"mean": 11.856041666666666
|
|
},
|
|
"total": {
|
|
"count": 4800,
|
|
"p10": 0.4107431098818779,
|
|
"median": 0.8829612950794399,
|
|
"p90": 1.4518962780828588,
|
|
"mean": 0.9056679941666214
|
|
}
|
|
},
|
|
"paired": {
|
|
"effective_heads_delta": {
|
|
"count": 4800,
|
|
"p10": -0.12757638238450753,
|
|
"median": 0.0037255281277426278,
|
|
"p90": 0.10112692014932954,
|
|
"mean": -0.00571218556137241
|
|
},
|
|
"increased_fraction": 0.5208333333333334,
|
|
"decreased_fraction": 0.4791666666666667,
|
|
"winner_agreement": 0.98125
|
|
},
|
|
"per_text": {
|
|
"gsm8k-test-0000": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 7.7989315507884305,
|
|
"median": 10.521758099510357,
|
|
"p90": 12.900443813085287,
|
|
"mean": 10.413248549181015
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 7.705277296646966,
|
|
"median": 10.515329005859204,
|
|
"p90": 12.93002657004937,
|
|
"mean": 10.395004783500097
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.14676809507078134,
|
|
"median": 0.004664857327336236,
|
|
"p90": 0.09345818785291339,
|
|
"mean": -0.01824376568091725
|
|
}
|
|
},
|
|
"gsm8k-test-0002": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 7.421168885724907,
|
|
"median": 10.089098930849946,
|
|
"p90": 12.800938492049042,
|
|
"mean": 10.070913598602885
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 7.428372247200956,
|
|
"median": 10.104854688534296,
|
|
"p90": 12.80511577506368,
|
|
"mean": 10.069668789579037
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.12106868600811112,
|
|
"median": 0.00750254328635247,
|
|
"p90": 0.09535499714872045,
|
|
"mean": -0.0012448090238489822
|
|
}
|
|
},
|
|
"gsm8k-test-0003": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 7.503553341013432,
|
|
"median": 10.018123459522165,
|
|
"p90": 12.583518311707772,
|
|
"mean": 10.084090114274893
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 7.459289228486346,
|
|
"median": 10.039510574599497,
|
|
"p90": 12.60156308027857,
|
|
"mean": 10.082087289747665
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.13171799446835042,
|
|
"median": -0.002012921035841053,
|
|
"p90": 0.12220187488991908,
|
|
"mean": -0.0020028245272282457
|
|
}
|
|
},
|
|
"gsm8k-test-0004": {
|
|
"base": {
|
|
"count": 1200,
|
|
"p10": 7.85864274660863,
|
|
"median": 10.79465641409404,
|
|
"p90": 12.796374361069722,
|
|
"mean": 10.512063342893834
|
|
},
|
|
"finetuned": {
|
|
"count": 1200,
|
|
"p10": 7.781806995703267,
|
|
"median": 10.808550667269296,
|
|
"p90": 12.815653984573563,
|
|
"mean": 10.510705999880338
|
|
},
|
|
"paired_delta": {
|
|
"count": 1200,
|
|
"p10": -0.10302150633895835,
|
|
"median": 0.004346750055341353,
|
|
"p90": 0.09226896386400388,
|
|
"mean": -0.0013573430134951603
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|