96 lines
2.6 KiB
JSON
96 lines
2.6 KiB
JSON
{
|
|
"sets": {
|
|
"test4": {
|
|
"native": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.1577236725451122,
|
|
"mean_margin": 2.6222333908081055,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"direct": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.09467621731363618,
|
|
"mean_margin": 3.157541592915853,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"permutation": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.10225362466948813,
|
|
"mean_margin": 3.069669246673584,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"mix": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.10946264729409035,
|
|
"mean_margin": 2.9876694679260254,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"local32": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.11173090120474012,
|
|
"mean_margin": 2.9486522674560547,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"mix_local32": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.1155495746250453,
|
|
"mean_margin": 2.9375176429748535,
|
|
"prediction_agreement_with_native": 12
|
|
}
|
|
},
|
|
"test8": {
|
|
"native": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.13243424793047875,
|
|
"mean_margin": 2.652334133783976,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"direct": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.07933497390150783,
|
|
"mean_margin": 3.1618269284566245,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"permutation": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.08440917052410739,
|
|
"mean_margin": 3.135169267654419,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"mix": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.09210604798693496,
|
|
"mean_margin": 3.0242624282836914,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"local32": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.09402417229313187,
|
|
"mean_margin": 2.9872288703918457,
|
|
"prediction_agreement_with_native": 12
|
|
},
|
|
"mix_local32": {
|
|
"n": 12,
|
|
"correct": 12,
|
|
"mean_choice_nll": 0.09681914153941336,
|
|
"mean_margin": 2.970529397328695,
|
|
"prediction_agreement_with_native": 12
|
|
}
|
|
}
|
|
},
|
|
"native_last_layer_replay_parity": true,
|
|
"calibration_reused": true,
|
|
"seconds": 681.1676366906613
|
|
}
|