440 lines
13 KiB
JSON
440 lines
13 KiB
JSON
{
|
|
"scale": 2,
|
|
"modules": [
|
|
{
|
|
"module": "base_model.model.model.layers.0.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.005800894498759106,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.0.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.005587309558024041,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.0.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11553687855202736,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98080
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.0.linear_attn.out_proj",
|
|
"delta_frobenius": 0.07066058472168732,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.1.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.007598507552503654,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.1.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.005952066438734307,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.1.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11372388303772911,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.1.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06772797765001827,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.10.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.006948404408305538,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.10.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.006482788372589787,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.10.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.12268737050249612,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.10.linear_attn.out_proj",
|
|
"delta_frobenius": 0.07115427316588817,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.12.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.008057688891653094,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.12.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.0062793159111755705,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.12.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.12858468661902908,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.12.linear_attn.out_proj",
|
|
"delta_frobenius": 0.07182839421740687,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.13.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.008557439622189963,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.13.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.007115977035191728,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.13.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11796408280860549,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.13.linear_attn.out_proj",
|
|
"delta_frobenius": 0.07138495398589927,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.14.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.006949849201574363,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.14.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.00591766667398592,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.14.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.1261444517982034,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.14.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06629817232217884,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.16.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.005377119705864388,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.16.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.0062457490691171765,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.16.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11211941675598038,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.16.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06683234587772328,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.17.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.005651240023844441,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.17.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.005017673680232789,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.17.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.10839220497781887,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.17.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06235235087547191,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.18.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.005962822244385937,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.18.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.005433131707293152,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.18.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.10842222773312146,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.18.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06376462519822836,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.2.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.005603124386020784,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.2.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.005609122443998325,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.2.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11155973909218166,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.2.linear_attn.out_proj",
|
|
"delta_frobenius": 0.07515596969802131,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.20.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.004718516392369627,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.20.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.004928997611173398,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.20.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11031072258726142,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.20.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06584978157807943,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.21.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.006667677921336111,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.21.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.004916474716387209,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.21.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.1212656379357851,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.21.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06388752120982902,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.22.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.00581111977053266,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.22.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.007386389271654893,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.22.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.1259512522444594,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.22.linear_attn.out_proj",
|
|
"delta_frobenius": 0.0680125186972645,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.4.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.0059997460012478345,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.4.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.00656546247149169,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.4.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11382676425612075,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.4.linear_attn.out_proj",
|
|
"delta_frobenius": 0.07099685915533233,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.5.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.005806313752452173,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.5.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.0054898577193761905,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.5.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.10713946674607827,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.5.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06715117041696626,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.6.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.006425389171683832,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.6.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.005934459708371355,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.6.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11102787853966796,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.6.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06802617128857469,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.8.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.006018741866067713,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.8.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.005080820350280815,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.8.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11049952559087846,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.8.linear_attn.out_proj",
|
|
"delta_frobenius": 0.06629254535251425,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.9.linear_attn.in_proj_a",
|
|
"delta_frobenius": 0.008455871110880087,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.9.linear_attn.in_proj_b",
|
|
"delta_frobenius": 0.0057509725702040776,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 256
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.9.linear_attn.in_proj_qkv",
|
|
"delta_frobenius": 0.11615958892285279,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 98304
|
|
},
|
|
{
|
|
"module": "base_model.model.model.layers.9.linear_attn.out_proj",
|
|
"delta_frobenius": 0.07103984196594977,
|
|
"A_nonzero": 32768,
|
|
"B_nonzero": 32768
|
|
}
|
|
],
|
|
"nonzero_update_modules": 72,
|
|
"norm_identity": "||2BA||_F^2 = 4 trace((B^T B)(A A^T))"
|
|
}
|