49 lines
1.8 KiB
JSON
49 lines
1.8 KiB
JSON
{
|
|
"scope": "all 23 adjacent layer transitions; source attention head contribution to next effective Q/K/V",
|
|
"samples": "4 previously authored zh/en texts, first32 tokens, descriptive",
|
|
"primary_queries": [
|
|
7,
|
|
8,
|
|
9,
|
|
10,
|
|
11,
|
|
12,
|
|
13,
|
|
14,
|
|
15,
|
|
16,
|
|
17,
|
|
18,
|
|
19,
|
|
20,
|
|
21,
|
|
22,
|
|
23,
|
|
24,
|
|
25,
|
|
26,
|
|
27,
|
|
28,
|
|
29,
|
|
30,
|
|
31
|
|
],
|
|
"influence": "L2 change / baseline token recipient-head L2; QKV score sqrt(mean of three squared relative changes))",
|
|
"linear_QK": "after causal convolution/SiLU and delta-rule L2 normalization; Q additionally divided by sqrt(key dim)",
|
|
"full_QK": "QK norm and partial RoPE applied; K/V repeated to query heads for indexing (not independent KV observations)",
|
|
"single_ablation": "remove complete source head projected output at all tokens; recompute source postattention normalization, MLP, residual and next input normalization",
|
|
"joint_ablation": "weak/strong/random half and all attention heads; one coherent token-varying removal mask for each recipient query head; ranking uses QKV single effects; no fitting",
|
|
"continuous": "effective heads=1/sum(p^2); fraction=effective/H; concentration=(sum(p^2)-1/H)/(1-1/H)",
|
|
"zero_total": "undefined concentration (NaN arrays), count explicitly",
|
|
"cost_budget_seconds": 7200,
|
|
"no_training": true,
|
|
"keep_all_weights_and_caches": true,
|
|
"limitations": [
|
|
"not final LM quality or pruning proof",
|
|
"individual effects not additive; normalized scores not true contribution probabilities",
|
|
"token-varying masks are input-specific diagnostic and not a deployable fixed head subset",
|
|
"linear attention does not have an ordinary softmax probability matrix",
|
|
"Q/K/V scales handled separately; combined QKV is a declared balanced relative-effect score"
|
|
]
|
|
}
|