Quantify adjacent-layer head influence concentration on CPU
This commit is contained in:
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"scope": "all 23 adjacent layer transitions; source attention head contribution to next effective Q/K/V",
|
||||
"samples": "4 previously authored zh/en texts, first32 tokens, descriptive",
|
||||
"primary_queries": [
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
12,
|
||||
13,
|
||||
14,
|
||||
15,
|
||||
16,
|
||||
17,
|
||||
18,
|
||||
19,
|
||||
20,
|
||||
21,
|
||||
22,
|
||||
23,
|
||||
24,
|
||||
25,
|
||||
26,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31
|
||||
],
|
||||
"influence": "L2 change / baseline token recipient-head L2; QKV score sqrt(mean of three squared relative changes))",
|
||||
"linear_QK": "after causal convolution/SiLU and delta-rule L2 normalization; Q additionally divided by sqrt(key dim)",
|
||||
"full_QK": "QK norm and partial RoPE applied; K/V repeated to query heads for indexing (not independent KV observations)",
|
||||
"single_ablation": "remove complete source head projected output at all tokens; recompute source postattention normalization, MLP, residual and next input normalization",
|
||||
"joint_ablation": "weak/strong/random half and all attention heads; one coherent token-varying removal mask for each recipient query head; ranking uses QKV single effects; no fitting",
|
||||
"continuous": "effective heads=1/sum(p^2); fraction=effective/H; concentration=(sum(p^2)-1/H)/(1-1/H)",
|
||||
"zero_total": "undefined concentration (NaN arrays), count explicitly",
|
||||
"cost_budget_seconds": 7200,
|
||||
"no_training": true,
|
||||
"keep_all_weights_and_caches": true,
|
||||
"limitations": [
|
||||
"not final LM quality or pruning proof",
|
||||
"individual effects not additive; normalized scores not true contribution probabilities",
|
||||
"token-varying masks are input-specific diagnostic and not a deployable fixed head subset",
|
||||
"linear attention does not have an ordinary softmax probability matrix",
|
||||
"Q/K/V scales handled separately; combined QKV is a declared balanced relative-effect score"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user