Archive research charts and compare GSM8K LoRA head concentration
This commit is contained in:
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"primary": "same six full-attention receivers and original4x32 token cohort as published8.931822/16",
|
||||
"adapter_repo": "Avifenesh/gdn-beta-gate-qwen3.5-2b-grpo",
|
||||
"adapter_revision": "038d8be9bd0edd3883fa019651b02bdf65609468",
|
||||
"adapter_file": "lora/seed0/adapter.safetensors",
|
||||
"adapter_sha256": "93a3f43a4356f84da0d13ecc622fee13e41ba15d237cdaad6094f243899c62d6",
|
||||
"base_revision": "15852e8c16360a2fea060d615a32b45270f8a8fc",
|
||||
"scale": 2,
|
||||
"rank": 16,
|
||||
"arm": "pure LoRA",
|
||||
"training": "publisher-reported GSM8K GRPO; we do no training",
|
||||
"sample_choice": "four original texts; first4 canonical GSM8K test questions with >=32tokens, length selection only, first32tokens; question only",
|
||||
"source_layers": [
|
||||
2,
|
||||
6,
|
||||
10,
|
||||
14,
|
||||
18,
|
||||
22
|
||||
],
|
||||
"primary_queries": [
|
||||
7,
|
||||
8,
|
||||
9,
|
||||
10,
|
||||
11,
|
||||
12,
|
||||
13,
|
||||
14,
|
||||
15,
|
||||
16,
|
||||
17,
|
||||
18,
|
||||
19,
|
||||
20,
|
||||
21,
|
||||
22,
|
||||
23,
|
||||
24,
|
||||
25,
|
||||
26,
|
||||
27,
|
||||
28,
|
||||
29,
|
||||
30,
|
||||
31
|
||||
],
|
||||
"main_metric": "concentration(sqrt(single_JS)); same FP32 tensors and sequence-wide donor-head deletion as prior run",
|
||||
"cohort_pairing": "each comparison uses same question/tokenIDs before and after; base legacy arrays reused with elementwise checks",
|
||||
"runtime": "read-only FP32 effective W+2BA; no BF16 re-rounding or quantization",
|
||||
"controls": "head sum, null mask, explicit low-rank versus effective-weight full forward relL2<1e-4",
|
||||
"budget_seconds": 7200,
|
||||
"keep_all_weights_and_caches": true,
|
||||
"limitations": [
|
||||
"one existing training seed",
|
||||
"4 texts per cohort, 32 tokens each",
|
||||
"dependent query/head observations, not iid textual samples",
|
||||
"no generation or accuracy measurement",
|
||||
"GDN targets only; ordinary attention weights stay unchanged",
|
||||
"no pruning or functional interchangeability claim"
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user