验证 Qwen3.5-2B 的免训练注意力共享和 LiSA 式轻量校准 #10

Merged
Zayden-Jung merged 1 commits from codex/lisa-attention-pilot into main 2026-11-22 14:12:36 +00:00
8 changed files with 3073 additions and 0 deletions
+26
View File
@@ -40,3 +40,29 @@ All weights and caches stay on the server until explicit user deletion. Git cont
all figure tables and the selected small probability matrices; large caches have exact
paths/sizes/hashes in `external-caches.json`. Old validation texts were used in earlier
experiments; the four fresh texts were fixed before the new cross-layer measurements.
## LiSA-style single-layer pilot
The pilot reuses layer 19's actual pre-softmax scores at layer 23, with the
recipient's own values, output gate and output projection. It compares native
attention, direct reuse, a frozen one-to-one head assignment, linear mixing
of donor heads, a 32-coordinate recipient-only control, and joint donor/recipient
mixing. The linear variants use a closed-form ridge fit to the native scores,
centred within each causal key row. The recipient branch takes a fixed subset of
post-RoPE coordinates rather than learning a low-rank Q/K projection. The
calibration uses 12 synthetic prompts without answer labels. All checkpoint
weights are frozen.
Evaluation scores the four candidate option tokens on 24 disjoint retrieval
prompts, including 12 with more table entries than calibration. Choice NLL
normalizes over those four tokens. Run against the retained checkpoint on the
remote CPU environment:
```sh
OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 venv/bin/python -B -m crossqkv.lisa_pilot \
--source runs/cpu-qwen35-2b --out runs/NEW_LISA_DIRECTORY
```
The diagnostic forward still computes native full Q/K and does not measure
latency or parameter savings. Refer to LiSA (arXiv:2408.01890, §5.1) for its
learned nonlinear head aligner and learned low-rank Q/K projections.
+265
View File
@@ -0,0 +1,265 @@
"""Single-site LiSA-style pilot on the retained Qwen3.5-2B checkpoint."""
import argparse
import hashlib
import json
from pathlib import Path
import random
import time
import numpy as np
from tokenizers import Tokenizer
from crossqkv.core import QKVModel, assignment
from functional.model import PREFIX
from functional.run import write_json
from pilot.tensors import decode
DONOR, TARGET = 19, 23
WIDTH = 32
MODES = ('native', 'direct', 'permutation', 'mix', 'local32', 'mix_local32')
def scores(q, k):
keys = np.repeat(k, q.shape[2] // k.shape[2], axis=2)
return np.einsum('bthd,bshd->bhts', q, keys) / np.float32(np.sqrt(q.shape[-1]))
def probabilities(x):
x = np.where(np.tri(x.shape[-1], dtype=bool), x, -np.inf)
exp = np.exp(x - x.max(axis=-1, keepdims=True))
return exp / exp.sum(axis=-1, keepdims=True)
def centered_rows(x, lengths):
values = []
for i, length in enumerate(lengths):
query, key = np.tril_indices(length)
board = x[i, :, :length, :length]
mean = np.where(np.tri(length, dtype=bool), board, 0).sum(-1) / np.arange(1, length + 1)
centered = board - mean[..., None]
values.append(centered[:, query, key].T)
return np.concatenate(values).astype(np.float64)
def solve(x, y):
gram = x.T @ x
ridge = 1e-3 * np.trace(gram) / gram.shape[0]
return np.linalg.solve(gram + max(ridge, 1e-12) * np.eye(len(gram)), x.T @ y).astype(np.float32)
def fit(donor, target, local):
cost = np.mean((target[:, :, None] - donor[:, None, :]) ** 2, axis=0)
perm = assignment(cost)
return {
'permutation': list(perm),
'mix': solve(donor, target),
'local32': solve(local, target),
'mix_local32': solve(np.concatenate((donor, local), axis=1), target),
}
def predict(donor, local, mode, mapping):
if mode == 'direct':
return donor
if mode == 'permutation':
return donor[:, mapping[mode]]
if mode == 'mix':
inputs = donor
elif mode == 'local32':
inputs = local
else:
inputs = np.concatenate((donor, local), axis=1)
return np.einsum('bhts,hi->bits', inputs, mapping[mode])
class Study(QKVModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
dim = self.config['head_dim']
if dim % WIDTH:
raise ValueError('Expected head dimension divisible by fixed local width')
# Deterministic channel sample in post-RoPE Q/K coordinates.
self.coordinates = np.arange(0, dim, dim // WIDTH)
self.capture_scores = False
def normalization(self, x, name):
if self.capture_scores and name == f'layers.{TARGET}.input_layernorm':
self.recipient_input = x
return super().normalization(x, name)
def full_attention(self, h, layer, shared_p=None):
if not self.capture_scores or layer not in (DONOR, TARGET):
return super().full_attention(h, layer, shared_p)
parts = self.parts(h, layer, heads=False)
if layer == DONOR:
self.donor = scores(parts['q'], parts['k'])
else:
q, k = parts['q'], parts['k']
self.teacher = scores(q, k)
self.local = scores(q[..., self.coordinates], k[..., self.coordinates])
self.recipient = parts
return parts['output'], parts['p']
def replay(self, modes):
if TARGET != len(self.config['layer_types']) - 1:
raise ValueError('Replay requires changing only the final layer')
parts = self.recipient
repeats = self.config['num_attention_heads'] // self.config['num_key_value_heads']
values = np.repeat(parts['v'], repeats, axis=2)
contexts = [np.einsum('bhts,bshd->bthd',
probabilities(predict(self.donor, self.local, mode, self.mapping)),
values) for mode in modes]
context = np.concatenate([parts['context'], *contexts])
gate = np.concatenate([parts['gate']] * (1 + len(modes)))
residual = np.concatenate([self.recipient_input] * (1 + len(modes)))
hidden = residual + self.project(context, gate, TARGET)
mlp_input = self.normalization(hidden, f'layers.{TARGET}.post_attention_layernorm')
return np.split(self.normalization(hidden + self.mlp(mlp_input, TARGET), 'norm'),
1 + len(modes))
def dataset(split, count):
seed = {'calibration': 2026112201, 'test4': 2026112202, 'test8': 2026112203}[split]
rng = random.Random(seed)
size = 8 if split == 'test8' else 4
words = ('amber', 'birch', 'cedar', 'dune', 'elm', 'fern', 'grove', 'heath',
'iris', 'jade', 'kelp', 'larch', 'maple', 'nettle', 'olive', 'pine')
examples = []
answers = [i % 4 for i in range(count)]
rng.shuffle(answers)
for i, answer in enumerate(answers):
keys = rng.sample(words, size)
values = rng.sample(range(10, 100), size)
index = i % size
other = rng.sample([v for j, v in enumerate(values) if j != index], 3)
other.insert(answer, values[index])
items = '\n'.join(f'{k}: {v}' for k, v in zip(keys, values))
options = '\n'.join(f'{j + 1}. {v}' for j, v in enumerate(other))
prompt = (f'Use the lookup table. Answer with only the option number.\n{items}\n'
f'Which option is the value for {keys[index]}?\n{options}\nAnswer:')
examples.append({'id': f'{split}-{i:03}', 'prompt': prompt, 'answer_index': answer,
'evidence_position': index, 'entries': size})
return examples
def batch(tokenizer, examples):
encoded = []
choices = None
for ex in examples:
continuations = [tokenizer.encode(f"{ex['prompt']} {i}", add_special_tokens=False).ids
for i in range(1, 5)]
prefix = continuations[0][:-1]
labels = [row[-1] for row in continuations]
if any(row[:-1] != prefix for row in continuations) or len(set(labels)) != 4:
raise ValueError('Choices must differ only at the final token')
if choices is not None and labels != choices:
raise ValueError('Choice tokenization varies by prompt')
encoded.append(prefix)
choices = labels
lengths = [len(ids) for ids in encoded]
ids = np.array([row + [0] * (max(lengths) - len(row)) for row in encoded], dtype=np.int64)
return ids, lengths, choices
def evaluate(lm, tokenizer, examples, batch_size=4):
results = {mode: [] for mode in MODES}
for start in range(0, len(examples), batch_size):
group = examples[start:start + batch_size]
ids, lengths, choices = batch(tokenizer, group)
n = len(group)
lm.capture_scores = True
native, _ = lm.forward(ids)
lm.capture_scores = False
replay_native, *replayed = lm.replay(MODES[1:])
if not np.allclose(replay_native, native, atol=2e-5, rtol=1e-5):
raise ValueError('Native last-layer replay parity failed')
token_rows = lm.tensors[PREFIX + 'embed_tokens.weight']
weight = decode(token_rows.raw.reshape(token_rows.shape)[choices], token_rows.dtype)
token_rows.evict()
for mode, hidden in zip(MODES, [native, *replayed]):
logits = hidden[np.arange(n), np.array(lengths) - 1] @ weight.T
for ex, row, length in zip(group, logits.astype(float), lengths):
gold = ex['answer_index']
results[mode].append({
'id': ex['id'], 'length': length, 'gold': gold,
'logits': row.tolist(), 'predicted': int(row.argmax()),
'choice_nll': float(np.logaddexp.reduce(row) - row[gold]),
'margin': float(row[gold] - np.max(np.delete(row, gold))),
})
print(f"{group[-1]['id']}: evaluated {n} prompts in all six modes", flush=True)
return results
def summarize(results):
return {split: {mode: {
'n': len(rows),
'correct': sum(row['gold'] == row['predicted'] for row in rows),
'mean_choice_nll': float(np.mean([row['choice_nll'] for row in rows])),
'mean_margin': float(np.mean([row['margin'] for row in rows])),
'prediction_agreement_with_native': sum(row['predicted'] == base['predicted']
for row, base in zip(rows, modes['native'])),
} for mode, rows in modes.items()} for split, modes in results.items()}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
parser.add_argument('--calibration-run', type=Path, help='Reuse a matching completed calibration')
parser.add_argument('--seconds', type=int, default=1800)
args = parser.parse_args()
start = time.monotonic()
args.out.mkdir()
source = args.source
config = json.loads((source / 'model_config.json').read_text())
tokenizer = Tokenizer.from_file(str(source / 'tokenizer.json'))
lm = Study(source / 'weights', config, seconds=args.seconds)
kinds = lm.config['layer_types']
if any(kinds[i] != 'full_attention' for i in (DONOR, TARGET)):
raise ValueError('Requires the retained architecture with full attention at 19 and 23')
source_files = ['model_config.json', 'tokenizer.json', 'manifest.json']
protocol = {'donor': DONOR, 'target': TARGET, 'local_coordinate_count': WIDTH,
'modes': MODES, 'source': str(source),
'source_sha256': {f: hashlib.sha256((source/f).read_bytes()).hexdigest() for f in source_files},
'choice_nll_scope': 'normalized over four option-number logits only'}
problems = {s: dataset(s, n) for s, n in [('calibration', 12), ('test4', 12), ('test8', 12)]}
if len({e['prompt'] for entries in problems.values() for e in entries}) != 36:
raise ValueError('Repeated prompt across pilot sample')
write_json(args.out / 'protocol.json', protocol)
write_json(args.out / 'problems.json', problems)
if args.calibration_run:
previous = args.calibration_run
if (json.loads((previous / 'protocol.json').read_text()) !=
json.loads((args.out / 'protocol.json').read_text()) or
json.loads((previous / 'problems.json').read_text()) != problems):
raise ValueError('Calibration inputs or checkpoint provenance differ')
lm.mapping = {k: np.asarray(v, dtype=np.int64 if k == 'permutation' else np.float32)
for k, v in json.loads((previous / 'calibration.json').read_text()).items()}
else:
ids, lengths, _ = batch(tokenizer, problems['calibration'])
lm.capture_scores = True
native, _ = lm.forward(ids)
measured = [centered_rows(x, lengths) for x in (lm.donor, lm.teacher, lm.local)]
lm.capture_scores = False
reference, _ = lm.forward(ids[:4])
if not np.allclose(native[:4], reference, atol=1e-5, rtol=1e-5):
raise ValueError('Capture path disagrees with the unmodified reference')
lm.mapping = fit(*measured)
print('Calibration ready', flush=True)
write_json(args.out / 'calibration.json', {
k: np.asarray(v).tolist() for k, v in lm.mapping.items()
})
results = {}
for split in ('test4', 'test8'):
results[split] = evaluate(lm, tokenizer, problems[split])
print(split, summarize(results)[split], flush=True)
write_json(args.out / 'results.json', results)
write_json(args.out / 'summary.json', {'sets': summarize(results),
'native_last_layer_replay_parity': True,
'calibration_reused': bool(args.calibration_run),
'seconds': time.monotonic() - start})
if __name__ == '__main__':
main()
+23
View File
@@ -0,0 +1,23 @@
# Qwen3.5-2B 跨层注意力共享先导实验
固定原 Qwen3.5-2B 检查点,测试第 19 层实际前向产生的 softmax 前注意力分数能否替代第 23 层的分数。层编号从零开始。所有替换保留接收端的 V、gate 和输出投影。
| 方法 | 4 项表正确 | 8 项表正确 | 合计选择 NLL |
| --- | ---: | ---: | ---: |
| 原模型 | 12/12 | 12/12 | 0.145079 |
| 原序直接共享 | 12/12 | 12/12 | 0.087006 |
| 头重排后共享 | 12/12 | 12/12 | 0.093331 |
| 共享分数的线性头混合 | 12/12 | 12/12 | 0.100784 |
| 接收层 32 坐标,无共享 | 12/12 | 12/12 | 0.102878 |
| 共享分数与接收层 32 坐标联合拟合 | 12/12 | 12/12 | 0.106184 |
选择 NLL 在四个选项 token 上归一化后按题取均值。直接共享在这 24 道题上保持准确率,也在每道题上降低了选择 NLL。更低的损失反映正确选项相对其余三个选项的得分,样本中的准确率已经达到上限。这个结果支持冻结权重的单层替换在这些合成检索题上可用。轻量拟合没有带来额外收益。
## 实验与检查
- 12 道 4 项表校准题用于冻结头排列、闭式岭回归映射。拟合只使用原模型内部注意力分数,不使用正确选项标签。全套 36 条提示词无重复。
- Qwen 权重全程冻结。没有反向传播,也没有训练 LiSA 原版的非线性头对齐网络或低秩 Q/K 投影。32 坐标来自确定性下采样的接收层 post-RoPE Q/K。
- 远程 CPU 的 8 项跨层回归测试通过。合成数值检查通过,覆盖排列方向、因果遮罩和联合拟合重建。校准时的原模型路径一致性检查以及六批留出题的末层原样重放一致性检查均通过。
- 两次较慢的远程执行切换后保留旧目录,最终版本沿用第二次已完成的校准,并核对原检查点文件指纹及全部提示词。末层重放阶段耗时约 681 秒。逐题分数、答案和 NLL 已从原始 logits 独立复核,最终运行脚本 SHA256 为 `a64d1b50a10bdde893c4bc274365c0c072f13c0a2359b079cfa8421b9ce068f4`。
评测只覆盖一个模型、一个接收层、一类合成任务、约束于四个选项的读出。诊断仍计算原始 Q/K,未测吞吐、压缩率、自由生成或多层累积替换效果。实验方法和复现入口见 [crossqkv/README.md](../../crossqkv/README.md)。
@@ -0,0 +1,338 @@
{
"permutation": [
0,
2,
4,
3,
7,
6,
5,
1
],
"mix": [
[
0.24426038563251495,
0.055573295801877975,
-0.12299608439207077,
-0.09282179176807404,
-0.047870442271232605,
0.07772502303123474,
-0.22192631661891937,
0.29638272523880005
],
[
0.03558433800935745,
0.03896130993962288,
-0.18795007467269897,
0.002625135239213705,
-0.16552278399467468,
-0.023057416081428528,
-0.39929720759391785,
0.43178120255470276
],
[
0.1871931552886963,
0.44749122858047485,
-0.04751807823777199,
0.04143448546528816,
0.23914210498332977,
0.18653616309165955,
0.12161331623792648,
-0.20545999705791473
],
[
-0.027203943580389023,
-0.23345178365707397,
-0.03906247019767761,
0.0939483568072319,
0.11926761269569397,
-0.08872664719820023,
0.14460818469524384,
0.06796323508024216
],
[
0.09347549825906754,
0.2144748717546463,
1.078420877456665,
0.5173454880714417,
0.5393254160881042,
0.31546834111213684,
0.7830604314804077,
0.09864718466997147
],
[
-0.015639908611774445,
0.13545557856559753,
0.20001526176929474,
-0.0403873436152935,
-0.14243841171264648,
0.10669009387493134,
0.22260507941246033,
-0.008936765603721142
],
[
0.2405557930469513,
0.054704487323760986,
0.011206595227122307,
0.2406461238861084,
-0.026383306831121445,
0.25195246934890747,
0.09300804138183594,
0.10066510736942291
],
[
0.02644052356481552,
0.10034456104040146,
0.07825803011655807,
0.10650064051151276,
0.3668587803840637,
-0.007412006612867117,
0.061572253704071045,
-0.06577901542186737
]
],
"local32": [
[
0.2867194414138794,
0.06671284139156342,
-0.2674592435359955,
-0.3276604115962982,
-0.4070841670036316,
-0.006673475727438927,
-0.2074313908815384,
0.12887436151504517
],
[
-0.0516749806702137,
0.4364052712917328,
0.4763493239879608,
0.569825291633606,
0.5895766019821167,
0.23220211267471313,
0.4529169201850891,
0.020792270079255104
],
[
0.030566174536943436,
-0.0014433575561270118,
0.9291507601737976,
0.07762160897254944,
0.09363061934709549,
0.11929107457399368,
0.6625307202339172,
0.011318162083625793
],
[
0.6921118497848511,
0.3367474377155304,
-0.20861877501010895,
0.6061498522758484,
0.4020693004131317,
0.4614187479019165,
-0.120954729616642,
0.6977306008338928
],
[
0.010575470514595509,
0.03395608440041542,
-0.17418035864830017,
0.05686786770820618,
0.3456997573375702,
-0.16309495270252228,
0.13770024478435516,
-0.06023705378174782
],
[
0.34056422114372253,
0.3800618350505829,
0.46083998680114746,
0.379044771194458,
0.38687247037887573,
0.7362816333770752,
0.2947295904159546,
-0.053121648728847504
],
[
0.03288573399186134,
0.11208789050579071,
0.5949627161026001,
0.09720376133918762,
0.012026847340166569,
0.062014296650886536,
0.8560540080070496,
0.004886063747107983
],
[
0.3228500783443451,
0.3110385239124298,
-0.03597158193588257,
0.004630092531442642,
0.1255260556936264,
0.3001147508621216,
-0.2873961925506592,
0.7489231824874878
]
],
"mix_local32": [
[
0.11679738759994507,
0.06494305282831192,
-0.07491274178028107,
-0.10080373287200928,
-0.02167605608701706,
-0.05343199521303177,
-0.1710113286972046,
0.25583943724632263
],
[
0.07517975568771362,
0.04044078290462494,
-0.06897048652172089,
0.022282306104898453,
-0.14309072494506836,
0.033187106251716614,
-0.3017081618309021,
0.44381454586982727
],
[
0.06857457011938095,
0.3892330229282379,
-0.10634169727563858,
-0.019304350018501282,
0.19891692698001862,
0.07254264503717422,
0.06347434967756271,
-0.31816524267196655
],
[
0.09148014336824417,
-0.1973782479763031,
0.152879536151886,
0.11245211213827133,
0.12135685235261917,
0.12074458599090576,
0.3753202259540558,
0.044213682413101196
],
[
0.07850243151187897,
0.17286451160907745,
0.8361716866493225,
0.4728657901287079,
0.4632956087589264,
0.20232141017913818,
0.5647081732749939,
0.14378811419010162
],
[
-0.08318367600440979,
0.05281839147210121,
-0.0541967898607254,
-0.07725174725055695,
-0.19812145829200745,
-0.0031219127122312784,
-0.04434914141893387,
-0.08314008265733719
],
[
0.21213339269161224,
0.07709750533103943,
0.045971959829330444,
0.21323992311954498,
0.025503091514110565,
0.26300355792045593,
0.12342528253793716,
0.12137588858604431
],
[
0.09544969350099564,
0.0986594632267952,
0.04683060944080353,
0.12203476577997208,
0.31474506855010986,
0.005084820557385683,
0.022342870011925697,
-0.03607412800192833
],
[
0.5112781524658203,
0.1685691624879837,
0.03565533459186554,
-0.017969932407140732,
-0.11261212080717087,
0.21835851669311523,
0.06214938685297966,
0.369602233171463
],
[
-0.40601295232772827,
0.12985582649707794,
-0.03226114436984062,
0.0588277243077755,
0.1966109573841095,
-0.195071741938591,
0.021230820566415787,
-0.3407396376132965
],
[
0.05574870482087135,
0.002938202116638422,
0.8687096834182739,
0.05353480949997902,
0.06125132739543915,
0.1233755350112915,
0.607818603515625,
0.06634242087602615
],
[
0.17747589945793152,
-0.11643446981906891,
-0.4716564118862152,
0.21365293860435486,
-0.053643301129341125,
0.035281628370285034,
-0.4258069097995758,
0.28920620679855347
],
[
-0.07137177139520645,
-0.0010721894213929772,
-0.30434563755989075,
-0.04535682126879692,
0.11665429919958115,
-0.21108229458332062,
-0.05042998865246773,
-0.023268967866897583
],
[
0.20552901923656464,
0.10354588180780411,
0.18191908299922943,
0.15948699414730072,
0.154840886592865,
0.581689715385437,
0.12202811241149902,
-0.10293511301279068
],
[
0.015075353905558586,
0.04981930926442146,
0.5468234419822693,
0.05587880685925484,
-0.008960379287600517,
0.026398485526442528,
0.831177294254303,
0.0010817323345690966
],
[
0.2027735859155655,
0.22851699590682983,
0.04834882915019989,
-0.009833958931267262,
0.15699052810668945,
0.20191335678100586,
-0.24067029356956482,
0.5740934014320374
]
]
}
+260
View File
@@ -0,0 +1,260 @@
{
"calibration": [
{
"id": "calibration-000",
"prompt": "Use the lookup table. Answer with only the option number.\nbirch: 19\nfern: 45\ndune: 31\nlarch: 86\nWhich option is the value for birch?\n1. 19\n2. 31\n3. 45\n4. 86\nAnswer:",
"answer_index": 0,
"evidence_position": 0,
"entries": 4
},
{
"id": "calibration-001",
"prompt": "Use the lookup table. Answer with only the option number.\nnettle: 34\npine: 47\nmaple: 93\nkelp: 83\nWhich option is the value for pine?\n1. 34\n2. 93\n3. 47\n4. 83\nAnswer:",
"answer_index": 2,
"evidence_position": 1,
"entries": 4
},
{
"id": "calibration-002",
"prompt": "Use the lookup table. Answer with only the option number.\nfern: 93\ncedar: 38\niris: 53\nnettle: 61\nWhich option is the value for iris?\n1. 61\n2. 53\n3. 93\n4. 38\nAnswer:",
"answer_index": 1,
"evidence_position": 2,
"entries": 4
},
{
"id": "calibration-003",
"prompt": "Use the lookup table. Answer with only the option number.\nfern: 19\nelm: 52\ngrove: 13\nolive: 39\nWhich option is the value for olive?\n1. 13\n2. 19\n3. 52\n4. 39\nAnswer:",
"answer_index": 3,
"evidence_position": 3,
"entries": 4
},
{
"id": "calibration-004",
"prompt": "Use the lookup table. Answer with only the option number.\nlarch: 24\ndune: 97\npine: 13\nolive: 57\nWhich option is the value for larch?\n1. 13\n2. 57\n3. 97\n4. 24\nAnswer:",
"answer_index": 3,
"evidence_position": 0,
"entries": 4
},
{
"id": "calibration-005",
"prompt": "Use the lookup table. Answer with only the option number.\nfern: 90\nmaple: 72\nbirch: 81\nolive: 65\nWhich option is the value for maple?\n1. 65\n2. 72\n3. 81\n4. 90\nAnswer:",
"answer_index": 1,
"evidence_position": 1,
"entries": 4
},
{
"id": "calibration-006",
"prompt": "Use the lookup table. Answer with only the option number.\nlarch: 19\nbirch: 40\ndune: 14\namber: 53\nWhich option is the value for dune?\n1. 14\n2. 40\n3. 53\n4. 19\nAnswer:",
"answer_index": 0,
"evidence_position": 2,
"entries": 4
},
{
"id": "calibration-007",
"prompt": "Use the lookup table. Answer with only the option number.\nbirch: 47\nkelp: 15\niris: 95\nmaple: 25\nWhich option is the value for maple?\n1. 95\n2. 47\n3. 15\n4. 25\nAnswer:",
"answer_index": 3,
"evidence_position": 3,
"entries": 4
},
{
"id": "calibration-008",
"prompt": "Use the lookup table. Answer with only the option number.\nelm: 78\niris: 49\njade: 61\npine: 46\nWhich option is the value for elm?\n1. 46\n2. 61\n3. 78\n4. 49\nAnswer:",
"answer_index": 2,
"evidence_position": 0,
"entries": 4
},
{
"id": "calibration-009",
"prompt": "Use the lookup table. Answer with only the option number.\nbirch: 51\niris: 80\ncedar: 81\nfern: 54\nWhich option is the value for iris?\n1. 80\n2. 81\n3. 54\n4. 51\nAnswer:",
"answer_index": 0,
"evidence_position": 1,
"entries": 4
},
{
"id": "calibration-010",
"prompt": "Use the lookup table. Answer with only the option number.\nheath: 69\ncedar: 80\nbirch: 37\ndune: 19\nWhich option is the value for birch?\n1. 69\n2. 80\n3. 37\n4. 19\nAnswer:",
"answer_index": 2,
"evidence_position": 2,
"entries": 4
},
{
"id": "calibration-011",
"prompt": "Use the lookup table. Answer with only the option number.\nbirch: 89\njade: 86\nfern: 65\ncedar: 20\nWhich option is the value for cedar?\n1. 65\n2. 20\n3. 89\n4. 86\nAnswer:",
"answer_index": 1,
"evidence_position": 3,
"entries": 4
}
],
"test4": [
{
"id": "test4-000",
"prompt": "Use the lookup table. Answer with only the option number.\ngrove: 25\namber: 97\ndune: 93\nheath: 80\nWhich option is the value for grove?\n1. 97\n2. 80\n3. 25\n4. 93\nAnswer:",
"answer_index": 2,
"evidence_position": 0,
"entries": 4
},
{
"id": "test4-001",
"prompt": "Use the lookup table. Answer with only the option number.\nkelp: 50\nelm: 92\ngrove: 27\nnettle: 46\nWhich option is the value for elm?\n1. 27\n2. 50\n3. 92\n4. 46\nAnswer:",
"answer_index": 2,
"evidence_position": 1,
"entries": 4
},
{
"id": "test4-002",
"prompt": "Use the lookup table. Answer with only the option number.\nmaple: 64\nolive: 99\nbirch: 95\nelm: 31\nWhich option is the value for birch?\n1. 64\n2. 31\n3. 99\n4. 95\nAnswer:",
"answer_index": 3,
"evidence_position": 2,
"entries": 4
},
{
"id": "test4-003",
"prompt": "Use the lookup table. Answer with only the option number.\nkelp: 95\nmaple: 88\niris: 77\nlarch: 57\nWhich option is the value for larch?\n1. 77\n2. 57\n3. 95\n4. 88\nAnswer:",
"answer_index": 1,
"evidence_position": 3,
"entries": 4
},
{
"id": "test4-004",
"prompt": "Use the lookup table. Answer with only the option number.\ndune: 51\ncedar: 55\nlarch: 98\nnettle: 94\nWhich option is the value for dune?\n1. 51\n2. 94\n3. 55\n4. 98\nAnswer:",
"answer_index": 0,
"evidence_position": 0,
"entries": 4
},
{
"id": "test4-005",
"prompt": "Use the lookup table. Answer with only the option number.\nelm: 42\nolive: 46\nheath: 11\ndune: 54\nWhich option is the value for olive?\n1. 11\n2. 46\n3. 42\n4. 54\nAnswer:",
"answer_index": 1,
"evidence_position": 1,
"entries": 4
},
{
"id": "test4-006",
"prompt": "Use the lookup table. Answer with only the option number.\nmaple: 94\ndune: 40\npine: 76\njade: 10\nWhich option is the value for pine?\n1. 10\n2. 94\n3. 76\n4. 40\nAnswer:",
"answer_index": 2,
"evidence_position": 2,
"entries": 4
},
{
"id": "test4-007",
"prompt": "Use the lookup table. Answer with only the option number.\nkelp: 54\niris: 92\npine: 11\ncedar: 13\nWhich option is the value for cedar?\n1. 13\n2. 11\n3. 54\n4. 92\nAnswer:",
"answer_index": 0,
"evidence_position": 3,
"entries": 4
},
{
"id": "test4-008",
"prompt": "Use the lookup table. Answer with only the option number.\nlarch: 40\nelm: 91\nbirch: 30\npine: 78\nWhich option is the value for larch?\n1. 30\n2. 40\n3. 78\n4. 91\nAnswer:",
"answer_index": 1,
"evidence_position": 0,
"entries": 4
},
{
"id": "test4-009",
"prompt": "Use the lookup table. Answer with only the option number.\njade: 66\npine: 45\nkelp: 46\nheath: 52\nWhich option is the value for pine?\n1. 66\n2. 52\n3. 46\n4. 45\nAnswer:",
"answer_index": 3,
"evidence_position": 1,
"entries": 4
},
{
"id": "test4-010",
"prompt": "Use the lookup table. Answer with only the option number.\namber: 93\ndune: 32\nheath: 72\njade: 46\nWhich option is the value for heath?\n1. 32\n2. 46\n3. 93\n4. 72\nAnswer:",
"answer_index": 3,
"evidence_position": 2,
"entries": 4
},
{
"id": "test4-011",
"prompt": "Use the lookup table. Answer with only the option number.\nnettle: 99\nfern: 36\nbirch: 49\nolive: 25\nWhich option is the value for olive?\n1. 25\n2. 99\n3. 49\n4. 36\nAnswer:",
"answer_index": 0,
"evidence_position": 3,
"entries": 4
}
],
"test8": [
{
"id": "test8-000",
"prompt": "Use the lookup table. Answer with only the option number.\ncedar: 53\ndune: 77\nelm: 44\nmaple: 93\nlarch: 39\nheath: 78\ngrove: 41\nbirch: 48\nWhich option is the value for cedar?\n1. 77\n2. 48\n3. 53\n4. 41\nAnswer:",
"answer_index": 2,
"evidence_position": 0,
"entries": 8
},
{
"id": "test8-001",
"prompt": "Use the lookup table. Answer with only the option number.\nolive: 87\nnettle: 76\nbirch: 44\nmaple: 69\npine: 41\ncedar: 56\ndune: 45\nkelp: 94\nWhich option is the value for nettle?\n1. 76\n2. 56\n3. 41\n4. 94\nAnswer:",
"answer_index": 0,
"evidence_position": 1,
"entries": 8
},
{
"id": "test8-002",
"prompt": "Use the lookup table. Answer with only the option number.\nmaple: 80\nbirch: 21\nfern: 75\ngrove: 52\ncedar: 59\niris: 73\njade: 32\npine: 83\nWhich option is the value for fern?\n1. 32\n2. 80\n3. 75\n4. 73\nAnswer:",
"answer_index": 2,
"evidence_position": 2,
"entries": 8
},
{
"id": "test8-003",
"prompt": "Use the lookup table. Answer with only the option number.\nlarch: 91\nbirch: 39\nheath: 27\npine: 96\nkelp: 49\nolive: 85\nelm: 18\nmaple: 66\nWhich option is the value for pine?\n1. 66\n2. 96\n3. 49\n4. 18\nAnswer:",
"answer_index": 1,
"evidence_position": 3,
"entries": 8
},
{
"id": "test8-004",
"prompt": "Use the lookup table. Answer with only the option number.\ncedar: 19\nbirch: 84\nkelp: 85\njade: 72\nnettle: 87\niris: 96\nmaple: 56\nheath: 95\nWhich option is the value for nettle?\n1. 85\n2. 87\n3. 95\n4. 19\nAnswer:",
"answer_index": 1,
"evidence_position": 4,
"entries": 8
},
{
"id": "test8-005",
"prompt": "Use the lookup table. Answer with only the option number.\niris: 36\nnettle: 41\nfern: 96\nmaple: 84\nheath: 37\npine: 57\ndune: 17\nbirch: 46\nWhich option is the value for pine?\n1. 17\n2. 37\n3. 41\n4. 57\nAnswer:",
"answer_index": 3,
"evidence_position": 5,
"entries": 8
},
{
"id": "test8-006",
"prompt": "Use the lookup table. Answer with only the option number.\nfern: 49\njade: 67\nnettle: 71\nmaple: 89\nbirch: 44\npine: 25\nkelp: 27\ngrove: 41\nWhich option is the value for kelp?\n1. 25\n2. 27\n3. 44\n4. 41\nAnswer:",
"answer_index": 1,
"evidence_position": 6,
"entries": 8
},
{
"id": "test8-007",
"prompt": "Use the lookup table. Answer with only the option number.\nnettle: 71\nolive: 95\nmaple: 16\niris: 54\njade: 61\namber: 72\nfern: 38\ngrove: 13\nWhich option is the value for grove?\n1. 72\n2. 38\n3. 13\n4. 95\nAnswer:",
"answer_index": 2,
"evidence_position": 7,
"entries": 8
},
{
"id": "test8-008",
"prompt": "Use the lookup table. Answer with only the option number.\njade: 98\nkelp: 18\nelm: 84\niris: 67\nbirch: 64\nfern: 10\nlarch: 79\nmaple: 76\nWhich option is the value for jade?\n1. 10\n2. 67\n3. 84\n4. 98\nAnswer:",
"answer_index": 3,
"evidence_position": 0,
"entries": 8
},
{
"id": "test8-009",
"prompt": "Use the lookup table. Answer with only the option number.\ngrove: 66\nheath: 58\nfern: 59\nmaple: 97\nolive: 52\namber: 29\nkelp: 24\ndune: 67\nWhich option is the value for heath?\n1. 58\n2. 59\n3. 67\n4. 52\nAnswer:",
"answer_index": 0,
"evidence_position": 1,
"entries": 8
},
{
"id": "test8-010",
"prompt": "Use the lookup table. Answer with only the option number.\nbirch: 52\namber: 72\ndune: 80\npine: 79\nfern: 32\nnettle: 14\nkelp: 84\nelm: 59\nWhich option is the value for dune?\n1. 52\n2. 84\n3. 72\n4. 80\nAnswer:",
"answer_index": 3,
"evidence_position": 2,
"entries": 8
},
{
"id": "test8-011",
"prompt": "Use the lookup table. Answer with only the option number.\nfern: 26\ngrove: 47\nbirch: 58\ndune: 99\nmaple: 24\ncedar: 19\njade: 36\nkelp: 23\nWhich option is the value for dune?\n1. 99\n2. 36\n3. 23\n4. 58\nAnswer:",
"answer_index": 0,
"evidence_position": 3,
"entries": 8
}
]
}
@@ -0,0 +1,20 @@
{
"donor": 19,
"target": 23,
"local_coordinate_count": 32,
"modes": [
"native",
"direct",
"permutation",
"mix",
"local32",
"mix_local32"
],
"source": "/opt/exp-model-parameter-similarity/runs/cpu-qwen35-2b",
"source_sha256": {
"model_config.json": "ed1c1723241f23f7f4e23430759cbd7dcfb4103cbdfe052bfe7626b57c2615b4",
"tokenizer.json": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42",
"manifest.json": "28efbaf94a5034333339a35f08ead151459a7d20c6c2d2f07a451e7f3bf154de"
},
"choice_nll_scope": "normalized over four option-number logits only"
}
File diff suppressed because it is too large Load Diff
+95
View File
@@ -0,0 +1,95 @@
{
"sets": {
"test4": {
"native": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.1577236725451122,
"mean_margin": 2.6222333908081055,
"prediction_agreement_with_native": 12
},
"direct": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.09467621731363618,
"mean_margin": 3.157541592915853,
"prediction_agreement_with_native": 12
},
"permutation": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.10225362466948813,
"mean_margin": 3.069669246673584,
"prediction_agreement_with_native": 12
},
"mix": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.10946264729409035,
"mean_margin": 2.9876694679260254,
"prediction_agreement_with_native": 12
},
"local32": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.11173090120474012,
"mean_margin": 2.9486522674560547,
"prediction_agreement_with_native": 12
},
"mix_local32": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.1155495746250453,
"mean_margin": 2.9375176429748535,
"prediction_agreement_with_native": 12
}
},
"test8": {
"native": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.13243424793047875,
"mean_margin": 2.652334133783976,
"prediction_agreement_with_native": 12
},
"direct": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.07933497390150783,
"mean_margin": 3.1618269284566245,
"prediction_agreement_with_native": 12
},
"permutation": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.08440917052410739,
"mean_margin": 3.135169267654419,
"prediction_agreement_with_native": 12
},
"mix": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.09210604798693496,
"mean_margin": 3.0242624282836914,
"prediction_agreement_with_native": 12
},
"local32": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.09402417229313187,
"mean_margin": 2.9872288703918457,
"prediction_agreement_with_native": 12
},
"mix_local32": {
"n": 12,
"correct": 12,
"mean_choice_nll": 0.09681914153941336,
"mean_margin": 2.970529397328695,
"prediction_agreement_with_native": 12
}
}
},
"native_last_layer_replay_parity": true,
"calibration_reused": true,
"seconds": 681.1676366906613
}