From 31a257ba174b43c83385d7f9d3222c33ca868171 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sun, 22 Nov 2026 20:20:25 +0800 Subject: [PATCH] Evaluate training-free attention reuse and LiSA-style calibration --- crossqkv/README.md | 26 + crossqkv/lisa_pilot.py | 265 +++ results/lisa-attention-pilot/REPORT.md | 23 + results/lisa-attention-pilot/calibration.json | 338 +++ results/lisa-attention-pilot/problems.json | 260 +++ results/lisa-attention-pilot/protocol.json | 20 + results/lisa-attention-pilot/results.json | 2046 +++++++++++++++++ results/lisa-attention-pilot/summary.json | 95 + 8 files changed, 3073 insertions(+) create mode 100644 crossqkv/lisa_pilot.py create mode 100644 results/lisa-attention-pilot/REPORT.md create mode 100644 results/lisa-attention-pilot/calibration.json create mode 100644 results/lisa-attention-pilot/problems.json create mode 100644 results/lisa-attention-pilot/protocol.json create mode 100644 results/lisa-attention-pilot/results.json create mode 100644 results/lisa-attention-pilot/summary.json diff --git a/crossqkv/README.md b/crossqkv/README.md index 5141a61..b3a79db 100644 --- a/crossqkv/README.md +++ b/crossqkv/README.md @@ -40,3 +40,29 @@ All weights and caches stay on the server until explicit user deletion. Git cont all figure tables and the selected small probability matrices; large caches have exact paths/sizes/hashes in `external-caches.json`. Old validation texts were used in earlier experiments; the four fresh texts were fixed before the new cross-layer measurements. + +## LiSA-style single-layer pilot + +The pilot reuses layer 19's actual pre-softmax scores at layer 23, with the +recipient's own values, output gate and output projection. It compares native +attention, direct reuse, a frozen one-to-one head assignment, linear mixing +of donor heads, a 32-coordinate recipient-only control, and joint donor/recipient +mixing. The linear variants use a closed-form ridge fit to the native scores, +centred within each causal key row. The recipient branch takes a fixed subset of +post-RoPE coordinates rather than learning a low-rank Q/K projection. The +calibration uses 12 synthetic prompts without answer labels. All checkpoint +weights are frozen. + +Evaluation scores the four candidate option tokens on 24 disjoint retrieval +prompts, including 12 with more table entries than calibration. Choice NLL +normalizes over those four tokens. Run against the retained checkpoint on the +remote CPU environment: + +```sh +OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 venv/bin/python -B -m crossqkv.lisa_pilot \ + --source runs/cpu-qwen35-2b --out runs/NEW_LISA_DIRECTORY +``` + +The diagnostic forward still computes native full Q/K and does not measure +latency or parameter savings. Refer to LiSA (arXiv:2408.01890, §5.1) for its +learned nonlinear head aligner and learned low-rank Q/K projections. diff --git a/crossqkv/lisa_pilot.py b/crossqkv/lisa_pilot.py new file mode 100644 index 0000000..340d1b4 --- /dev/null +++ b/crossqkv/lisa_pilot.py @@ -0,0 +1,265 @@ +"""Single-site LiSA-style pilot on the retained Qwen3.5-2B checkpoint.""" +import argparse +import hashlib +import json +from pathlib import Path +import random +import time + +import numpy as np +from tokenizers import Tokenizer + +from crossqkv.core import QKVModel, assignment +from functional.model import PREFIX +from functional.run import write_json +from pilot.tensors import decode + +DONOR, TARGET = 19, 23 +WIDTH = 32 +MODES = ('native', 'direct', 'permutation', 'mix', 'local32', 'mix_local32') + + +def scores(q, k): + keys = np.repeat(k, q.shape[2] // k.shape[2], axis=2) + return np.einsum('bthd,bshd->bhts', q, keys) / np.float32(np.sqrt(q.shape[-1])) + + +def probabilities(x): + x = np.where(np.tri(x.shape[-1], dtype=bool), x, -np.inf) + exp = np.exp(x - x.max(axis=-1, keepdims=True)) + return exp / exp.sum(axis=-1, keepdims=True) + + +def centered_rows(x, lengths): + values = [] + for i, length in enumerate(lengths): + query, key = np.tril_indices(length) + board = x[i, :, :length, :length] + mean = np.where(np.tri(length, dtype=bool), board, 0).sum(-1) / np.arange(1, length + 1) + centered = board - mean[..., None] + values.append(centered[:, query, key].T) + return np.concatenate(values).astype(np.float64) + + +def solve(x, y): + gram = x.T @ x + ridge = 1e-3 * np.trace(gram) / gram.shape[0] + return np.linalg.solve(gram + max(ridge, 1e-12) * np.eye(len(gram)), x.T @ y).astype(np.float32) + + +def fit(donor, target, local): + cost = np.mean((target[:, :, None] - donor[:, None, :]) ** 2, axis=0) + perm = assignment(cost) + return { + 'permutation': list(perm), + 'mix': solve(donor, target), + 'local32': solve(local, target), + 'mix_local32': solve(np.concatenate((donor, local), axis=1), target), + } + + +def predict(donor, local, mode, mapping): + if mode == 'direct': + return donor + if mode == 'permutation': + return donor[:, mapping[mode]] + if mode == 'mix': + inputs = donor + elif mode == 'local32': + inputs = local + else: + inputs = np.concatenate((donor, local), axis=1) + return np.einsum('bhts,hi->bits', inputs, mapping[mode]) + + +class Study(QKVModel): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + dim = self.config['head_dim'] + if dim % WIDTH: + raise ValueError('Expected head dimension divisible by fixed local width') + # Deterministic channel sample in post-RoPE Q/K coordinates. + self.coordinates = np.arange(0, dim, dim // WIDTH) + self.capture_scores = False + + def normalization(self, x, name): + if self.capture_scores and name == f'layers.{TARGET}.input_layernorm': + self.recipient_input = x + return super().normalization(x, name) + + def full_attention(self, h, layer, shared_p=None): + if not self.capture_scores or layer not in (DONOR, TARGET): + return super().full_attention(h, layer, shared_p) + parts = self.parts(h, layer, heads=False) + if layer == DONOR: + self.donor = scores(parts['q'], parts['k']) + else: + q, k = parts['q'], parts['k'] + self.teacher = scores(q, k) + self.local = scores(q[..., self.coordinates], k[..., self.coordinates]) + self.recipient = parts + return parts['output'], parts['p'] + + def replay(self, modes): + if TARGET != len(self.config['layer_types']) - 1: + raise ValueError('Replay requires changing only the final layer') + parts = self.recipient + repeats = self.config['num_attention_heads'] // self.config['num_key_value_heads'] + values = np.repeat(parts['v'], repeats, axis=2) + contexts = [np.einsum('bhts,bshd->bthd', + probabilities(predict(self.donor, self.local, mode, self.mapping)), + values) for mode in modes] + context = np.concatenate([parts['context'], *contexts]) + gate = np.concatenate([parts['gate']] * (1 + len(modes))) + residual = np.concatenate([self.recipient_input] * (1 + len(modes))) + hidden = residual + self.project(context, gate, TARGET) + mlp_input = self.normalization(hidden, f'layers.{TARGET}.post_attention_layernorm') + return np.split(self.normalization(hidden + self.mlp(mlp_input, TARGET), 'norm'), + 1 + len(modes)) + + +def dataset(split, count): + seed = {'calibration': 2026112201, 'test4': 2026112202, 'test8': 2026112203}[split] + rng = random.Random(seed) + size = 8 if split == 'test8' else 4 + words = ('amber', 'birch', 'cedar', 'dune', 'elm', 'fern', 'grove', 'heath', + 'iris', 'jade', 'kelp', 'larch', 'maple', 'nettle', 'olive', 'pine') + examples = [] + answers = [i % 4 for i in range(count)] + rng.shuffle(answers) + for i, answer in enumerate(answers): + keys = rng.sample(words, size) + values = rng.sample(range(10, 100), size) + index = i % size + other = rng.sample([v for j, v in enumerate(values) if j != index], 3) + other.insert(answer, values[index]) + items = '\n'.join(f'{k}: {v}' for k, v in zip(keys, values)) + options = '\n'.join(f'{j + 1}. {v}' for j, v in enumerate(other)) + prompt = (f'Use the lookup table. Answer with only the option number.\n{items}\n' + f'Which option is the value for {keys[index]}?\n{options}\nAnswer:') + examples.append({'id': f'{split}-{i:03}', 'prompt': prompt, 'answer_index': answer, + 'evidence_position': index, 'entries': size}) + return examples + + +def batch(tokenizer, examples): + encoded = [] + choices = None + for ex in examples: + continuations = [tokenizer.encode(f"{ex['prompt']} {i}", add_special_tokens=False).ids + for i in range(1, 5)] + prefix = continuations[0][:-1] + labels = [row[-1] for row in continuations] + if any(row[:-1] != prefix for row in continuations) or len(set(labels)) != 4: + raise ValueError('Choices must differ only at the final token') + if choices is not None and labels != choices: + raise ValueError('Choice tokenization varies by prompt') + encoded.append(prefix) + choices = labels + lengths = [len(ids) for ids in encoded] + ids = np.array([row + [0] * (max(lengths) - len(row)) for row in encoded], dtype=np.int64) + return ids, lengths, choices + + +def evaluate(lm, tokenizer, examples, batch_size=4): + results = {mode: [] for mode in MODES} + for start in range(0, len(examples), batch_size): + group = examples[start:start + batch_size] + ids, lengths, choices = batch(tokenizer, group) + n = len(group) + lm.capture_scores = True + native, _ = lm.forward(ids) + lm.capture_scores = False + replay_native, *replayed = lm.replay(MODES[1:]) + if not np.allclose(replay_native, native, atol=2e-5, rtol=1e-5): + raise ValueError('Native last-layer replay parity failed') + token_rows = lm.tensors[PREFIX + 'embed_tokens.weight'] + weight = decode(token_rows.raw.reshape(token_rows.shape)[choices], token_rows.dtype) + token_rows.evict() + for mode, hidden in zip(MODES, [native, *replayed]): + logits = hidden[np.arange(n), np.array(lengths) - 1] @ weight.T + for ex, row, length in zip(group, logits.astype(float), lengths): + gold = ex['answer_index'] + results[mode].append({ + 'id': ex['id'], 'length': length, 'gold': gold, + 'logits': row.tolist(), 'predicted': int(row.argmax()), + 'choice_nll': float(np.logaddexp.reduce(row) - row[gold]), + 'margin': float(row[gold] - np.max(np.delete(row, gold))), + }) + print(f"{group[-1]['id']}: evaluated {n} prompts in all six modes", flush=True) + return results + + +def summarize(results): + return {split: {mode: { + 'n': len(rows), + 'correct': sum(row['gold'] == row['predicted'] for row in rows), + 'mean_choice_nll': float(np.mean([row['choice_nll'] for row in rows])), + 'mean_margin': float(np.mean([row['margin'] for row in rows])), + 'prediction_agreement_with_native': sum(row['predicted'] == base['predicted'] + for row, base in zip(rows, modes['native'])), + } for mode, rows in modes.items()} for split, modes in results.items()} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--source', type=Path, required=True) + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--calibration-run', type=Path, help='Reuse a matching completed calibration') + parser.add_argument('--seconds', type=int, default=1800) + args = parser.parse_args() + start = time.monotonic() + args.out.mkdir() + source = args.source + config = json.loads((source / 'model_config.json').read_text()) + tokenizer = Tokenizer.from_file(str(source / 'tokenizer.json')) + lm = Study(source / 'weights', config, seconds=args.seconds) + kinds = lm.config['layer_types'] + if any(kinds[i] != 'full_attention' for i in (DONOR, TARGET)): + raise ValueError('Requires the retained architecture with full attention at 19 and 23') + source_files = ['model_config.json', 'tokenizer.json', 'manifest.json'] + protocol = {'donor': DONOR, 'target': TARGET, 'local_coordinate_count': WIDTH, + 'modes': MODES, 'source': str(source), + 'source_sha256': {f: hashlib.sha256((source/f).read_bytes()).hexdigest() for f in source_files}, + 'choice_nll_scope': 'normalized over four option-number logits only'} + problems = {s: dataset(s, n) for s, n in [('calibration', 12), ('test4', 12), ('test8', 12)]} + if len({e['prompt'] for entries in problems.values() for e in entries}) != 36: + raise ValueError('Repeated prompt across pilot sample') + write_json(args.out / 'protocol.json', protocol) + write_json(args.out / 'problems.json', problems) + + if args.calibration_run: + previous = args.calibration_run + if (json.loads((previous / 'protocol.json').read_text()) != + json.loads((args.out / 'protocol.json').read_text()) or + json.loads((previous / 'problems.json').read_text()) != problems): + raise ValueError('Calibration inputs or checkpoint provenance differ') + lm.mapping = {k: np.asarray(v, dtype=np.int64 if k == 'permutation' else np.float32) + for k, v in json.loads((previous / 'calibration.json').read_text()).items()} + else: + ids, lengths, _ = batch(tokenizer, problems['calibration']) + lm.capture_scores = True + native, _ = lm.forward(ids) + measured = [centered_rows(x, lengths) for x in (lm.donor, lm.teacher, lm.local)] + lm.capture_scores = False + reference, _ = lm.forward(ids[:4]) + if not np.allclose(native[:4], reference, atol=1e-5, rtol=1e-5): + raise ValueError('Capture path disagrees with the unmodified reference') + lm.mapping = fit(*measured) + print('Calibration ready', flush=True) + write_json(args.out / 'calibration.json', { + k: np.asarray(v).tolist() for k, v in lm.mapping.items() + }) + results = {} + for split in ('test4', 'test8'): + results[split] = evaluate(lm, tokenizer, problems[split]) + print(split, summarize(results)[split], flush=True) + write_json(args.out / 'results.json', results) + write_json(args.out / 'summary.json', {'sets': summarize(results), + 'native_last_layer_replay_parity': True, + 'calibration_reused': bool(args.calibration_run), + 'seconds': time.monotonic() - start}) + + +if __name__ == '__main__': + main() diff --git a/results/lisa-attention-pilot/REPORT.md b/results/lisa-attention-pilot/REPORT.md new file mode 100644 index 0000000..629e64b --- /dev/null +++ b/results/lisa-attention-pilot/REPORT.md @@ -0,0 +1,23 @@ +# Qwen3.5-2B 跨层注意力共享先导实验 + +固定原 Qwen3.5-2B 检查点,测试第 19 层实际前向产生的 softmax 前注意力分数能否替代第 23 层的分数。层编号从零开始。所有替换保留接收端的 V、gate 和输出投影。 + +| 方法 | 4 项表正确 | 8 项表正确 | 合计选择 NLL | +| --- | ---: | ---: | ---: | +| 原模型 | 12/12 | 12/12 | 0.145079 | +| 原序直接共享 | 12/12 | 12/12 | 0.087006 | +| 头重排后共享 | 12/12 | 12/12 | 0.093331 | +| 共享分数的线性头混合 | 12/12 | 12/12 | 0.100784 | +| 接收层 32 坐标,无共享 | 12/12 | 12/12 | 0.102878 | +| 共享分数与接收层 32 坐标联合拟合 | 12/12 | 12/12 | 0.106184 | + +选择 NLL 在四个选项 token 上归一化后按题取均值。直接共享在这 24 道题上保持准确率,也在每道题上降低了选择 NLL。更低的损失反映正确选项相对其余三个选项的得分,样本中的准确率已经达到上限。这个结果支持冻结权重的单层替换在这些合成检索题上可用。轻量拟合没有带来额外收益。 + +## 实验与检查 + +- 12 道 4 项表校准题用于冻结头排列、闭式岭回归映射。拟合只使用原模型内部注意力分数,不使用正确选项标签。全套 36 条提示词无重复。 +- Qwen 权重全程冻结。没有反向传播,也没有训练 LiSA 原版的非线性头对齐网络或低秩 Q/K 投影。32 坐标来自确定性下采样的接收层 post-RoPE Q/K。 +- 远程 CPU 的 8 项跨层回归测试通过。合成数值检查通过,覆盖排列方向、因果遮罩和联合拟合重建。校准时的原模型路径一致性检查以及六批留出题的末层原样重放一致性检查均通过。 +- 两次较慢的远程执行切换后保留旧目录,最终版本沿用第二次已完成的校准,并核对原检查点文件指纹及全部提示词。末层重放阶段耗时约 681 秒。逐题分数、答案和 NLL 已从原始 logits 独立复核,最终运行脚本 SHA256 为 `a64d1b50a10bdde893c4bc274365c0c072f13c0a2359b079cfa8421b9ce068f4`。 + +评测只覆盖一个模型、一个接收层、一类合成任务、约束于四个选项的读出。诊断仍计算原始 Q/K,未测吞吐、压缩率、自由生成或多层累积替换效果。实验方法和复现入口见 [crossqkv/README.md](../../crossqkv/README.md)。 diff --git a/results/lisa-attention-pilot/calibration.json b/results/lisa-attention-pilot/calibration.json new file mode 100644 index 0000000..9c3d4e8 --- /dev/null +++ b/results/lisa-attention-pilot/calibration.json @@ -0,0 +1,338 @@ +{ + "permutation": [ + 0, + 2, + 4, + 3, + 7, + 6, + 5, + 1 + ], + "mix": [ + [ + 0.24426038563251495, + 0.055573295801877975, + -0.12299608439207077, + -0.09282179176807404, + -0.047870442271232605, + 0.07772502303123474, + -0.22192631661891937, + 0.29638272523880005 + ], + [ + 0.03558433800935745, + 0.03896130993962288, + -0.18795007467269897, + 0.002625135239213705, + -0.16552278399467468, + -0.023057416081428528, + -0.39929720759391785, + 0.43178120255470276 + ], + [ + 0.1871931552886963, + 0.44749122858047485, + -0.04751807823777199, + 0.04143448546528816, + 0.23914210498332977, + 0.18653616309165955, + 0.12161331623792648, + -0.20545999705791473 + ], + [ + -0.027203943580389023, + -0.23345178365707397, + -0.03906247019767761, + 0.0939483568072319, + 0.11926761269569397, + -0.08872664719820023, + 0.14460818469524384, + 0.06796323508024216 + ], + [ + 0.09347549825906754, + 0.2144748717546463, + 1.078420877456665, + 0.5173454880714417, + 0.5393254160881042, + 0.31546834111213684, + 0.7830604314804077, + 0.09864718466997147 + ], + [ + -0.015639908611774445, + 0.13545557856559753, + 0.20001526176929474, + -0.0403873436152935, + -0.14243841171264648, + 0.10669009387493134, + 0.22260507941246033, + -0.008936765603721142 + ], + [ + 0.2405557930469513, + 0.054704487323760986, + 0.011206595227122307, + 0.2406461238861084, + -0.026383306831121445, + 0.25195246934890747, + 0.09300804138183594, + 0.10066510736942291 + ], + [ + 0.02644052356481552, + 0.10034456104040146, + 0.07825803011655807, + 0.10650064051151276, + 0.3668587803840637, + -0.007412006612867117, + 0.061572253704071045, + -0.06577901542186737 + ] + ], + "local32": [ + [ + 0.2867194414138794, + 0.06671284139156342, + -0.2674592435359955, + -0.3276604115962982, + -0.4070841670036316, + -0.006673475727438927, + -0.2074313908815384, + 0.12887436151504517 + ], + [ + -0.0516749806702137, + 0.4364052712917328, + 0.4763493239879608, + 0.569825291633606, + 0.5895766019821167, + 0.23220211267471313, + 0.4529169201850891, + 0.020792270079255104 + ], + [ + 0.030566174536943436, + -0.0014433575561270118, + 0.9291507601737976, + 0.07762160897254944, + 0.09363061934709549, + 0.11929107457399368, + 0.6625307202339172, + 0.011318162083625793 + ], + [ + 0.6921118497848511, + 0.3367474377155304, + -0.20861877501010895, + 0.6061498522758484, + 0.4020693004131317, + 0.4614187479019165, + -0.120954729616642, + 0.6977306008338928 + ], + [ + 0.010575470514595509, + 0.03395608440041542, + -0.17418035864830017, + 0.05686786770820618, + 0.3456997573375702, + -0.16309495270252228, + 0.13770024478435516, + -0.06023705378174782 + ], + [ + 0.34056422114372253, + 0.3800618350505829, + 0.46083998680114746, + 0.379044771194458, + 0.38687247037887573, + 0.7362816333770752, + 0.2947295904159546, + -0.053121648728847504 + ], + [ + 0.03288573399186134, + 0.11208789050579071, + 0.5949627161026001, + 0.09720376133918762, + 0.012026847340166569, + 0.062014296650886536, + 0.8560540080070496, + 0.004886063747107983 + ], + [ + 0.3228500783443451, + 0.3110385239124298, + -0.03597158193588257, + 0.004630092531442642, + 0.1255260556936264, + 0.3001147508621216, + -0.2873961925506592, + 0.7489231824874878 + ] + ], + "mix_local32": [ + [ + 0.11679738759994507, + 0.06494305282831192, + -0.07491274178028107, + -0.10080373287200928, + -0.02167605608701706, + -0.05343199521303177, + -0.1710113286972046, + 0.25583943724632263 + ], + [ + 0.07517975568771362, + 0.04044078290462494, + -0.06897048652172089, + 0.022282306104898453, + -0.14309072494506836, + 0.033187106251716614, + -0.3017081618309021, + 0.44381454586982727 + ], + [ + 0.06857457011938095, + 0.3892330229282379, + -0.10634169727563858, + -0.019304350018501282, + 0.19891692698001862, + 0.07254264503717422, + 0.06347434967756271, + -0.31816524267196655 + ], + [ + 0.09148014336824417, + -0.1973782479763031, + 0.152879536151886, + 0.11245211213827133, + 0.12135685235261917, + 0.12074458599090576, + 0.3753202259540558, + 0.044213682413101196 + ], + [ + 0.07850243151187897, + 0.17286451160907745, + 0.8361716866493225, + 0.4728657901287079, + 0.4632956087589264, + 0.20232141017913818, + 0.5647081732749939, + 0.14378811419010162 + ], + [ + -0.08318367600440979, + 0.05281839147210121, + -0.0541967898607254, + -0.07725174725055695, + -0.19812145829200745, + -0.0031219127122312784, + -0.04434914141893387, + -0.08314008265733719 + ], + [ + 0.21213339269161224, + 0.07709750533103943, + 0.045971959829330444, + 0.21323992311954498, + 0.025503091514110565, + 0.26300355792045593, + 0.12342528253793716, + 0.12137588858604431 + ], + [ + 0.09544969350099564, + 0.0986594632267952, + 0.04683060944080353, + 0.12203476577997208, + 0.31474506855010986, + 0.005084820557385683, + 0.022342870011925697, + -0.03607412800192833 + ], + [ + 0.5112781524658203, + 0.1685691624879837, + 0.03565533459186554, + -0.017969932407140732, + -0.11261212080717087, + 0.21835851669311523, + 0.06214938685297966, + 0.369602233171463 + ], + [ + -0.40601295232772827, + 0.12985582649707794, + -0.03226114436984062, + 0.0588277243077755, + 0.1966109573841095, + -0.195071741938591, + 0.021230820566415787, + -0.3407396376132965 + ], + [ + 0.05574870482087135, + 0.002938202116638422, + 0.8687096834182739, + 0.05353480949997902, + 0.06125132739543915, + 0.1233755350112915, + 0.607818603515625, + 0.06634242087602615 + ], + [ + 0.17747589945793152, + -0.11643446981906891, + -0.4716564118862152, + 0.21365293860435486, + -0.053643301129341125, + 0.035281628370285034, + -0.4258069097995758, + 0.28920620679855347 + ], + [ + -0.07137177139520645, + -0.0010721894213929772, + -0.30434563755989075, + -0.04535682126879692, + 0.11665429919958115, + -0.21108229458332062, + -0.05042998865246773, + -0.023268967866897583 + ], + [ + 0.20552901923656464, + 0.10354588180780411, + 0.18191908299922943, + 0.15948699414730072, + 0.154840886592865, + 0.581689715385437, + 0.12202811241149902, + -0.10293511301279068 + ], + [ + 0.015075353905558586, + 0.04981930926442146, + 0.5468234419822693, + 0.05587880685925484, + -0.008960379287600517, + 0.026398485526442528, + 0.831177294254303, + 0.0010817323345690966 + ], + [ + 0.2027735859155655, + 0.22851699590682983, + 0.04834882915019989, + -0.009833958931267262, + 0.15699052810668945, + 0.20191335678100586, + -0.24067029356956482, + 0.5740934014320374 + ] + ] +} diff --git a/results/lisa-attention-pilot/problems.json b/results/lisa-attention-pilot/problems.json new file mode 100644 index 0000000..13c2d9a --- /dev/null +++ b/results/lisa-attention-pilot/problems.json @@ -0,0 +1,260 @@ +{ + "calibration": [ + { + "id": "calibration-000", + "prompt": "Use the lookup table. Answer with only the option number.\nbirch: 19\nfern: 45\ndune: 31\nlarch: 86\nWhich option is the value for birch?\n1. 19\n2. 31\n3. 45\n4. 86\nAnswer:", + "answer_index": 0, + "evidence_position": 0, + "entries": 4 + }, + { + "id": "calibration-001", + "prompt": "Use the lookup table. Answer with only the option number.\nnettle: 34\npine: 47\nmaple: 93\nkelp: 83\nWhich option is the value for pine?\n1. 34\n2. 93\n3. 47\n4. 83\nAnswer:", + "answer_index": 2, + "evidence_position": 1, + "entries": 4 + }, + { + "id": "calibration-002", + "prompt": "Use the lookup table. Answer with only the option number.\nfern: 93\ncedar: 38\niris: 53\nnettle: 61\nWhich option is the value for iris?\n1. 61\n2. 53\n3. 93\n4. 38\nAnswer:", + "answer_index": 1, + "evidence_position": 2, + "entries": 4 + }, + { + "id": "calibration-003", + "prompt": "Use the lookup table. Answer with only the option number.\nfern: 19\nelm: 52\ngrove: 13\nolive: 39\nWhich option is the value for olive?\n1. 13\n2. 19\n3. 52\n4. 39\nAnswer:", + "answer_index": 3, + "evidence_position": 3, + "entries": 4 + }, + { + "id": "calibration-004", + "prompt": "Use the lookup table. Answer with only the option number.\nlarch: 24\ndune: 97\npine: 13\nolive: 57\nWhich option is the value for larch?\n1. 13\n2. 57\n3. 97\n4. 24\nAnswer:", + "answer_index": 3, + "evidence_position": 0, + "entries": 4 + }, + { + "id": "calibration-005", + "prompt": "Use the lookup table. Answer with only the option number.\nfern: 90\nmaple: 72\nbirch: 81\nolive: 65\nWhich option is the value for maple?\n1. 65\n2. 72\n3. 81\n4. 90\nAnswer:", + "answer_index": 1, + "evidence_position": 1, + "entries": 4 + }, + { + "id": "calibration-006", + "prompt": "Use the lookup table. Answer with only the option number.\nlarch: 19\nbirch: 40\ndune: 14\namber: 53\nWhich option is the value for dune?\n1. 14\n2. 40\n3. 53\n4. 19\nAnswer:", + "answer_index": 0, + "evidence_position": 2, + "entries": 4 + }, + { + "id": "calibration-007", + "prompt": "Use the lookup table. Answer with only the option number.\nbirch: 47\nkelp: 15\niris: 95\nmaple: 25\nWhich option is the value for maple?\n1. 95\n2. 47\n3. 15\n4. 25\nAnswer:", + "answer_index": 3, + "evidence_position": 3, + "entries": 4 + }, + { + "id": "calibration-008", + "prompt": "Use the lookup table. Answer with only the option number.\nelm: 78\niris: 49\njade: 61\npine: 46\nWhich option is the value for elm?\n1. 46\n2. 61\n3. 78\n4. 49\nAnswer:", + "answer_index": 2, + "evidence_position": 0, + "entries": 4 + }, + { + "id": "calibration-009", + "prompt": "Use the lookup table. Answer with only the option number.\nbirch: 51\niris: 80\ncedar: 81\nfern: 54\nWhich option is the value for iris?\n1. 80\n2. 81\n3. 54\n4. 51\nAnswer:", + "answer_index": 0, + "evidence_position": 1, + "entries": 4 + }, + { + "id": "calibration-010", + "prompt": "Use the lookup table. Answer with only the option number.\nheath: 69\ncedar: 80\nbirch: 37\ndune: 19\nWhich option is the value for birch?\n1. 69\n2. 80\n3. 37\n4. 19\nAnswer:", + "answer_index": 2, + "evidence_position": 2, + "entries": 4 + }, + { + "id": "calibration-011", + "prompt": "Use the lookup table. Answer with only the option number.\nbirch: 89\njade: 86\nfern: 65\ncedar: 20\nWhich option is the value for cedar?\n1. 65\n2. 20\n3. 89\n4. 86\nAnswer:", + "answer_index": 1, + "evidence_position": 3, + "entries": 4 + } + ], + "test4": [ + { + "id": "test4-000", + "prompt": "Use the lookup table. Answer with only the option number.\ngrove: 25\namber: 97\ndune: 93\nheath: 80\nWhich option is the value for grove?\n1. 97\n2. 80\n3. 25\n4. 93\nAnswer:", + "answer_index": 2, + "evidence_position": 0, + "entries": 4 + }, + { + "id": "test4-001", + "prompt": "Use the lookup table. Answer with only the option number.\nkelp: 50\nelm: 92\ngrove: 27\nnettle: 46\nWhich option is the value for elm?\n1. 27\n2. 50\n3. 92\n4. 46\nAnswer:", + "answer_index": 2, + "evidence_position": 1, + "entries": 4 + }, + { + "id": "test4-002", + "prompt": "Use the lookup table. Answer with only the option number.\nmaple: 64\nolive: 99\nbirch: 95\nelm: 31\nWhich option is the value for birch?\n1. 64\n2. 31\n3. 99\n4. 95\nAnswer:", + "answer_index": 3, + "evidence_position": 2, + "entries": 4 + }, + { + "id": "test4-003", + "prompt": "Use the lookup table. Answer with only the option number.\nkelp: 95\nmaple: 88\niris: 77\nlarch: 57\nWhich option is the value for larch?\n1. 77\n2. 57\n3. 95\n4. 88\nAnswer:", + "answer_index": 1, + "evidence_position": 3, + "entries": 4 + }, + { + "id": "test4-004", + "prompt": "Use the lookup table. Answer with only the option number.\ndune: 51\ncedar: 55\nlarch: 98\nnettle: 94\nWhich option is the value for dune?\n1. 51\n2. 94\n3. 55\n4. 98\nAnswer:", + "answer_index": 0, + "evidence_position": 0, + "entries": 4 + }, + { + "id": "test4-005", + "prompt": "Use the lookup table. Answer with only the option number.\nelm: 42\nolive: 46\nheath: 11\ndune: 54\nWhich option is the value for olive?\n1. 11\n2. 46\n3. 42\n4. 54\nAnswer:", + "answer_index": 1, + "evidence_position": 1, + "entries": 4 + }, + { + "id": "test4-006", + "prompt": "Use the lookup table. Answer with only the option number.\nmaple: 94\ndune: 40\npine: 76\njade: 10\nWhich option is the value for pine?\n1. 10\n2. 94\n3. 76\n4. 40\nAnswer:", + "answer_index": 2, + "evidence_position": 2, + "entries": 4 + }, + { + "id": "test4-007", + "prompt": "Use the lookup table. Answer with only the option number.\nkelp: 54\niris: 92\npine: 11\ncedar: 13\nWhich option is the value for cedar?\n1. 13\n2. 11\n3. 54\n4. 92\nAnswer:", + "answer_index": 0, + "evidence_position": 3, + "entries": 4 + }, + { + "id": "test4-008", + "prompt": "Use the lookup table. Answer with only the option number.\nlarch: 40\nelm: 91\nbirch: 30\npine: 78\nWhich option is the value for larch?\n1. 30\n2. 40\n3. 78\n4. 91\nAnswer:", + "answer_index": 1, + "evidence_position": 0, + "entries": 4 + }, + { + "id": "test4-009", + "prompt": "Use the lookup table. Answer with only the option number.\njade: 66\npine: 45\nkelp: 46\nheath: 52\nWhich option is the value for pine?\n1. 66\n2. 52\n3. 46\n4. 45\nAnswer:", + "answer_index": 3, + "evidence_position": 1, + "entries": 4 + }, + { + "id": "test4-010", + "prompt": "Use the lookup table. Answer with only the option number.\namber: 93\ndune: 32\nheath: 72\njade: 46\nWhich option is the value for heath?\n1. 32\n2. 46\n3. 93\n4. 72\nAnswer:", + "answer_index": 3, + "evidence_position": 2, + "entries": 4 + }, + { + "id": "test4-011", + "prompt": "Use the lookup table. Answer with only the option number.\nnettle: 99\nfern: 36\nbirch: 49\nolive: 25\nWhich option is the value for olive?\n1. 25\n2. 99\n3. 49\n4. 36\nAnswer:", + "answer_index": 0, + "evidence_position": 3, + "entries": 4 + } + ], + "test8": [ + { + "id": "test8-000", + "prompt": "Use the lookup table. Answer with only the option number.\ncedar: 53\ndune: 77\nelm: 44\nmaple: 93\nlarch: 39\nheath: 78\ngrove: 41\nbirch: 48\nWhich option is the value for cedar?\n1. 77\n2. 48\n3. 53\n4. 41\nAnswer:", + "answer_index": 2, + "evidence_position": 0, + "entries": 8 + }, + { + "id": "test8-001", + "prompt": "Use the lookup table. Answer with only the option number.\nolive: 87\nnettle: 76\nbirch: 44\nmaple: 69\npine: 41\ncedar: 56\ndune: 45\nkelp: 94\nWhich option is the value for nettle?\n1. 76\n2. 56\n3. 41\n4. 94\nAnswer:", + "answer_index": 0, + "evidence_position": 1, + "entries": 8 + }, + { + "id": "test8-002", + "prompt": "Use the lookup table. Answer with only the option number.\nmaple: 80\nbirch: 21\nfern: 75\ngrove: 52\ncedar: 59\niris: 73\njade: 32\npine: 83\nWhich option is the value for fern?\n1. 32\n2. 80\n3. 75\n4. 73\nAnswer:", + "answer_index": 2, + "evidence_position": 2, + "entries": 8 + }, + { + "id": "test8-003", + "prompt": "Use the lookup table. Answer with only the option number.\nlarch: 91\nbirch: 39\nheath: 27\npine: 96\nkelp: 49\nolive: 85\nelm: 18\nmaple: 66\nWhich option is the value for pine?\n1. 66\n2. 96\n3. 49\n4. 18\nAnswer:", + "answer_index": 1, + "evidence_position": 3, + "entries": 8 + }, + { + "id": "test8-004", + "prompt": "Use the lookup table. Answer with only the option number.\ncedar: 19\nbirch: 84\nkelp: 85\njade: 72\nnettle: 87\niris: 96\nmaple: 56\nheath: 95\nWhich option is the value for nettle?\n1. 85\n2. 87\n3. 95\n4. 19\nAnswer:", + "answer_index": 1, + "evidence_position": 4, + "entries": 8 + }, + { + "id": "test8-005", + "prompt": "Use the lookup table. Answer with only the option number.\niris: 36\nnettle: 41\nfern: 96\nmaple: 84\nheath: 37\npine: 57\ndune: 17\nbirch: 46\nWhich option is the value for pine?\n1. 17\n2. 37\n3. 41\n4. 57\nAnswer:", + "answer_index": 3, + "evidence_position": 5, + "entries": 8 + }, + { + "id": "test8-006", + "prompt": "Use the lookup table. Answer with only the option number.\nfern: 49\njade: 67\nnettle: 71\nmaple: 89\nbirch: 44\npine: 25\nkelp: 27\ngrove: 41\nWhich option is the value for kelp?\n1. 25\n2. 27\n3. 44\n4. 41\nAnswer:", + "answer_index": 1, + "evidence_position": 6, + "entries": 8 + }, + { + "id": "test8-007", + "prompt": "Use the lookup table. Answer with only the option number.\nnettle: 71\nolive: 95\nmaple: 16\niris: 54\njade: 61\namber: 72\nfern: 38\ngrove: 13\nWhich option is the value for grove?\n1. 72\n2. 38\n3. 13\n4. 95\nAnswer:", + "answer_index": 2, + "evidence_position": 7, + "entries": 8 + }, + { + "id": "test8-008", + "prompt": "Use the lookup table. Answer with only the option number.\njade: 98\nkelp: 18\nelm: 84\niris: 67\nbirch: 64\nfern: 10\nlarch: 79\nmaple: 76\nWhich option is the value for jade?\n1. 10\n2. 67\n3. 84\n4. 98\nAnswer:", + "answer_index": 3, + "evidence_position": 0, + "entries": 8 + }, + { + "id": "test8-009", + "prompt": "Use the lookup table. Answer with only the option number.\ngrove: 66\nheath: 58\nfern: 59\nmaple: 97\nolive: 52\namber: 29\nkelp: 24\ndune: 67\nWhich option is the value for heath?\n1. 58\n2. 59\n3. 67\n4. 52\nAnswer:", + "answer_index": 0, + "evidence_position": 1, + "entries": 8 + }, + { + "id": "test8-010", + "prompt": "Use the lookup table. Answer with only the option number.\nbirch: 52\namber: 72\ndune: 80\npine: 79\nfern: 32\nnettle: 14\nkelp: 84\nelm: 59\nWhich option is the value for dune?\n1. 52\n2. 84\n3. 72\n4. 80\nAnswer:", + "answer_index": 3, + "evidence_position": 2, + "entries": 8 + }, + { + "id": "test8-011", + "prompt": "Use the lookup table. Answer with only the option number.\nfern: 26\ngrove: 47\nbirch: 58\ndune: 99\nmaple: 24\ncedar: 19\njade: 36\nkelp: 23\nWhich option is the value for dune?\n1. 99\n2. 36\n3. 23\n4. 58\nAnswer:", + "answer_index": 0, + "evidence_position": 3, + "entries": 8 + } + ] +} diff --git a/results/lisa-attention-pilot/protocol.json b/results/lisa-attention-pilot/protocol.json new file mode 100644 index 0000000..a33d25f --- /dev/null +++ b/results/lisa-attention-pilot/protocol.json @@ -0,0 +1,20 @@ +{ + "donor": 19, + "target": 23, + "local_coordinate_count": 32, + "modes": [ + "native", + "direct", + "permutation", + "mix", + "local32", + "mix_local32" + ], + "source": "/opt/exp-model-parameter-similarity/runs/cpu-qwen35-2b", + "source_sha256": { + "model_config.json": "ed1c1723241f23f7f4e23430759cbd7dcfb4103cbdfe052bfe7626b57c2615b4", + "tokenizer.json": "5f9e4d4901a92b997e463c1f46055088b6cca5ca61a6522d1b9f64c4bb81cb42", + "manifest.json": "28efbaf94a5034333339a35f08ead151459a7d20c6c2d2f07a451e7f3bf154de" + }, + "choice_nll_scope": "normalized over four option-number logits only" +} diff --git a/results/lisa-attention-pilot/results.json b/results/lisa-attention-pilot/results.json new file mode 100644 index 0000000..1cae663 --- /dev/null +++ b/results/lisa-attention-pilot/results.json @@ -0,0 +1,2046 @@ +{ + "test4": { + "native": [ + { + "id": "test4-000", + "length": 77, + "gold": 2, + "logits": [ + 16.92061996459961, + 16.922327041625977, + 19.394906997680664, + 15.507345199584961 + ], + "predicted": 2, + "choice_nll": 0.17318426263449993, + "margin": 2.4725799560546875 + }, + { + "id": "test4-001", + "length": 76, + "gold": 2, + "logits": [ + 16.66771697998047, + 17.064931869506836, + 19.260812759399414, + 15.345693588256836 + ], + "predicted": 2, + "choice_nll": 0.1872981830086573, + "margin": 2.195880889892578 + }, + { + "id": "test4-002", + "length": 76, + "gold": 3, + "logits": [ + 16.61077880859375, + 17.0664119720459, + 17.198772430419922, + 19.629444122314453 + ], + "predicted": 3, + "choice_nll": 0.19385051788351504, + "margin": 2.4306716918945312 + }, + { + "id": "test4-003", + "length": 77, + "gold": 1, + "logits": [ + 17.165882110595703, + 19.786636352539062, + 16.527240753173828, + 17.013317108154297 + ], + "predicted": 1, + "choice_nll": 0.1600878245647408, + "margin": 2.6207542419433594 + }, + { + "id": "test4-004", + "length": 78, + "gold": 0, + "logits": [ + 20.242183685302734, + 16.718568801879883, + 15.4391508102417, + 16.694799423217773 + ], + "predicted": 0, + "choice_nll": 0.06437975449530242, + "margin": 3.5236148834228516 + }, + { + "id": "test4-005", + "length": 75, + "gold": 1, + "logits": [ + 16.752986907958984, + 19.551319122314453, + 15.010839462280273, + 17.115896224975586 + ], + "predicted": 1, + "choice_nll": 0.1476788370677511, + "margin": 2.435422897338867 + }, + { + "id": "test4-006", + "length": 75, + "gold": 2, + "logits": [ + 16.664661407470703, + 16.093791961669922, + 19.441322326660156, + 16.71326446533203 + ], + "predicted": 2, + "choice_nll": 0.1507992324956895, + "margin": 2.728057861328125 + }, + { + "id": "test4-007", + "length": 75, + "gold": 0, + "logits": [ + 19.531505584716797, + 15.948738098144531, + 16.08673095703125, + 16.903282165527344 + ], + "predicted": 0, + "choice_nll": 0.1239128692077891, + "margin": 2.628223419189453 + }, + { + "id": "test4-008", + "length": 76, + "gold": 1, + "logits": [ + 17.78899383544922, + 19.541301727294922, + 16.09391212463379, + 17.74379539489746 + ], + "predicted": 1, + "choice_nll": 0.31547735763984264, + "margin": 1.7523078918457031 + }, + { + "id": "test4-009", + "length": 75, + "gold": 3, + "logits": [ + 16.140657424926758, + 16.253543853759766, + 15.751502990722656, + 19.281545639038086 + ], + "predicted": 3, + "choice_nll": 0.11418574322741293, + "margin": 3.0280017852783203 + }, + { + "id": "test4-010", + "length": 76, + "gold": 3, + "logits": [ + 15.820877075195312, + 15.211450576782227, + 15.880693435668945, + 18.885398864746094 + ], + "predicted": 3, + "choice_nll": 0.11476121814442308, + "margin": 3.0047054290771484 + }, + { + "id": "test4-011", + "length": 75, + "gold": 0, + "logits": [ + 19.712730407714844, + 16.96941375732422, + 15.948404312133789, + 17.066150665283203 + ], + "predicted": 0, + "choice_nll": 0.1470682701717223, + "margin": 2.6465797424316406 + } + ], + "direct": [ + { + "id": "test4-000", + "length": 77, + "gold": 2, + "logits": [ + 17.364742279052734, + 17.484119415283203, + 20.596452713012695, + 17.13409423828125 + ], + "predicted": 2, + "choice_nll": 0.10916164787965599, + "margin": 3.112333297729492 + }, + { + "id": "test4-001", + "length": 76, + "gold": 2, + "logits": [ + 16.89239501953125, + 17.523624420166016, + 20.572509765625, + 16.63075065612793 + ], + "predicted": 2, + "choice_nll": 0.0880528697612668, + "margin": 3.0488853454589844 + }, + { + "id": "test4-002", + "length": 76, + "gold": 3, + "logits": [ + 16.964942932128906, + 17.530786514282227, + 18.02515411376953, + 20.928306579589844 + ], + "predicted": 3, + "choice_nll": 0.10192937991611828, + "margin": 2.9031524658203125 + }, + { + "id": "test4-003", + "length": 77, + "gold": 1, + "logits": [ + 17.772186279296875, + 20.95670509338379, + 17.769495010375977, + 17.73946762084961 + ], + "predicted": 1, + "choice_nll": 0.11578160876083032, + "margin": 3.184518814086914 + }, + { + "id": "test4-004", + "length": 78, + "gold": 0, + "logits": [ + 21.163890838623047, + 17.601322174072266, + 16.24397087097168, + 17.23781967163086 + ], + "predicted": 0, + "choice_nll": 0.053907130858892316, + "margin": 3.5625686645507812 + }, + { + "id": "test4-005", + "length": 75, + "gold": 1, + "logits": [ + 17.442947387695312, + 20.876922607421875, + 16.38556671142578, + 17.348365783691406 + ], + "predicted": 1, + "choice_nll": 0.07028242744232926, + "margin": 3.4339752197265625 + }, + { + "id": "test4-006", + "length": 75, + "gold": 2, + "logits": [ + 16.726591110229492, + 16.539100646972656, + 20.73170280456543, + 17.801864624023438 + ], + "predicted": 2, + "choice_nll": 0.08317765606176408, + "margin": 2.929838180541992 + }, + { + "id": "test4-007", + "length": 75, + "gold": 0, + "logits": [ + 20.862197875976562, + 17.06369400024414, + 17.01694107055664, + 17.710298538208008 + ], + "predicted": 0, + "choice_nll": 0.08301305176791729, + "margin": 3.1518993377685547 + }, + { + "id": "test4-008", + "length": 76, + "gold": 1, + "logits": [ + 18.473052978515625, + 20.670269012451172, + 17.35849380493164, + 18.34456443786621 + ], + "predicted": 1, + "choice_nll": 0.21935883063818906, + "margin": 2.197216033935547 + }, + { + "id": "test4-009", + "length": 75, + "gold": 3, + "logits": [ + 16.380903244018555, + 16.678552627563477, + 16.524158477783203, + 20.58717918395996 + ], + "predicted": 3, + "choice_nll": 0.05085165513703771, + "margin": 3.9086265563964844 + }, + { + "id": "test4-010", + "length": 76, + "gold": 3, + "logits": [ + 16.152225494384766, + 15.434473037719727, + 16.798120498657227, + 20.383840560913086 + ], + "predicted": 3, + "choice_nll": 0.04815520030673426, + "margin": 3.5857200622558594 + }, + { + "id": "test4-011", + "length": 75, + "gold": 0, + "logits": [ + 20.741100311279297, + 17.869335174560547, + 16.730234146118164, + 17.624149322509766 + ], + "predicted": 0, + "choice_nll": 0.11244314923289878, + "margin": 2.87176513671875 + } + ], + "permutation": [ + { + "id": "test4-000", + "length": 77, + "gold": 2, + "logits": [ + 16.735118865966797, + 16.851457595825195, + 20.01123809814453, + 16.02765655517578 + ], + "predicted": 2, + "choice_nll": 0.09424457326851865, + "margin": 3.159780502319336 + }, + { + "id": "test4-001", + "length": 76, + "gold": 2, + "logits": [ + 16.536579132080078, + 17.14190101623535, + 19.92595863342285, + 15.70170783996582 + ], + "predicted": 2, + "choice_nll": 0.10449801001254855, + "margin": 2.7840576171875 + }, + { + "id": "test4-002", + "length": 76, + "gold": 3, + "logits": [ + 16.771465301513672, + 17.265377044677734, + 17.96112823486328, + 20.62438201904297 + ], + "predicted": 3, + "choice_nll": 0.11841269844893532, + "margin": 2.6632537841796875 + }, + { + "id": "test4-003", + "length": 77, + "gold": 1, + "logits": [ + 17.27682113647461, + 20.29172134399414, + 17.236806869506836, + 17.24854278564453 + ], + "predicted": 1, + "choice_nll": 0.1344090028097611, + "margin": 3.0149002075195312 + }, + { + "id": "test4-004", + "length": 78, + "gold": 0, + "logits": [ + 20.807220458984375, + 17.271419525146484, + 16.10713005065918, + 16.970504760742188 + ], + "predicted": 0, + "choice_nll": 0.05807471551611698, + "margin": 3.5358009338378906 + }, + { + "id": "test4-005", + "length": 75, + "gold": 1, + "logits": [ + 16.716829299926758, + 20.035757064819336, + 15.445991516113281, + 16.596508026123047 + ], + "predicted": 1, + "choice_nll": 0.07551149993994244, + "margin": 3.318927764892578 + }, + { + "id": "test4-006", + "length": 75, + "gold": 2, + "logits": [ + 16.318437576293945, + 16.15199089050293, + 20.18669891357422, + 17.07793426513672 + ], + "predicted": 2, + "choice_nll": 0.07995802583964462, + "margin": 3.1087646484375 + }, + { + "id": "test4-007", + "length": 75, + "gold": 0, + "logits": [ + 20.392377853393555, + 16.65134048461914, + 16.7242431640625, + 17.324676513671875 + ], + "predicted": 0, + "choice_nll": 0.09146780188607906, + "margin": 3.0677013397216797 + }, + { + "id": "test4-008", + "length": 76, + "gold": 1, + "logits": [ + 17.70355987548828, + 19.81635284423828, + 16.50213623046875, + 17.58397674560547 + ], + "predicted": 1, + "choice_nll": 0.23470492719658864, + "margin": 2.11279296875 + }, + { + "id": "test4-009", + "length": 75, + "gold": 3, + "logits": [ + 16.12575340270996, + 16.33115577697754, + 16.3835506439209, + 20.088977813720703 + ], + "predicted": 3, + "choice_nll": 0.06478158273214873, + "margin": 3.7054271697998047 + }, + { + "id": "test4-010", + "length": 76, + "gold": 3, + "logits": [ + 15.735990524291992, + 15.136345863342285, + 16.392471313476562, + 19.817602157592773 + ], + "predicted": 3, + "choice_nll": 0.0570347746152251, + "margin": 3.425130844116211 + }, + { + "id": "test4-011", + "length": 75, + "gold": 0, + "logits": [ + 20.419654846191406, + 17.480161666870117, + 16.555973052978516, + 17.357961654663086 + ], + "predicted": 0, + "choice_nll": 0.11394588376834847, + "margin": 2.939493179321289 + } + ], + "mix": [ + { + "id": "test4-000", + "length": 77, + "gold": 2, + "logits": [ + 16.904003143310547, + 16.947982788085938, + 19.80469512939453, + 15.784209251403809 + ], + "predicted": 2, + "choice_nll": 0.12255983281767513, + "margin": 2.8567123413085938 + }, + { + "id": "test4-001", + "length": 76, + "gold": 2, + "logits": [ + 16.601720809936523, + 17.173847198486328, + 19.795743942260742, + 15.517343521118164 + ], + "predicted": 2, + "choice_nll": 0.12003497934032481, + "margin": 2.621896743774414 + }, + { + "id": "test4-002", + "length": 76, + "gold": 3, + "logits": [ + 16.63176727294922, + 17.198333740234375, + 17.518476486206055, + 20.18014144897461 + ], + "predicted": 3, + "choice_nll": 0.1391568327769157, + "margin": 2.6616649627685547 + }, + { + "id": "test4-003", + "length": 77, + "gold": 1, + "logits": [ + 17.334075927734375, + 20.28510284423828, + 17.041175842285156, + 17.1342716217041 + ], + "predicted": 1, + "choice_nll": 0.12585076563465236, + "margin": 2.9510269165039062 + }, + { + "id": "test4-004", + "length": 78, + "gold": 0, + "logits": [ + 20.726375579833984, + 17.157695770263672, + 15.784757614135742, + 16.76156997680664 + ], + "predicted": 0, + "choice_nll": 0.05288444301934092, + "margin": 3.5686798095703125 + }, + { + "id": "test4-005", + "length": 75, + "gold": 1, + "logits": [ + 16.85169792175293, + 20.062850952148438, + 15.416029930114746, + 16.643123626708984 + ], + "predicted": 1, + "choice_nll": 0.07938727148394165, + "margin": 3.211153030395508 + }, + { + "id": "test4-006", + "length": 75, + "gold": 2, + "logits": [ + 16.45294761657715, + 16.244197845458984, + 19.974834442138672, + 16.874691009521484 + ], + "predicted": 2, + "choice_nll": 0.09400382427712373, + "margin": 3.1001434326171875 + }, + { + "id": "test4-007", + "length": 75, + "gold": 0, + "logits": [ + 20.287551879882812, + 16.538028717041016, + 16.52773666381836, + 17.132783889770508 + ], + "predicted": 0, + "choice_nll": 0.08568702594606847, + "margin": 3.1547679901123047 + }, + { + "id": "test4-008", + "length": 76, + "gold": 1, + "logits": [ + 17.858036041259766, + 19.87386703491211, + 16.403911590576172, + 17.60442352294922 + ], + "predicted": 1, + "choice_nll": 0.2372024728592308, + "margin": 2.0158309936523438 + }, + { + "id": "test4-009", + "length": 75, + "gold": 3, + "logits": [ + 16.081506729125977, + 16.368045806884766, + 16.125843048095703, + 19.80476188659668 + ], + "predicted": 3, + "choice_nll": 0.07841889155891124, + "margin": 3.436716079711914 + }, + { + "id": "test4-010", + "length": 76, + "gold": 3, + "logits": [ + 15.82723617553711, + 15.249456405639648, + 16.20761489868164, + 19.536048889160156 + ], + "predicted": 3, + "choice_nll": 0.07149009200395184, + "margin": 3.3284339904785156 + }, + { + "id": "test4-011", + "length": 75, + "gold": 0, + "logits": [ + 20.312652587890625, + 17.367645263671875, + 16.299882888793945, + 17.14519691467285 + ], + "predicted": 0, + "choice_nll": 0.10687533581094755, + "margin": 2.94500732421875 + } + ], + "local32": [ + { + "id": "test4-000", + "length": 77, + "gold": 2, + "logits": [ + 16.768281936645508, + 17.311460494995117, + 19.63459014892578, + 15.489089965820312 + ], + "predicted": 2, + "choice_nll": 0.157610958629661, + "margin": 2.323129653930664 + }, + { + "id": "test4-001", + "length": 76, + "gold": 2, + "logits": [ + 16.6364803314209, + 17.357803344726562, + 19.72966766357422, + 15.513666152954102 + ], + "predicted": 2, + "choice_nll": 0.14273256980572313, + "margin": 2.3718643188476562 + }, + { + "id": "test4-002", + "length": 76, + "gold": 3, + "logits": [ + 16.709047317504883, + 17.33123779296875, + 17.52016830444336, + 20.236637115478516 + ], + "predicted": 3, + "choice_nll": 0.13994483537429048, + "margin": 2.7164688110351562 + }, + { + "id": "test4-003", + "length": 77, + "gold": 1, + "logits": [ + 17.359933853149414, + 20.35503387451172, + 16.92527961730957, + 17.053436279296875 + ], + "predicted": 1, + "choice_nll": 0.11265954477408613, + "margin": 2.9951000213623047 + }, + { + "id": "test4-004", + "length": 78, + "gold": 0, + "logits": [ + 20.63446044921875, + 16.994281768798828, + 15.579039573669434, + 16.66025161743164 + ], + "predicted": 0, + "choice_nll": 0.05013830436688238, + "margin": 3.640178680419922 + }, + { + "id": "test4-005", + "length": 75, + "gold": 1, + "logits": [ + 17.03122329711914, + 20.273269653320312, + 15.627067565917969, + 16.974788665771484 + ], + "predicted": 1, + "choice_nll": 0.08215222400320599, + "margin": 3.242046356201172 + }, + { + "id": "test4-006", + "length": 75, + "gold": 2, + "logits": [ + 16.465789794921875, + 16.297840118408203, + 19.978214263916016, + 16.75771141052246 + ], + "predicted": 2, + "choice_nll": 0.09072973228391845, + "margin": 3.2205028533935547 + }, + { + "id": "test4-007", + "length": 75, + "gold": 0, + "logits": [ + 20.154361724853516, + 16.509387969970703, + 16.47445297241211, + 17.052249908447266 + ], + "predicted": 0, + "choice_nll": 0.0919422775377221, + "margin": 3.10211181640625 + }, + { + "id": "test4-008", + "length": 76, + "gold": 1, + "logits": [ + 17.889354705810547, + 20.010948181152344, + 16.408912658691406, + 17.630643844604492 + ], + "predicted": 1, + "choice_nll": 0.21481382697915663, + "margin": 2.121593475341797 + }, + { + "id": "test4-009", + "length": 75, + "gold": 3, + "logits": [ + 16.102310180664062, + 16.465564727783203, + 16.051025390625, + 19.966941833496094 + ], + "predicted": 3, + "choice_nll": 0.06863831348178095, + "margin": 3.5013771057128906 + }, + { + "id": "test4-010", + "length": 76, + "gold": 3, + "logits": [ + 15.895734786987305, + 15.589902877807617, + 16.233055114746094, + 19.59174346923828 + ], + "predicted": 3, + "choice_nll": 0.07500098312795345, + "margin": 3.3586883544921875 + }, + { + "id": "test4-011", + "length": 75, + "gold": 0, + "logits": [ + 20.25992202758789, + 17.46915626525879, + 16.2247257232666, + 17.093454360961914 + ], + "predicted": 0, + "choice_nll": 0.1144072440925008, + "margin": 2.7907657623291016 + } + ], + "mix_local32": [ + { + "id": "test4-000", + "length": 77, + "gold": 2, + "logits": [ + 16.764371871948242, + 17.00044059753418, + 19.639976501464844, + 15.802705764770508 + ], + "predicted": 2, + "choice_nll": 0.13917810053367674, + "margin": 2.639535903930664 + }, + { + "id": "test4-001", + "length": 76, + "gold": 2, + "logits": [ + 16.51125144958496, + 17.120214462280273, + 19.664173126220703, + 15.384360313415527 + ], + "predicted": 2, + "choice_nll": 0.1267446925753113, + "margin": 2.5439586639404297 + }, + { + "id": "test4-002", + "length": 76, + "gold": 3, + "logits": [ + 16.56474494934082, + 17.175416946411133, + 17.4773006439209, + 20.06832504272461 + ], + "predicted": 3, + "choice_nll": 0.14880572611904697, + "margin": 2.591024398803711 + }, + { + "id": "test4-003", + "length": 77, + "gold": 1, + "logits": [ + 17.24075698852539, + 20.17008399963379, + 16.954517364501953, + 17.078100204467773 + ], + "predicted": 1, + "choice_nll": 0.13013079842269093, + "margin": 2.9293270111083984 + }, + { + "id": "test4-004", + "length": 78, + "gold": 0, + "logits": [ + 20.519777297973633, + 16.96579933166504, + 15.603675842285156, + 16.66135597229004 + ], + "predicted": 0, + "choice_nll": 0.05547210320765572, + "margin": 3.5539779663085938 + }, + { + "id": "test4-005", + "length": 75, + "gold": 1, + "logits": [ + 16.810771942138672, + 20.05539321899414, + 15.5126953125, + 16.78658676147461 + ], + "predicted": 1, + "choice_nll": 0.0840467957427542, + "margin": 3.2446212768554688 + }, + { + "id": "test4-006", + "length": 75, + "gold": 2, + "logits": [ + 16.373920440673828, + 16.191879272460938, + 19.85903549194336, + 16.77511978149414 + ], + "predicted": 2, + "choice_nll": 0.0971075633755909, + "margin": 3.0839157104492188 + }, + { + "id": "test4-007", + "length": 75, + "gold": 0, + "logits": [ + 20.009281158447266, + 16.344844818115234, + 16.389373779296875, + 16.997432708740234 + ], + "predicted": 0, + "choice_nll": 0.09676767565088085, + "margin": 3.0118484497070312 + }, + { + "id": "test4-008", + "length": 76, + "gold": 1, + "logits": [ + 17.73645782470703, + 19.777111053466797, + 16.366344451904297, + 17.577911376953125 + ], + "predicted": 1, + "choice_nll": 0.24204502994872357, + "margin": 2.0406532287597656 + }, + { + "id": "test4-009", + "length": 75, + "gold": 3, + "logits": [ + 16.033611297607422, + 16.368921279907227, + 16.13457679748535, + 19.85881996154785 + ], + "predicted": 3, + "choice_nll": 0.0736678494417653, + "margin": 3.489898681640625 + }, + { + "id": "test4-010", + "length": 76, + "gold": 3, + "logits": [ + 15.750129699707031, + 15.218042373657227, + 16.16313934326172, + 19.433889389038086 + ], + "predicted": 3, + "choice_nll": 0.07498322377873023, + "margin": 3.270750045776367 + }, + { + "id": "test4-011", + "length": 75, + "gold": 0, + "logits": [ + 20.141040802001953, + 17.290340423583984, + 16.214134216308594, + 17.090572357177734 + ], + "predicted": 0, + "choice_nll": 0.11764533670371691, + "margin": 2.8507003784179688 + } + ] + }, + "test8": { + "native": [ + { + "id": "test8-000", + "length": 104, + "gold": 2, + "logits": [ + 16.621517181396484, + 15.6837158203125, + 19.539764404296875, + 15.766950607299805 + ], + "predicted": 2, + "choice_nll": 0.09364243280192852, + "margin": 2.9182472229003906 + }, + { + "id": "test8-001", + "length": 104, + "gold": 0, + "logits": [ + 19.910175323486328, + 17.037057876586914, + 14.699175834655762, + 16.34076499938965 + ], + "predicted": 0, + "choice_nll": 0.08631633328393207, + "margin": 2.873117446899414 + }, + { + "id": "test8-002", + "length": 101, + "gold": 2, + "logits": [ + 16.154502868652344, + 16.303647994995117, + 19.360422134399414, + 16.323978424072266 + ], + "predicted": 2, + "choice_nll": 0.1271313230847717, + "margin": 3.0364437103271484 + }, + { + "id": "test8-003", + "length": 102, + "gold": 1, + "logits": [ + 16.977914810180664, + 19.704490661621094, + 16.104690551757812, + 16.536231994628906 + ], + "predicted": 1, + "choice_nll": 0.1264995862156688, + "margin": 2.7265758514404297 + }, + { + "id": "test8-004", + "length": 104, + "gold": 1, + "logits": [ + 17.780155181884766, + 20.037870407104492, + 15.882946014404297, + 16.51515007019043 + ], + "predicted": 1, + "choice_nll": 0.13958381866148528, + "margin": 2.2577152252197266 + }, + { + "id": "test8-005", + "length": 102, + "gold": 3, + "logits": [ + 16.11016082763672, + 15.046707153320312, + 16.25098419189453, + 19.12643051147461 + ], + "predicted": 3, + "choice_nll": 0.11536831046881346, + "margin": 2.875446319580078 + }, + { + "id": "test8-006", + "length": 103, + "gold": 1, + "logits": [ + 17.50149154663086, + 19.71080207824707, + 16.276233673095703, + 17.329715728759766 + ], + "predicted": 1, + "choice_nll": 0.21063826198889757, + "margin": 2.209310531616211 + }, + { + "id": "test8-007", + "length": 101, + "gold": 2, + "logits": [ + 17.72989273071289, + 16.66652488708496, + 19.25482940673828, + 16.512344360351562 + ], + "predicted": 2, + "choice_nll": 0.30541805593539806, + "margin": 1.5249366760253906 + }, + { + "id": "test8-008", + "length": 101, + "gold": 3, + "logits": [ + 15.743095397949219, + 14.437026977539062, + 14.758697509765625, + 18.819074630737305 + ], + "predicted": 3, + "choice_nll": 0.07314513348602603, + "margin": 3.075979232788086 + }, + { + "id": "test8-009", + "length": 103, + "gold": 0, + "logits": [ + 19.703142166137695, + 17.259708404541016, + 14.362197875976562, + 15.329724311828613 + ], + "predicted": 0, + "choice_nll": 0.09917677992543972, + "margin": 2.4434337615966797 + }, + { + "id": "test8-010", + "length": 102, + "gold": 3, + "logits": [ + 15.37771224975586, + 15.089055061340332, + 15.907177925109863, + 18.435256958007812 + ], + "predicted": 3, + "choice_nll": 0.15017106665785818, + "margin": 2.528079032897949 + }, + { + "id": "test8-011", + "length": 104, + "gold": 0, + "logits": [ + 19.813644409179688, + 15.93521499633789, + 15.06078815460205, + 16.454919815063477 + ], + "predicted": 0, + "choice_nll": 0.062119872655525654, + "margin": 3.358724594116211 + } + ], + "direct": [ + { + "id": "test8-000", + "length": 104, + "gold": 2, + "logits": [ + 17.027599334716797, + 16.32143783569336, + 20.945619583129883, + 17.534263610839844 + ], + "predicted": 2, + "choice_nll": 0.060802038606034614, + "margin": 3.411355972290039 + }, + { + "id": "test8-001", + "length": 104, + "gold": 0, + "logits": [ + 21.23013687133789, + 18.268930435180664, + 15.86952018737793, + 16.913991928100586 + ], + "predicted": 0, + "choice_nll": 0.06747704411975519, + "margin": 2.9612064361572266 + }, + { + "id": "test8-002", + "length": 101, + "gold": 2, + "logits": [ + 16.320068359375, + 16.743480682373047, + 20.798635482788086, + 17.609949111938477 + ], + "predicted": 2, + "choice_nll": 0.06757308600666434, + "margin": 3.1886863708496094 + }, + { + "id": "test8-003", + "length": 102, + "gold": 1, + "logits": [ + 17.669597625732422, + 21.058837890625, + 17.394561767578125, + 17.152141571044922 + ], + "predicted": 1, + "choice_nll": 0.07646446093087889, + "margin": 3.389240264892578 + }, + { + "id": "test8-004", + "length": 104, + "gold": 1, + "logits": [ + 18.601776123046875, + 21.2442626953125, + 17.07803726196289, + 17.163686752319336 + ], + "predicted": 1, + "choice_nll": 0.09857074173563873, + "margin": 2.642486572265625 + }, + { + "id": "test8-005", + "length": 102, + "gold": 3, + "logits": [ + 16.30388069152832, + 15.301328659057617, + 17.018505096435547, + 20.478261947631836 + ], + "predicted": 3, + "choice_nll": 0.051137317296088725, + "margin": 3.459756851196289 + }, + { + "id": "test8-006", + "length": 103, + "gold": 1, + "logits": [ + 18.18574333190918, + 20.979576110839844, + 17.665725708007812, + 17.943092346191406 + ], + "predicted": 1, + "choice_nll": 0.13589838142454624, + "margin": 2.793832778930664 + }, + { + "id": "test8-007", + "length": 101, + "gold": 2, + "logits": [ + 18.081012725830078, + 17.32012939453125, + 20.515100479125977, + 17.622295379638672 + ], + "predicted": 2, + "choice_nll": 0.1689542261914596, + "margin": 2.4340877532958984 + }, + { + "id": "test8-008", + "length": 101, + "gold": 3, + "logits": [ + 16.11387825012207, + 14.85595703125, + 15.462492942810059, + 20.17431640625 + ], + "predicted": 3, + "choice_nll": 0.030655892602528212, + "margin": 4.06043815612793 + }, + { + "id": "test8-009", + "length": 103, + "gold": 0, + "logits": [ + 20.941967010498047, + 18.273719787597656, + 15.545938491821289, + 16.07107925415039 + ], + "predicted": 0, + "choice_nll": 0.07841815109428296, + "margin": 2.6682472229003906 + }, + { + "id": "test8-010", + "length": 102, + "gold": 3, + "logits": [ + 15.425253868103027, + 15.160043716430664, + 16.495376586914062, + 19.508586883544922 + ], + "predicted": 3, + "choice_nll": 0.07595176119570368, + "margin": 3.0132102966308594 + }, + { + "id": "test8-011", + "length": 104, + "gold": 0, + "logits": [ + 21.029796600341797, + 16.812767028808594, + 15.968327522277832, + 17.110422134399414 + ], + "predicted": 0, + "choice_nll": 0.0401165856145127, + "margin": 3.919374465942383 + } + ], + "permutation": [ + { + "id": "test8-000", + "length": 104, + "gold": 2, + "logits": [ + 16.41683006286621, + 15.874307632446289, + 20.36672592163086, + 16.380746841430664 + ], + "predicted": 2, + "choice_nll": 0.04786068343766914, + "margin": 3.9498958587646484 + }, + { + "id": "test8-001", + "length": 104, + "gold": 0, + "logits": [ + 20.881309509277344, + 17.881874084472656, + 15.739431381225586, + 16.607463836669922 + ], + "predicted": 0, + "choice_nll": 0.06727539788814596, + "margin": 2.9994354248046875 + }, + { + "id": "test8-002", + "length": 101, + "gold": 2, + "logits": [ + 15.93596076965332, + 16.399188995361328, + 20.25973129272461, + 16.793418884277344 + ], + "predicted": 2, + "choice_nll": 0.0634802088265971, + "margin": 3.4663124084472656 + }, + { + "id": "test8-003", + "length": 102, + "gold": 1, + "logits": [ + 17.1326961517334, + 20.331687927246094, + 16.871540069580078, + 16.728639602661133 + ], + "predicted": 1, + "choice_nll": 0.0948273315057584, + "margin": 3.1989917755126953 + }, + { + "id": "test8-004", + "length": 104, + "gold": 1, + "logits": [ + 18.17231559753418, + 20.81039047241211, + 16.813732147216797, + 16.828582763671875 + ], + "predicted": 1, + "choice_nll": 0.10303266548946155, + "margin": 2.6380748748779297 + }, + { + "id": "test8-005", + "length": 102, + "gold": 3, + "logits": [ + 15.93782901763916, + 14.986690521240234, + 16.80826759338379, + 20.000774383544922 + ], + "predicted": 3, + "choice_nll": 0.06289103598087209, + "margin": 3.192506790161133 + }, + { + "id": "test8-006", + "length": 103, + "gold": 1, + "logits": [ + 17.528846740722656, + 20.230667114257812, + 16.9373779296875, + 17.321441650390625 + ], + "predicted": 1, + "choice_nll": 0.14732694279686598, + "margin": 2.7018203735351562 + }, + { + "id": "test8-007", + "length": 101, + "gold": 2, + "logits": [ + 17.383575439453125, + 16.735610961914062, + 19.901023864746094, + 16.78569221496582 + ], + "predicted": 2, + "choice_nll": 0.15462971432325645, + "margin": 2.5174484252929688 + }, + { + "id": "test8-008", + "length": 101, + "gold": 3, + "logits": [ + 15.46757984161377, + 14.24659538269043, + 15.1040678024292, + 19.398195266723633 + ], + "predicted": 3, + "choice_nll": 0.038326271065322715, + "margin": 3.9306154251098633 + }, + { + "id": "test8-009", + "length": 103, + "gold": 0, + "logits": [ + 20.513242721557617, + 17.899044036865234, + 15.244423866271973, + 15.596056938171387 + ], + "predicted": 0, + "choice_nll": 0.0822211024349393, + "margin": 2.614198684692383 + }, + { + "id": "test8-010", + "length": 102, + "gold": 3, + "logits": [ + 15.136404991149902, + 14.857576370239258, + 16.346385955810547, + 18.947023391723633 + ], + "predicted": 3, + "choice_nll": 0.10715720580159882, + "margin": 2.600637435913086 + }, + { + "id": "test8-011", + "length": 104, + "gold": 0, + "logits": [ + 20.547317504882812, + 16.275556564331055, + 15.814254760742188, + 16.7352237701416 + ], + "predicted": 0, + "choice_nll": 0.0438814867388011, + "margin": 3.812093734741211 + } + ], + "mix": [ + { + "id": "test8-000", + "length": 104, + "gold": 2, + "logits": [ + 16.656455993652344, + 15.963043212890625, + 20.130435943603516, + 16.185543060302734 + ], + "predicted": 2, + "choice_nll": 0.06376258527380685, + "margin": 3.473979949951172 + }, + { + "id": "test8-001", + "length": 104, + "gold": 0, + "logits": [ + 20.695301055908203, + 17.657249450683594, + 15.355867385864258, + 16.47412872314453 + ], + "predicted": 0, + "choice_nll": 0.06523345963270444, + "margin": 3.0380516052246094 + }, + { + "id": "test8-002", + "length": 101, + "gold": 2, + "logits": [ + 16.018646240234375, + 16.429607391357422, + 19.96056365966797, + 16.52069854736328 + ], + "predicted": 2, + "choice_nll": 0.077661607477296, + "margin": 3.4398651123046875 + }, + { + "id": "test8-003", + "length": 102, + "gold": 1, + "logits": [ + 17.207809448242188, + 20.319873809814453, + 16.6602840423584, + 16.672626495361328 + ], + "predicted": 1, + "choice_nll": 0.09195439341256417, + "margin": 3.1120643615722656 + }, + { + "id": "test8-004", + "length": 104, + "gold": 1, + "logits": [ + 18.084674835205078, + 20.60417366027832, + 16.380268096923828, + 16.57954978942871 + ], + "predicted": 1, + "choice_nll": 0.10706934675925694, + "margin": 2.519498825073242 + }, + { + "id": "test8-005", + "length": 102, + "gold": 3, + "logits": [ + 15.972607612609863, + 15.096407890319824, + 16.525115966796875, + 19.669971466064453 + ], + "predicted": 3, + "choice_nll": 0.07527726235955257, + "margin": 3.144855499267578 + }, + { + "id": "test8-006", + "length": 103, + "gold": 1, + "logits": [ + 17.56075096130371, + 20.16659927368164, + 16.71448516845703, + 17.23257827758789 + ], + "predicted": 1, + "choice_nll": 0.14730026158472853, + "margin": 2.6058483123779297 + }, + { + "id": "test8-007", + "length": 101, + "gold": 2, + "logits": [ + 17.515575408935547, + 16.794235229492188, + 19.657886505126953, + 16.56887435913086 + ], + "predicted": 2, + "choice_nll": 0.1988428330975509, + "margin": 2.1423110961914062 + }, + { + "id": "test8-008", + "length": 101, + "gold": 3, + "logits": [ + 15.708566665649414, + 14.451610565185547, + 15.025554656982422, + 19.25101089477539 + ], + "predicted": 3, + "choice_nll": 0.050499032483784845, + "margin": 3.5424442291259766 + }, + { + "id": "test8-009", + "length": 103, + "gold": 0, + "logits": [ + 20.375516891479492, + 17.677104949951172, + 14.9122314453125, + 15.52299690246582 + ], + "predicted": 0, + "choice_nll": 0.07636883120305171, + "margin": 2.6984119415283203 + }, + { + "id": "test8-010", + "length": 102, + "gold": 3, + "logits": [ + 15.208725929260254, + 14.982850074768066, + 16.130722045898438, + 18.8231201171875 + ], + "predicted": 3, + "choice_nll": 0.10987575380120163, + "margin": 2.6923980712890625 + }, + { + "id": "test8-011", + "length": 104, + "gold": 0, + "logits": [ + 20.460208892822266, + 16.208419799804688, + 15.55892562866211, + 16.57878875732422 + ], + "predicted": 0, + "choice_nll": 0.04142720875772099, + "margin": 3.881420135498047 + } + ], + "local32": [ + { + "id": "test8-000", + "length": 104, + "gold": 2, + "logits": [ + 16.636749267578125, + 15.927006721496582, + 19.93193817138672, + 16.024133682250977 + ], + "predicted": 2, + "choice_nll": 0.07266583969864726, + "margin": 3.2951889038085938 + }, + { + "id": "test8-001", + "length": 104, + "gold": 0, + "logits": [ + 20.592639923095703, + 17.70699119567871, + 15.276535034179688, + 16.508323669433594 + ], + "predicted": 0, + "choice_nll": 0.07470392304934137, + "margin": 2.885648727416992 + }, + { + "id": "test8-002", + "length": 101, + "gold": 2, + "logits": [ + 16.043025970458984, + 16.44487190246582, + 19.91189956665039, + 16.435810089111328 + ], + "predicted": 2, + "choice_nll": 0.07975311752792535, + "margin": 3.4670276641845703 + }, + { + "id": "test8-003", + "length": 102, + "gold": 1, + "logits": [ + 17.36279296875, + 20.37843132019043, + 16.671070098876953, + 16.859935760498047 + ], + "predicted": 1, + "choice_nll": 0.09821571428122411, + "margin": 3.0156383514404297 + }, + { + "id": "test8-004", + "length": 104, + "gold": 1, + "logits": [ + 18.133665084838867, + 20.609472274780273, + 16.24533462524414, + 16.53144073486328 + ], + "predicted": 1, + "choice_nll": 0.10774301323520419, + "margin": 2.4758071899414062 + }, + { + "id": "test8-005", + "length": 102, + "gold": 3, + "logits": [ + 15.948246955871582, + 15.189212799072266, + 16.511062622070312, + 19.601673126220703 + ], + "predicted": 3, + "choice_nll": 0.08019813335819848, + "margin": 3.0906105041503906 + }, + { + "id": "test8-006", + "length": 103, + "gold": 1, + "logits": [ + 17.56306266784668, + 20.211214065551758, + 16.551334381103516, + 17.203041076660156 + ], + "predicted": 1, + "choice_nll": 0.13618980143947113, + "margin": 2.648151397705078 + }, + { + "id": "test8-007", + "length": 101, + "gold": 2, + "logits": [ + 17.600358963012695, + 16.896236419677734, + 19.731307983398438, + 16.484975814819336 + ], + "predicted": 2, + "choice_nll": 0.19585915726820957, + "margin": 2.130949020385742 + }, + { + "id": "test8-008", + "length": 101, + "gold": 3, + "logits": [ + 15.708930015563965, + 14.538978576660156, + 14.848932266235352, + 19.25113868713379 + ], + "predicted": 3, + "choice_nll": 0.04896628669165892, + "margin": 3.542208671569824 + }, + { + "id": "test8-009", + "length": 103, + "gold": 0, + "logits": [ + 20.179096221923828, + 17.587549209594727, + 14.729844093322754, + 15.438987731933594 + ], + "predicted": 0, + "choice_nll": 0.08428718561754067, + "margin": 2.5915470123291016 + }, + { + "id": "test8-010", + "length": 102, + "gold": 3, + "logits": [ + 15.198800086975098, + 15.14482593536377, + 15.9717378616333, + 18.817840576171875 + ], + "predicted": 3, + "choice_nll": 0.10461076483347753, + "margin": 2.846102714538574 + }, + { + "id": "test8-011", + "length": 104, + "gold": 0, + "logits": [ + 20.36150360107422, + 16.33473014831543, + 15.425701141357422, + 16.503637313842773 + ], + "predicted": 0, + "choice_nll": 0.04509713051668385, + "margin": 3.8578662872314453 + } + ], + "mix_local32": [ + { + "id": "test8-000", + "length": 104, + "gold": 2, + "logits": [ + 16.48821449279785, + 15.82789134979248, + 19.950706481933594, + 16.07608985900879 + ], + "predicted": 2, + "choice_nll": 0.06608047047249599, + "margin": 3.462491989135742 + }, + { + "id": "test8-001", + "length": 104, + "gold": 0, + "logits": [ + 20.476268768310547, + 17.536632537841797, + 15.147422790527344, + 16.40839958190918 + ], + "predicted": 0, + "choice_nll": 0.07217966802835463, + "margin": 2.93963623046875 + }, + { + "id": "test8-002", + "length": 101, + "gold": 2, + "logits": [ + 15.928197860717773, + 16.346342086791992, + 19.835521697998047, + 16.4910945892334 + ], + "predicted": 2, + "choice_nll": 0.08240962501135485, + "margin": 3.3444271087646484 + }, + { + "id": "test8-003", + "length": 102, + "gold": 1, + "logits": [ + 17.126182556152344, + 20.212139129638672, + 16.588687896728516, + 16.70325469970703 + ], + "predicted": 1, + "choice_nll": 0.09740525486199303, + "margin": 3.085956573486328 + }, + { + "id": "test8-004", + "length": 104, + "gold": 1, + "logits": [ + 17.962039947509766, + 20.461090087890625, + 16.237640380859375, + 16.502544403076172 + ], + "predicted": 1, + "choice_nll": 0.10966294699075263, + "margin": 2.4990501403808594 + }, + { + "id": "test8-005", + "length": 102, + "gold": 3, + "logits": [ + 15.93612289428711, + 15.119791030883789, + 16.579944610595703, + 19.627403259277344 + ], + "predicted": 3, + "choice_nll": 0.08014507385777492, + "margin": 3.0474586486816406 + }, + { + "id": "test8-006", + "length": 103, + "gold": 1, + "logits": [ + 17.51056480407715, + 20.132099151611328, + 16.667896270751953, + 17.31209945678711 + ], + "predicted": 1, + "choice_nll": 0.151514477952027, + "margin": 2.6215343475341797 + }, + { + "id": "test8-007", + "length": 101, + "gold": 2, + "logits": [ + 17.463537216186523, + 16.77313232421875, + 19.56870460510254, + 16.468242645263672 + ], + "predicted": 2, + "choice_nll": 0.2053327452837479, + "margin": 2.1051673889160156 + }, + { + "id": "test8-008", + "length": 101, + "gold": 3, + "logits": [ + 15.562572479248047, + 14.373685836791992, + 14.887626647949219, + 19.113208770751953 + ], + "predicted": 3, + "choice_nll": 0.05075585736743449, + "margin": 3.5506362915039062 + }, + { + "id": "test8-009", + "length": 103, + "gold": 0, + "logits": [ + 20.13544273376465, + 17.511119842529297, + 14.688599586486816, + 15.444183349609375 + ], + "predicted": 0, + "choice_nll": 0.0824771291701012, + "margin": 2.6243228912353516 + }, + { + "id": "test8-010", + "length": 102, + "gold": 3, + "logits": [ + 15.188861846923828, + 15.025588035583496, + 16.08652687072754, + 18.736934661865234 + ], + "predicted": 3, + "choice_nll": 0.11675765867213528, + "margin": 2.6504077911376953 + }, + { + "id": "test8-011", + "length": 104, + "gold": 0, + "logits": [ + 20.209115982055664, + 16.06259536743164, + 15.389223098754883, + 16.493852615356445 + ], + "predicted": 0, + "choice_nll": 0.04710879080478847, + "margin": 3.7152633666992188 + } + ] + } +} diff --git a/results/lisa-attention-pilot/summary.json b/results/lisa-attention-pilot/summary.json new file mode 100644 index 0000000..99817e0 --- /dev/null +++ b/results/lisa-attention-pilot/summary.json @@ -0,0 +1,95 @@ +{ + "sets": { + "test4": { + "native": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.1577236725451122, + "mean_margin": 2.6222333908081055, + "prediction_agreement_with_native": 12 + }, + "direct": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.09467621731363618, + "mean_margin": 3.157541592915853, + "prediction_agreement_with_native": 12 + }, + "permutation": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.10225362466948813, + "mean_margin": 3.069669246673584, + "prediction_agreement_with_native": 12 + }, + "mix": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.10946264729409035, + "mean_margin": 2.9876694679260254, + "prediction_agreement_with_native": 12 + }, + "local32": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.11173090120474012, + "mean_margin": 2.9486522674560547, + "prediction_agreement_with_native": 12 + }, + "mix_local32": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.1155495746250453, + "mean_margin": 2.9375176429748535, + "prediction_agreement_with_native": 12 + } + }, + "test8": { + "native": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.13243424793047875, + "mean_margin": 2.652334133783976, + "prediction_agreement_with_native": 12 + }, + "direct": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.07933497390150783, + "mean_margin": 3.1618269284566245, + "prediction_agreement_with_native": 12 + }, + "permutation": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.08440917052410739, + "mean_margin": 3.135169267654419, + "prediction_agreement_with_native": 12 + }, + "mix": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.09210604798693496, + "mean_margin": 3.0242624282836914, + "prediction_agreement_with_native": 12 + }, + "local32": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.09402417229313187, + "mean_margin": 2.9872288703918457, + "prediction_agreement_with_native": 12 + }, + "mix_local32": { + "n": 12, + "correct": 12, + "mean_choice_nll": 0.09681914153941336, + "mean_margin": 2.970529397328695, + "prediction_agreement_with_native": 12 + } + } + }, + "native_last_layer_replay_parity": true, + "calibration_reused": true, + "seconds": 681.1676366906613 +} -- 2.52.0