266 lines
12 KiB
Python
266 lines
12 KiB
Python
"""Single-site LiSA-style pilot on the retained Qwen3.5-2B checkpoint."""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import random
|
|
import time
|
|
|
|
import numpy as np
|
|
from tokenizers import Tokenizer
|
|
|
|
from crossqkv.core import QKVModel, assignment
|
|
from functional.model import PREFIX
|
|
from functional.run import write_json
|
|
from pilot.tensors import decode
|
|
|
|
DONOR, TARGET = 19, 23
|
|
WIDTH = 32
|
|
MODES = ('native', 'direct', 'permutation', 'mix', 'local32', 'mix_local32')
|
|
|
|
|
|
def scores(q, k):
|
|
keys = np.repeat(k, q.shape[2] // k.shape[2], axis=2)
|
|
return np.einsum('bthd,bshd->bhts', q, keys) / np.float32(np.sqrt(q.shape[-1]))
|
|
|
|
|
|
def probabilities(x):
|
|
x = np.where(np.tri(x.shape[-1], dtype=bool), x, -np.inf)
|
|
exp = np.exp(x - x.max(axis=-1, keepdims=True))
|
|
return exp / exp.sum(axis=-1, keepdims=True)
|
|
|
|
|
|
def centered_rows(x, lengths):
|
|
values = []
|
|
for i, length in enumerate(lengths):
|
|
query, key = np.tril_indices(length)
|
|
board = x[i, :, :length, :length]
|
|
mean = np.where(np.tri(length, dtype=bool), board, 0).sum(-1) / np.arange(1, length + 1)
|
|
centered = board - mean[..., None]
|
|
values.append(centered[:, query, key].T)
|
|
return np.concatenate(values).astype(np.float64)
|
|
|
|
|
|
def solve(x, y):
|
|
gram = x.T @ x
|
|
ridge = 1e-3 * np.trace(gram) / gram.shape[0]
|
|
return np.linalg.solve(gram + max(ridge, 1e-12) * np.eye(len(gram)), x.T @ y).astype(np.float32)
|
|
|
|
|
|
def fit(donor, target, local):
|
|
cost = np.mean((target[:, :, None] - donor[:, None, :]) ** 2, axis=0)
|
|
perm = assignment(cost)
|
|
return {
|
|
'permutation': list(perm),
|
|
'mix': solve(donor, target),
|
|
'local32': solve(local, target),
|
|
'mix_local32': solve(np.concatenate((donor, local), axis=1), target),
|
|
}
|
|
|
|
|
|
def predict(donor, local, mode, mapping):
|
|
if mode == 'direct':
|
|
return donor
|
|
if mode == 'permutation':
|
|
return donor[:, mapping[mode]]
|
|
if mode == 'mix':
|
|
inputs = donor
|
|
elif mode == 'local32':
|
|
inputs = local
|
|
else:
|
|
inputs = np.concatenate((donor, local), axis=1)
|
|
return np.einsum('bhts,hi->bits', inputs, mapping[mode])
|
|
|
|
|
|
class Study(QKVModel):
|
|
def __init__(self, *args, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
dim = self.config['head_dim']
|
|
if dim % WIDTH:
|
|
raise ValueError('Expected head dimension divisible by fixed local width')
|
|
# Deterministic channel sample in post-RoPE Q/K coordinates.
|
|
self.coordinates = np.arange(0, dim, dim // WIDTH)
|
|
self.capture_scores = False
|
|
|
|
def normalization(self, x, name):
|
|
if self.capture_scores and name == f'layers.{TARGET}.input_layernorm':
|
|
self.recipient_input = x
|
|
return super().normalization(x, name)
|
|
|
|
def full_attention(self, h, layer, shared_p=None):
|
|
if not self.capture_scores or layer not in (DONOR, TARGET):
|
|
return super().full_attention(h, layer, shared_p)
|
|
parts = self.parts(h, layer, heads=False)
|
|
if layer == DONOR:
|
|
self.donor = scores(parts['q'], parts['k'])
|
|
else:
|
|
q, k = parts['q'], parts['k']
|
|
self.teacher = scores(q, k)
|
|
self.local = scores(q[..., self.coordinates], k[..., self.coordinates])
|
|
self.recipient = parts
|
|
return parts['output'], parts['p']
|
|
|
|
def replay(self, modes):
|
|
if TARGET != len(self.config['layer_types']) - 1:
|
|
raise ValueError('Replay requires changing only the final layer')
|
|
parts = self.recipient
|
|
repeats = self.config['num_attention_heads'] // self.config['num_key_value_heads']
|
|
values = np.repeat(parts['v'], repeats, axis=2)
|
|
contexts = [np.einsum('bhts,bshd->bthd',
|
|
probabilities(predict(self.donor, self.local, mode, self.mapping)),
|
|
values) for mode in modes]
|
|
context = np.concatenate([parts['context'], *contexts])
|
|
gate = np.concatenate([parts['gate']] * (1 + len(modes)))
|
|
residual = np.concatenate([self.recipient_input] * (1 + len(modes)))
|
|
hidden = residual + self.project(context, gate, TARGET)
|
|
mlp_input = self.normalization(hidden, f'layers.{TARGET}.post_attention_layernorm')
|
|
return np.split(self.normalization(hidden + self.mlp(mlp_input, TARGET), 'norm'),
|
|
1 + len(modes))
|
|
|
|
|
|
def dataset(split, count):
|
|
seed = {'calibration': 2026112201, 'test4': 2026112202, 'test8': 2026112203}[split]
|
|
rng = random.Random(seed)
|
|
size = 8 if split == 'test8' else 4
|
|
words = ('amber', 'birch', 'cedar', 'dune', 'elm', 'fern', 'grove', 'heath',
|
|
'iris', 'jade', 'kelp', 'larch', 'maple', 'nettle', 'olive', 'pine')
|
|
examples = []
|
|
answers = [i % 4 for i in range(count)]
|
|
rng.shuffle(answers)
|
|
for i, answer in enumerate(answers):
|
|
keys = rng.sample(words, size)
|
|
values = rng.sample(range(10, 100), size)
|
|
index = i % size
|
|
other = rng.sample([v for j, v in enumerate(values) if j != index], 3)
|
|
other.insert(answer, values[index])
|
|
items = '\n'.join(f'{k}: {v}' for k, v in zip(keys, values))
|
|
options = '\n'.join(f'{j + 1}. {v}' for j, v in enumerate(other))
|
|
prompt = (f'Use the lookup table. Answer with only the option number.\n{items}\n'
|
|
f'Which option is the value for {keys[index]}?\n{options}\nAnswer:')
|
|
examples.append({'id': f'{split}-{i:03}', 'prompt': prompt, 'answer_index': answer,
|
|
'evidence_position': index, 'entries': size})
|
|
return examples
|
|
|
|
|
|
def batch(tokenizer, examples):
|
|
encoded = []
|
|
choices = None
|
|
for ex in examples:
|
|
continuations = [tokenizer.encode(f"{ex['prompt']} {i}", add_special_tokens=False).ids
|
|
for i in range(1, 5)]
|
|
prefix = continuations[0][:-1]
|
|
labels = [row[-1] for row in continuations]
|
|
if any(row[:-1] != prefix for row in continuations) or len(set(labels)) != 4:
|
|
raise ValueError('Choices must differ only at the final token')
|
|
if choices is not None and labels != choices:
|
|
raise ValueError('Choice tokenization varies by prompt')
|
|
encoded.append(prefix)
|
|
choices = labels
|
|
lengths = [len(ids) for ids in encoded]
|
|
ids = np.array([row + [0] * (max(lengths) - len(row)) for row in encoded], dtype=np.int64)
|
|
return ids, lengths, choices
|
|
|
|
|
|
def evaluate(lm, tokenizer, examples, batch_size=4):
|
|
results = {mode: [] for mode in MODES}
|
|
for start in range(0, len(examples), batch_size):
|
|
group = examples[start:start + batch_size]
|
|
ids, lengths, choices = batch(tokenizer, group)
|
|
n = len(group)
|
|
lm.capture_scores = True
|
|
native, _ = lm.forward(ids)
|
|
lm.capture_scores = False
|
|
replay_native, *replayed = lm.replay(MODES[1:])
|
|
if not np.allclose(replay_native, native, atol=2e-5, rtol=1e-5):
|
|
raise ValueError('Native last-layer replay parity failed')
|
|
token_rows = lm.tensors[PREFIX + 'embed_tokens.weight']
|
|
weight = decode(token_rows.raw.reshape(token_rows.shape)[choices], token_rows.dtype)
|
|
token_rows.evict()
|
|
for mode, hidden in zip(MODES, [native, *replayed]):
|
|
logits = hidden[np.arange(n), np.array(lengths) - 1] @ weight.T
|
|
for ex, row, length in zip(group, logits.astype(float), lengths):
|
|
gold = ex['answer_index']
|
|
results[mode].append({
|
|
'id': ex['id'], 'length': length, 'gold': gold,
|
|
'logits': row.tolist(), 'predicted': int(row.argmax()),
|
|
'choice_nll': float(np.logaddexp.reduce(row) - row[gold]),
|
|
'margin': float(row[gold] - np.max(np.delete(row, gold))),
|
|
})
|
|
print(f"{group[-1]['id']}: evaluated {n} prompts in all six modes", flush=True)
|
|
return results
|
|
|
|
|
|
def summarize(results):
|
|
return {split: {mode: {
|
|
'n': len(rows),
|
|
'correct': sum(row['gold'] == row['predicted'] for row in rows),
|
|
'mean_choice_nll': float(np.mean([row['choice_nll'] for row in rows])),
|
|
'mean_margin': float(np.mean([row['margin'] for row in rows])),
|
|
'prediction_agreement_with_native': sum(row['predicted'] == base['predicted']
|
|
for row, base in zip(rows, modes['native'])),
|
|
} for mode, rows in modes.items()} for split, modes in results.items()}
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('--source', type=Path, required=True)
|
|
parser.add_argument('--out', type=Path, required=True)
|
|
parser.add_argument('--calibration-run', type=Path, help='Reuse a matching completed calibration')
|
|
parser.add_argument('--seconds', type=int, default=1800)
|
|
args = parser.parse_args()
|
|
start = time.monotonic()
|
|
args.out.mkdir()
|
|
source = args.source
|
|
config = json.loads((source / 'model_config.json').read_text())
|
|
tokenizer = Tokenizer.from_file(str(source / 'tokenizer.json'))
|
|
lm = Study(source / 'weights', config, seconds=args.seconds)
|
|
kinds = lm.config['layer_types']
|
|
if any(kinds[i] != 'full_attention' for i in (DONOR, TARGET)):
|
|
raise ValueError('Requires the retained architecture with full attention at 19 and 23')
|
|
source_files = ['model_config.json', 'tokenizer.json', 'manifest.json']
|
|
protocol = {'donor': DONOR, 'target': TARGET, 'local_coordinate_count': WIDTH,
|
|
'modes': MODES, 'source': str(source),
|
|
'source_sha256': {f: hashlib.sha256((source/f).read_bytes()).hexdigest() for f in source_files},
|
|
'choice_nll_scope': 'normalized over four option-number logits only'}
|
|
problems = {s: dataset(s, n) for s, n in [('calibration', 12), ('test4', 12), ('test8', 12)]}
|
|
if len({e['prompt'] for entries in problems.values() for e in entries}) != 36:
|
|
raise ValueError('Repeated prompt across pilot sample')
|
|
write_json(args.out / 'protocol.json', protocol)
|
|
write_json(args.out / 'problems.json', problems)
|
|
|
|
if args.calibration_run:
|
|
previous = args.calibration_run
|
|
if (json.loads((previous / 'protocol.json').read_text()) !=
|
|
json.loads((args.out / 'protocol.json').read_text()) or
|
|
json.loads((previous / 'problems.json').read_text()) != problems):
|
|
raise ValueError('Calibration inputs or checkpoint provenance differ')
|
|
lm.mapping = {k: np.asarray(v, dtype=np.int64 if k == 'permutation' else np.float32)
|
|
for k, v in json.loads((previous / 'calibration.json').read_text()).items()}
|
|
else:
|
|
ids, lengths, _ = batch(tokenizer, problems['calibration'])
|
|
lm.capture_scores = True
|
|
native, _ = lm.forward(ids)
|
|
measured = [centered_rows(x, lengths) for x in (lm.donor, lm.teacher, lm.local)]
|
|
lm.capture_scores = False
|
|
reference, _ = lm.forward(ids[:4])
|
|
if not np.allclose(native[:4], reference, atol=1e-5, rtol=1e-5):
|
|
raise ValueError('Capture path disagrees with the unmodified reference')
|
|
lm.mapping = fit(*measured)
|
|
print('Calibration ready', flush=True)
|
|
write_json(args.out / 'calibration.json', {
|
|
k: np.asarray(v).tolist() for k, v in lm.mapping.items()
|
|
})
|
|
results = {}
|
|
for split in ('test4', 'test8'):
|
|
results[split] = evaluate(lm, tokenizer, problems[split])
|
|
print(split, summarize(results)[split], flush=True)
|
|
write_json(args.out / 'results.json', results)
|
|
write_json(args.out / 'summary.json', {'sets': summarize(results),
|
|
'native_last_layer_replay_parity': True,
|
|
'calibration_reused': bool(args.calibration_run),
|
|
'seconds': time.monotonic() - start})
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|