Files
exp-model-parameter-similarity/crossqkv/lisa_pilot.py
T

266 lines
12 KiB
Python

"""Single-site LiSA-style pilot on the retained Qwen3.5-2B checkpoint."""
import argparse
import hashlib
import json
from pathlib import Path
import random
import time
import numpy as np
from tokenizers import Tokenizer
from crossqkv.core import QKVModel, assignment
from functional.model import PREFIX
from functional.run import write_json
from pilot.tensors import decode
DONOR, TARGET = 19, 23
WIDTH = 32
MODES = ('native', 'direct', 'permutation', 'mix', 'local32', 'mix_local32')
def scores(q, k):
keys = np.repeat(k, q.shape[2] // k.shape[2], axis=2)
return np.einsum('bthd,bshd->bhts', q, keys) / np.float32(np.sqrt(q.shape[-1]))
def probabilities(x):
x = np.where(np.tri(x.shape[-1], dtype=bool), x, -np.inf)
exp = np.exp(x - x.max(axis=-1, keepdims=True))
return exp / exp.sum(axis=-1, keepdims=True)
def centered_rows(x, lengths):
values = []
for i, length in enumerate(lengths):
query, key = np.tril_indices(length)
board = x[i, :, :length, :length]
mean = np.where(np.tri(length, dtype=bool), board, 0).sum(-1) / np.arange(1, length + 1)
centered = board - mean[..., None]
values.append(centered[:, query, key].T)
return np.concatenate(values).astype(np.float64)
def solve(x, y):
gram = x.T @ x
ridge = 1e-3 * np.trace(gram) / gram.shape[0]
return np.linalg.solve(gram + max(ridge, 1e-12) * np.eye(len(gram)), x.T @ y).astype(np.float32)
def fit(donor, target, local):
cost = np.mean((target[:, :, None] - donor[:, None, :]) ** 2, axis=0)
perm = assignment(cost)
return {
'permutation': list(perm),
'mix': solve(donor, target),
'local32': solve(local, target),
'mix_local32': solve(np.concatenate((donor, local), axis=1), target),
}
def predict(donor, local, mode, mapping):
if mode == 'direct':
return donor
if mode == 'permutation':
return donor[:, mapping[mode]]
if mode == 'mix':
inputs = donor
elif mode == 'local32':
inputs = local
else:
inputs = np.concatenate((donor, local), axis=1)
return np.einsum('bhts,hi->bits', inputs, mapping[mode])
class Study(QKVModel):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
dim = self.config['head_dim']
if dim % WIDTH:
raise ValueError('Expected head dimension divisible by fixed local width')
# Deterministic channel sample in post-RoPE Q/K coordinates.
self.coordinates = np.arange(0, dim, dim // WIDTH)
self.capture_scores = False
def normalization(self, x, name):
if self.capture_scores and name == f'layers.{TARGET}.input_layernorm':
self.recipient_input = x
return super().normalization(x, name)
def full_attention(self, h, layer, shared_p=None):
if not self.capture_scores or layer not in (DONOR, TARGET):
return super().full_attention(h, layer, shared_p)
parts = self.parts(h, layer, heads=False)
if layer == DONOR:
self.donor = scores(parts['q'], parts['k'])
else:
q, k = parts['q'], parts['k']
self.teacher = scores(q, k)
self.local = scores(q[..., self.coordinates], k[..., self.coordinates])
self.recipient = parts
return parts['output'], parts['p']
def replay(self, modes):
if TARGET != len(self.config['layer_types']) - 1:
raise ValueError('Replay requires changing only the final layer')
parts = self.recipient
repeats = self.config['num_attention_heads'] // self.config['num_key_value_heads']
values = np.repeat(parts['v'], repeats, axis=2)
contexts = [np.einsum('bhts,bshd->bthd',
probabilities(predict(self.donor, self.local, mode, self.mapping)),
values) for mode in modes]
context = np.concatenate([parts['context'], *contexts])
gate = np.concatenate([parts['gate']] * (1 + len(modes)))
residual = np.concatenate([self.recipient_input] * (1 + len(modes)))
hidden = residual + self.project(context, gate, TARGET)
mlp_input = self.normalization(hidden, f'layers.{TARGET}.post_attention_layernorm')
return np.split(self.normalization(hidden + self.mlp(mlp_input, TARGET), 'norm'),
1 + len(modes))
def dataset(split, count):
seed = {'calibration': 2026112201, 'test4': 2026112202, 'test8': 2026112203}[split]
rng = random.Random(seed)
size = 8 if split == 'test8' else 4
words = ('amber', 'birch', 'cedar', 'dune', 'elm', 'fern', 'grove', 'heath',
'iris', 'jade', 'kelp', 'larch', 'maple', 'nettle', 'olive', 'pine')
examples = []
answers = [i % 4 for i in range(count)]
rng.shuffle(answers)
for i, answer in enumerate(answers):
keys = rng.sample(words, size)
values = rng.sample(range(10, 100), size)
index = i % size
other = rng.sample([v for j, v in enumerate(values) if j != index], 3)
other.insert(answer, values[index])
items = '\n'.join(f'{k}: {v}' for k, v in zip(keys, values))
options = '\n'.join(f'{j + 1}. {v}' for j, v in enumerate(other))
prompt = (f'Use the lookup table. Answer with only the option number.\n{items}\n'
f'Which option is the value for {keys[index]}?\n{options}\nAnswer:')
examples.append({'id': f'{split}-{i:03}', 'prompt': prompt, 'answer_index': answer,
'evidence_position': index, 'entries': size})
return examples
def batch(tokenizer, examples):
encoded = []
choices = None
for ex in examples:
continuations = [tokenizer.encode(f"{ex['prompt']} {i}", add_special_tokens=False).ids
for i in range(1, 5)]
prefix = continuations[0][:-1]
labels = [row[-1] for row in continuations]
if any(row[:-1] != prefix for row in continuations) or len(set(labels)) != 4:
raise ValueError('Choices must differ only at the final token')
if choices is not None and labels != choices:
raise ValueError('Choice tokenization varies by prompt')
encoded.append(prefix)
choices = labels
lengths = [len(ids) for ids in encoded]
ids = np.array([row + [0] * (max(lengths) - len(row)) for row in encoded], dtype=np.int64)
return ids, lengths, choices
def evaluate(lm, tokenizer, examples, batch_size=4):
results = {mode: [] for mode in MODES}
for start in range(0, len(examples), batch_size):
group = examples[start:start + batch_size]
ids, lengths, choices = batch(tokenizer, group)
n = len(group)
lm.capture_scores = True
native, _ = lm.forward(ids)
lm.capture_scores = False
replay_native, *replayed = lm.replay(MODES[1:])
if not np.allclose(replay_native, native, atol=2e-5, rtol=1e-5):
raise ValueError('Native last-layer replay parity failed')
token_rows = lm.tensors[PREFIX + 'embed_tokens.weight']
weight = decode(token_rows.raw.reshape(token_rows.shape)[choices], token_rows.dtype)
token_rows.evict()
for mode, hidden in zip(MODES, [native, *replayed]):
logits = hidden[np.arange(n), np.array(lengths) - 1] @ weight.T
for ex, row, length in zip(group, logits.astype(float), lengths):
gold = ex['answer_index']
results[mode].append({
'id': ex['id'], 'length': length, 'gold': gold,
'logits': row.tolist(), 'predicted': int(row.argmax()),
'choice_nll': float(np.logaddexp.reduce(row) - row[gold]),
'margin': float(row[gold] - np.max(np.delete(row, gold))),
})
print(f"{group[-1]['id']}: evaluated {n} prompts in all six modes", flush=True)
return results
def summarize(results):
return {split: {mode: {
'n': len(rows),
'correct': sum(row['gold'] == row['predicted'] for row in rows),
'mean_choice_nll': float(np.mean([row['choice_nll'] for row in rows])),
'mean_margin': float(np.mean([row['margin'] for row in rows])),
'prediction_agreement_with_native': sum(row['predicted'] == base['predicted']
for row, base in zip(rows, modes['native'])),
} for mode, rows in modes.items()} for split, modes in results.items()}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--source', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
parser.add_argument('--calibration-run', type=Path, help='Reuse a matching completed calibration')
parser.add_argument('--seconds', type=int, default=1800)
args = parser.parse_args()
start = time.monotonic()
args.out.mkdir()
source = args.source
config = json.loads((source / 'model_config.json').read_text())
tokenizer = Tokenizer.from_file(str(source / 'tokenizer.json'))
lm = Study(source / 'weights', config, seconds=args.seconds)
kinds = lm.config['layer_types']
if any(kinds[i] != 'full_attention' for i in (DONOR, TARGET)):
raise ValueError('Requires the retained architecture with full attention at 19 and 23')
source_files = ['model_config.json', 'tokenizer.json', 'manifest.json']
protocol = {'donor': DONOR, 'target': TARGET, 'local_coordinate_count': WIDTH,
'modes': MODES, 'source': str(source),
'source_sha256': {f: hashlib.sha256((source/f).read_bytes()).hexdigest() for f in source_files},
'choice_nll_scope': 'normalized over four option-number logits only'}
problems = {s: dataset(s, n) for s, n in [('calibration', 12), ('test4', 12), ('test8', 12)]}
if len({e['prompt'] for entries in problems.values() for e in entries}) != 36:
raise ValueError('Repeated prompt across pilot sample')
write_json(args.out / 'protocol.json', protocol)
write_json(args.out / 'problems.json', problems)
if args.calibration_run:
previous = args.calibration_run
if (json.loads((previous / 'protocol.json').read_text()) !=
json.loads((args.out / 'protocol.json').read_text()) or
json.loads((previous / 'problems.json').read_text()) != problems):
raise ValueError('Calibration inputs or checkpoint provenance differ')
lm.mapping = {k: np.asarray(v, dtype=np.int64 if k == 'permutation' else np.float32)
for k, v in json.loads((previous / 'calibration.json').read_text()).items()}
else:
ids, lengths, _ = batch(tokenizer, problems['calibration'])
lm.capture_scores = True
native, _ = lm.forward(ids)
measured = [centered_rows(x, lengths) for x in (lm.donor, lm.teacher, lm.local)]
lm.capture_scores = False
reference, _ = lm.forward(ids[:4])
if not np.allclose(native[:4], reference, atol=1e-5, rtol=1e-5):
raise ValueError('Capture path disagrees with the unmodified reference')
lm.mapping = fit(*measured)
print('Calibration ready', flush=True)
write_json(args.out / 'calibration.json', {
k: np.asarray(v).tolist() for k, v in lm.mapping.items()
})
results = {}
for split in ('test4', 'test8'):
results[split] = evaluate(lm, tokenizer, problems[split])
print(split, summarize(results)[split], flush=True)
write_json(args.out / 'results.json', results)
write_json(args.out / 'summary.json', {'sets': summarize(results),
'native_last_layer_replay_parity': True,
'calibration_reused': bool(args.calibration_run),
'seconds': time.monotonic() - start})
if __name__ == '__main__':
main()