57 lines
3.7 KiB
Python
57 lines
3.7 KiB
Python
"""Fresh heldout confirmation and same-input versus actual-state interventions."""
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import numpy as np
|
|
from functional.run import write_json,save_csv
|
|
from .protocol import LAYERS
|
|
|
|
|
|
def run_fresh(source,root,c,tier,lm,status):
|
|
from tokenizers import Tokenizer
|
|
from .run import validate_selected,candidate_pass,summarize
|
|
samples=json.loads(Path(__file__).with_name('fresh_samples.json').read_text())['samples']
|
|
tokenizer=Tokenizer.from_file(str(source/'tokenizer.json'));ids=[]
|
|
for s in samples:
|
|
encoded=tokenizer.encode(s['text'],add_special_tokens=False).ids
|
|
assert len(encoded)>=32,s['id']
|
|
s.update(token_ids=encoded[:32],full_token_count=len(encoded),text_sha256=hashlib.sha256(s['text'].encode()).hexdigest())
|
|
ids.append(encoded[:32])
|
|
ids=np.array(ids,dtype=np.int64)
|
|
write_json(root/'fresh-samples-tokenized.json',{'samples':samples,'selection_already_frozen':True})
|
|
status['phase']='fresh_test_baseline';write_json(root/'status.json',status)
|
|
h,inputs=lm.forward(ids,capture=[('attn',i) for i in LAYERS])
|
|
np.savez(root/'cache'/'fresh-baseline-activations.npz',final=h,**inputs)
|
|
nll=lm.nll(h,ids);np.save(root/'fresh-baseline-token-nll.npy',nll)
|
|
checked=validate_selected(lm,c,inputs,samples,np.arange(len(samples)))
|
|
passed=candidate_pass(checked,tier)
|
|
write_json(root/'fresh-function-summary.json',summarize(checked))
|
|
save_csv(root/'fresh-output-metrics.csv',checked['output_rows']);save_csv(root/'fresh-probability-metrics.csv',checked['probability_rows']);save_csv(root/'fresh-value-metrics.csv',checked['value_rows'])
|
|
status.update(fresh_function_pass=passed,fresh_confirmation_tier=tier)
|
|
if not passed:return
|
|
status['phase']='same_input_network_substitution';write_json(root/'status.json',status)
|
|
interventions=[]
|
|
for reverse in (False,True):
|
|
lm.configure(c,reverse=reverse,cached=False)
|
|
t=time.monotonic();changed,_=lm.forward(ids);loss=lm.nll(changed,ids)
|
|
label=f'{lm.target_layer}_uses_{lm.donor_layer}'
|
|
np.save(root/f'network-{label}-token-nll.npy',loss)
|
|
delta=loss.mean(1)-nll.mean(1)
|
|
rec={'mode':'same_input_donor_function','target_layer':lm.target_layer,'donor_layer':lm.donor_layer,
|
|
'target_head_or_group':lm.target_head,'donor_head_or_group':lm.donor_head,
|
|
'sequence_nll_deltas':delta.tolist(),'mean_nll_delta':float(delta.mean()),'sequence_p95_nll_delta':float(np.quantile(delta,.95)),
|
|
'quality_pass':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05),'elapsed_seconds':time.monotonic()-t}
|
|
interventions.append(rec);write_json(root/'network-interventions.json',interventions)
|
|
# Temporal reuse is a separate intervention and only goes from earlier to later.
|
|
status['phase']='earlier_to_later_actual_cache';write_json(root/'status.json',status)
|
|
lm.configure(c,reverse=True,cached=True)
|
|
changed,_=lm.forward(ids);loss=lm.nll(changed,ids);np.save(root/'network-earlier-actual-cache-token-nll.npy',loss)
|
|
delta=loss.mean(1)-nll.mean(1)
|
|
cache={'mode':'earlier_actual_state_reused_later','donor_layer':lm.donor_layer,'target_layer':lm.target_layer,
|
|
'sequence_nll_deltas':delta.tolist(),'mean_nll_delta':float(delta.mean()),'sequence_p95_nll_delta':float(np.quantile(delta,.95)),
|
|
'quality_pass':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05),
|
|
'not_same_input_comparison':True,'no_reverse_future_state_claim':True}
|
|
write_json(root/'actual-cache-intervention.json',cache)
|
|
status.update(bidirectional_same_input_network_pass=all(r['quality_pass'] for r in interventions),actual_cache_quality_pass=cache['quality_pass'])
|