Validate repeated learned RMSNorm functions with CPU evidence
This commit is contained in:
@@ -0,0 +1,46 @@
|
||||
"""Package code-independent scientific outputs, retain large activation caches externally.
|
||||
|
||||
The experiment directory remains immutable and complete. The delivery directory
|
||||
contains all figure source tables/NLL arrays plus hashes/locations for three large
|
||||
reproducible activation caches. Never deletes model weights or cache files.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from .run import write_json
|
||||
|
||||
CACHES={'baseline-activations.npz','neuron-reuse-activations.npz','norm-repetition-input-profiles.npz'}
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);p.add_argument('--output',required=True,type=Path);a=p.parse_args()
|
||||
root,out=a.root.resolve(),a.output.resolve();assert root!=out
|
||||
out.mkdir(parents=True,exist_ok=True)
|
||||
full=json.loads((root/'checksums-functional.json').read_text())
|
||||
files=json.loads((root/'archive-files.json').read_text())
|
||||
for name in files:
|
||||
src=root/name
|
||||
if name in CACHES:continue
|
||||
dest=out/name;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,dest)
|
||||
external=[]
|
||||
for name in sorted(CACHES):
|
||||
src=root/name;digest=hashlib.sha256(src.read_bytes()).hexdigest();assert digest==full[name]
|
||||
external.append({'name':name,'server_path':str(src),'bytes':src.stat().st_size,'sha256':digest,'retained':True,'reproducible_by':'functional.run / functional.neuron_reuse / functional.norm_repetition'})
|
||||
write_json(out/'external-activation-cache.json',{'host':'8.211.154.134','files':external,'not_model_weights':True,
|
||||
'reason':'Large intermediate caches retained on the authorized server; all figure source tables and token NLL arrays are delivered in Git.'})
|
||||
shutil.copy2(root/'checksums-functional.json',out/'checksums-full-server.json')
|
||||
(out/'DELIVERY.md').write_text('# Scientific delivery and retained caches\n\nAll figure CSV/JSON source data, per-token NLL, protocols, reports, metadata and historical logs are included.\nThree large reproducible intermediate activation caches remain on 8.211.154.134 and in the original full local archive;\n`external-activation-cache.json` records exact locations, sizes and SHA256. No weight or cache was deleted.\nThe original full experiment manifest is `checksums-full-server.json`; the delivery-only manifest is\n`checksums-functional.json`. No model checkpoint or tokenizer payload is in this Git delivery.\n')
|
||||
validation=json.loads((out/'audit-validation.json').read_text());validation.update(delivery_excludes_external_activation_caches=True,full_experiment_checksums='checksums-full-server.json',figure_source_tables_and_token_nll_complete=True)
|
||||
write_json(out/'audit-validation.json',validation)
|
||||
checks={}
|
||||
for file in sorted(out.rglob('*')):
|
||||
if file.is_file() and file.name not in ('checksums-functional.json','audit-validation.json','archive-files.json'):
|
||||
checks[str(file.relative_to(out))]=hashlib.sha256(file.read_bytes()).hexdigest()
|
||||
write_json(out/'checksums-functional.json',checks)
|
||||
write_json(out/'archive-files.json',[*checks,'checksums-functional.json','audit-validation.json','archive-files.json'])
|
||||
print(json.dumps({'delivery_files':len(checks)+3,'checksummed_files':len(checks),'external_caches':len(external),'weights_and_caches_retained':True}))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
Reference in New Issue
Block a user