47 lines
3.3 KiB
Python
47 lines
3.3 KiB
Python
"""Package code-independent scientific outputs, retain large activation caches externally.
|
|
|
|
The experiment directory remains immutable and complete. The delivery directory
|
|
contains all figure source tables/NLL arrays plus hashes/locations for three large
|
|
reproducible activation caches. Never deletes model weights or cache files.
|
|
"""
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import shutil
|
|
from .run import write_json
|
|
|
|
CACHES={'baseline-activations.npz','neuron-reuse-activations.npz','norm-repetition-input-profiles.npz'}
|
|
|
|
|
|
def main():
|
|
p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);p.add_argument('--output',required=True,type=Path);a=p.parse_args()
|
|
root,out=a.root.resolve(),a.output.resolve();assert root!=out
|
|
out.mkdir(parents=True,exist_ok=True)
|
|
full=json.loads((root/'checksums-functional.json').read_text())
|
|
files=json.loads((root/'archive-files.json').read_text())
|
|
for name in files:
|
|
src=root/name
|
|
if name in CACHES:continue
|
|
dest=out/name;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,dest)
|
|
external=[]
|
|
for name in sorted(CACHES):
|
|
src=root/name;digest=hashlib.sha256(src.read_bytes()).hexdigest();assert digest==full[name]
|
|
external.append({'name':name,'server_path':str(src),'bytes':src.stat().st_size,'sha256':digest,'retained':True,'reproducible_by':'functional.run / functional.neuron_reuse / functional.norm_repetition'})
|
|
write_json(out/'external-activation-cache.json',{'host':'8.211.154.134','files':external,'not_model_weights':True,
|
|
'reason':'Large intermediate caches retained on the authorized server; all figure source tables and token NLL arrays are delivered in Git.'})
|
|
shutil.copy2(root/'checksums-functional.json',out/'checksums-full-server.json')
|
|
(out/'DELIVERY.md').write_text('# Scientific delivery and retained caches\n\nAll figure CSV/JSON source data, per-token NLL, protocols, reports, metadata and historical logs are included.\nThree large reproducible intermediate activation caches remain on 8.211.154.134 and in the original full local archive;\n`external-activation-cache.json` records exact locations, sizes and SHA256. No weight or cache was deleted.\nThe original full experiment manifest is `checksums-full-server.json`; the delivery-only manifest is\n`checksums-functional.json`. No model checkpoint or tokenizer payload is in this Git delivery.\n')
|
|
validation=json.loads((out/'audit-validation.json').read_text());validation.update(delivery_excludes_external_activation_caches=True,full_experiment_checksums='checksums-full-server.json',figure_source_tables_and_token_nll_complete=True)
|
|
write_json(out/'audit-validation.json',validation)
|
|
checks={}
|
|
for file in sorted(out.rglob('*')):
|
|
if file.is_file() and file.name not in ('checksums-functional.json','audit-validation.json','archive-files.json'):
|
|
checks[str(file.relative_to(out))]=hashlib.sha256(file.read_bytes()).hexdigest()
|
|
write_json(out/'checksums-functional.json',checks)
|
|
write_json(out/'archive-files.json',[*checks,'checksums-functional.json','audit-validation.json','archive-files.json'])
|
|
print(json.dumps({'delivery_files':len(checks)+3,'checksummed_files':len(checks),'external_caches':len(external),'weights_and_caches_retained':True}))
|
|
|
|
|
|
if __name__=='__main__':main()
|