Validate repeated learned RMSNorm functions with CPU evidence

This commit is contained in:
2026-11-09 14:14:59 +08:00
parent 573cbedd98
commit 6d1acbc26a
108 changed files with 6526 additions and 33 deletions
+26 -4
View File
@@ -22,13 +22,32 @@ def main():
state.update(kwargs,supervisor_elapsed_seconds=time.monotonic()-start)
write_json(root/'supervisor-status.json',state)
def execute(module,args,timeout,log):
with (root/log).open('w') as out:
r=subprocess.run([sys.executable,'-B','-m',module,*args],stdout=out,stderr=subprocess.STDOUT,timeout=timeout)
ledger_path=root/'compute-budget-ledger.json'
ledger=json.loads(ledger_path.read_text()) if ledger_path.exists() else {'spent_seconds':0.,'calls':[]}
timeout=min(timeout,int(7200-ledger['spent_seconds']))
if timeout<1:raise TimeoutError('Cumulative two-hour compute budget exhausted')
t=time.monotonic()
try:
with (root/log).open('w') as out:
r=subprocess.run([sys.executable,'-B','-m',module,*args],stdout=out,stderr=subprocess.STDOUT,timeout=timeout)
finally:
elapsed=time.monotonic()-t
ledger['spent_seconds']+=elapsed
ledger['calls'].append({'module':module,'seconds':elapsed})
write_json(ledger_path,ledger)
if r.returncode:raise RuntimeError(f'{module} failed: exit {r.returncode}, see {log}')
status()
try:
while not all(importlib.util.find_spec(name) for name in ('torch','tokenizers')):
while True:
ready=False
if all(importlib.util.find_spec(name) for name in ('torch','tokenizers')):
try:
r=subprocess.run([sys.executable,'-B','-c','import torch, tokenizers; torch.empty(1)'],capture_output=True,timeout=30)
ready=r.returncode==0
except subprocess.TimeoutExpired:pass
if ready:break
if time.monotonic()-start>3600:raise TimeoutError('Binary reference dependencies not ready within one hour')
status()
time.sleep(15)
compute_start=time.monotonic()
status(state='running',phase='reference_check')
@@ -47,15 +66,18 @@ def main():
except (OSError,ValueError,KeyError):ready=False
if ready:break
if time.monotonic()-start>a.download_wait_seconds:raise TimeoutError('Verified weight download not ready before wait deadline')
status()
time.sleep(15)
status(state='running',phase='real_weight_preflight')
compute_start=time.monotonic()
execute('functional.preflight',['--root',str(root)],1200,'preflight.log')
preflight_seconds=time.monotonic()-compute_start
remaining=int(7200-reference_seconds-preflight_seconds-60)
remaining=int(7200-json.loads((root/'compute-budget-ledger.json').read_text())['spent_seconds']-60)
if remaining<60:raise TimeoutError('No remaining bounded functional computation budget')
status(phase='functional_analysis',preflight_seconds=preflight_seconds,remaining_seconds=remaining)
execute('functional.run',['--root',str(root),'--seconds',str(remaining)],remaining+15,'functional-run.log')
status(phase='storage_accounting')
execute('functional.storage',['--root',str(root)],120,'storage-accounting.log')
status(phase='report')
execute('functional.report',['--root',str(root)],120,'functional-report.log')
status(state='completed',phase='done')