Validate repeated learned RMSNorm functions with CPU evidence
This commit is contained in:
+26
-4
@@ -22,13 +22,32 @@ def main():
|
||||
state.update(kwargs,supervisor_elapsed_seconds=time.monotonic()-start)
|
||||
write_json(root/'supervisor-status.json',state)
|
||||
def execute(module,args,timeout,log):
|
||||
with (root/log).open('w') as out:
|
||||
r=subprocess.run([sys.executable,'-B','-m',module,*args],stdout=out,stderr=subprocess.STDOUT,timeout=timeout)
|
||||
ledger_path=root/'compute-budget-ledger.json'
|
||||
ledger=json.loads(ledger_path.read_text()) if ledger_path.exists() else {'spent_seconds':0.,'calls':[]}
|
||||
timeout=min(timeout,int(7200-ledger['spent_seconds']))
|
||||
if timeout<1:raise TimeoutError('Cumulative two-hour compute budget exhausted')
|
||||
t=time.monotonic()
|
||||
try:
|
||||
with (root/log).open('w') as out:
|
||||
r=subprocess.run([sys.executable,'-B','-m',module,*args],stdout=out,stderr=subprocess.STDOUT,timeout=timeout)
|
||||
finally:
|
||||
elapsed=time.monotonic()-t
|
||||
ledger['spent_seconds']+=elapsed
|
||||
ledger['calls'].append({'module':module,'seconds':elapsed})
|
||||
write_json(ledger_path,ledger)
|
||||
if r.returncode:raise RuntimeError(f'{module} failed: exit {r.returncode}, see {log}')
|
||||
status()
|
||||
try:
|
||||
while not all(importlib.util.find_spec(name) for name in ('torch','tokenizers')):
|
||||
while True:
|
||||
ready=False
|
||||
if all(importlib.util.find_spec(name) for name in ('torch','tokenizers')):
|
||||
try:
|
||||
r=subprocess.run([sys.executable,'-B','-c','import torch, tokenizers; torch.empty(1)'],capture_output=True,timeout=30)
|
||||
ready=r.returncode==0
|
||||
except subprocess.TimeoutExpired:pass
|
||||
if ready:break
|
||||
if time.monotonic()-start>3600:raise TimeoutError('Binary reference dependencies not ready within one hour')
|
||||
status()
|
||||
time.sleep(15)
|
||||
compute_start=time.monotonic()
|
||||
status(state='running',phase='reference_check')
|
||||
@@ -47,15 +66,18 @@ def main():
|
||||
except (OSError,ValueError,KeyError):ready=False
|
||||
if ready:break
|
||||
if time.monotonic()-start>a.download_wait_seconds:raise TimeoutError('Verified weight download not ready before wait deadline')
|
||||
status()
|
||||
time.sleep(15)
|
||||
status(state='running',phase='real_weight_preflight')
|
||||
compute_start=time.monotonic()
|
||||
execute('functional.preflight',['--root',str(root)],1200,'preflight.log')
|
||||
preflight_seconds=time.monotonic()-compute_start
|
||||
remaining=int(7200-reference_seconds-preflight_seconds-60)
|
||||
remaining=int(7200-json.loads((root/'compute-budget-ledger.json').read_text())['spent_seconds']-60)
|
||||
if remaining<60:raise TimeoutError('No remaining bounded functional computation budget')
|
||||
status(phase='functional_analysis',preflight_seconds=preflight_seconds,remaining_seconds=remaining)
|
||||
execute('functional.run',['--root',str(root),'--seconds',str(remaining)],remaining+15,'functional-run.log')
|
||||
status(phase='storage_accounting')
|
||||
execute('functional.storage',['--root',str(root)],120,'storage-accounting.log')
|
||||
status(phase='report')
|
||||
execute('functional.report',['--root',str(root)],120,'functional-report.log')
|
||||
status(state='completed',phase='done')
|
||||
|
||||
Reference in New Issue
Block a user