Validate repeated learned RMSNorm functions with CPU evidence
This commit is contained in:
+24
-1
@@ -56,7 +56,8 @@ search extra candidates, mutate Git or delete weights. After reference checks, i
|
||||
8-token real text capacity check, then the bounded functional pilot. Reference, preflight
|
||||
and functional compute together are capped at two hours with a reporting reserve;
|
||||
download wait is recorded separately. Projection memory is checked, RSS soft stop is
|
||||
512 MiB, available-system-memory floor 256 MiB; sustained host swap IO also stops the run.
|
||||
512 MiB, available-system-memory floor 256 MiB; sustained host swap IO combined with low available memory or >128 MiB process swap also stops the run.
|
||||
A persistent ledger counts computation across restarts; early checks/preflight receive a conservative 180-second reserve.
|
||||
|
||||
`preflight.py` is a capacity smoke check, not a semantic oracle. `run.py` records exact
|
||||
sample/token IDs, fixed configuration, source hashes, activations, per-text/per-head metrics,
|
||||
@@ -66,3 +67,25 @@ saved data to produce PNG/PDF/SVG figures without reopening weights.
|
||||
No generation throughput is measured in this short-text pilot. Any parameter-tying storage
|
||||
benefit must be accounted for separately, including metadata. Neither same-input similarity
|
||||
nor favorable tiny heldout NLL demonstrates broad mutual interchangeability or GPU speedup.
|
||||
|
||||
## Current primary result and completed extensions
|
||||
|
||||
The latest user goal is learned-function repetition, independent of costs. Existing
|
||||
input RMSNorm modules at layer indices 12/13 have a proven worst two-direction real-arithmetic
|
||||
error bound of 11.3402%, worst observed heldout relative error 4.5285%, minimum cosine 0.999487,
|
||||
and small bidirectional heldout NLL changes. See [findings](../docs/CPU_FUNCTIONAL_FINDINGS.md).
|
||||
|
||||
- `neuron_reuse.py`: fixed four-layer scalar MLP function search, nontrivial activity
|
||||
gates, calibration-only selection and independent heldout/substitution logic; no pair passed.
|
||||
- `norm_repetition.py`: same-role learned RMSNorm bound across 552 pairs, one deterministic
|
||||
selection independent of any input samples, real probes and two full-network substitutions.
|
||||
- `gain_factorization.py`: architecture-derived identity on non-RoPE coordinates; a separate
|
||||
reparameterization result, not a learned-function repetition example.
|
||||
- `benchmark_factorization.py`: historical one-attempt CPU allocation optimization and
|
||||
bitwise equivalence check. Performance is no longer a success condition.
|
||||
- `storage.py`: explicit component layout accounting; the original candidate run selected
|
||||
no pair. This does not affect the primary function-repetition conclusion.
|
||||
|
||||
Do not expand the final positive claim to full attention or MLP bodies. Retain the
|
||||
fixed checkpoint until the user explicitly requests deletion. Full tests and audit are
|
||||
run remotely; no local build or model execution is required to inspect the archived results.
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Audit saved functional evidence and archive only outputs, never checkpoint weights."""
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import time
|
||||
import numpy as np
|
||||
from .model import StreamingLM
|
||||
from .norm_repetition import uniform_bounds
|
||||
from .run import write_json
|
||||
|
||||
EXCLUDE={'tokenizer.json','reference.py','checksums-functional.json','audit-validation.json','archive-files.json'}
|
||||
|
||||
|
||||
def archive_files(root):
|
||||
files=[p for p in root.iterdir() if p.is_file() and p.name not in EXCLUDE and not p.name.endswith(('.partial','.safetensors'))]
|
||||
files += [p for p in (root/'figures').glob('*') if p.is_file()]
|
||||
return sorted(files)
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);a=p.parse_args();root=a.root
|
||||
manifest=json.loads((root/'manifest.json').read_text())
|
||||
assert manifest['revision']=='15852e8c16360a2fea060d615a32b45270f8a8fc'
|
||||
for f in manifest['files']:
|
||||
if f['name'].endswith('.safetensors'):
|
||||
path=root/'weights'/f['name']
|
||||
assert path.is_file() and path.stat().st_size==f['bytes']
|
||||
assert f['sha256']==f['upstream_sha256']
|
||||
retention=json.loads((root/'retention-policy.json').read_text());assert retention['automatic_cleanup'] is False
|
||||
reference=json.loads((root/'reference-check.json').read_text())
|
||||
assert reference['passed'] and len(reference['tests'])==22
|
||||
assert reference['implementation_sha256']==hashlib.sha256(Path('functional/model.py').read_bytes()).hexdigest()
|
||||
tests=(root/'final-tests.txt').read_text();assert 'Ran 12 tests' in tests and '\nOK\n' in tests
|
||||
result=json.loads((root/'norm-repetition-summary.json').read_text());assert result['state']=='completed'
|
||||
assert result['learned_function_repetition_positive'] and result['bidirectional_network_quality_pass']
|
||||
cfg=json.loads((root/'model_config.json').read_text());lm=StreamingLM(root/'weights',cfg,seconds=60)
|
||||
c=result['selected'];ga=1+lm.small(f'layers.{c["a"]}.{c["role"]}.weight');gb=1+lm.small(f'layers.{c["b"]}.{c["role"]}.weight')
|
||||
bounds=uniform_bounds(ga,gb)
|
||||
np.testing.assert_allclose(bounds,[c['a_to_b_uniform_bound'],c['b_to_a_uniform_bound']],atol=1e-14,rtol=1e-14)
|
||||
with (root/'norm-repetition-uniform-bounds.csv').open() as f:assert len(list(csv.DictReader(f)))==552
|
||||
baseline=np.load(root/'baseline-heldout-token-nll.npy');assert baseline.shape==(4,31)
|
||||
for rec in result['interventions']:
|
||||
nll=np.load(root/f'norm-repetition-{rec["target"]}-uses-{rec["donor"]}-token-nll.npy')
|
||||
delta=nll.mean(1)-baseline.mean(1)
|
||||
np.testing.assert_allclose(delta,rec['sequence_nll_deltas'],atol=1e-13,rtol=1e-13)
|
||||
assert abs(delta.mean()-rec['mean_nll_delta'])<1e-13
|
||||
with (root/'norm-repetition-function-metrics.csv').open() as f:rows=list(csv.DictReader(f))
|
||||
held=[r for r in rows if r['split']=='heldout'];assert len(held)==8
|
||||
worst=max(max(float(r['relative_l2']),float(r['reverse_relative_l2'])) for r in held)
|
||||
assert abs(worst-result['worst_heldout_relative_l2'])<1e-14
|
||||
neuron=json.loads((root/'neuron-reuse-search-summary.json').read_text());assert sum(x['eligible_unique_pairs_checked'] for x in neuron)==7450139
|
||||
ledger=json.loads((root/'compute-budget-ledger.json').read_text());assert ledger['spent_seconds']+600<7200
|
||||
environment={'python':platform.python_version(),'platform':platform.platform(),'numpy':np.__version__,
|
||||
'packages':subprocess.run([str(Path(__import__('sys').executable)),'-m','pip','freeze'],capture_output=True,text=True,check=True).stdout.splitlines(),
|
||||
'weights_retained':True,'no_automatic_cleanup':True,'ledger_seconds_including_initial_reserve':ledger['spent_seconds'],
|
||||
'final_tests_report_audit_additional_budget_upper_reserve_seconds':600,
|
||||
'historical_error_logs':'Early reference adapter errors and preflight pressure stops; latest summaries/tests are authoritative.'}
|
||||
write_json(root/'functional-environment.json',environment)
|
||||
source_paths=[*Path('functional').glob('*.py'),Path('functional/reference.json'),Path('pilot/tensors.py'),Path('tests/test_functional.py')]
|
||||
write_json(root/'final-source-hashes.json',{str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source_paths)})
|
||||
files=archive_files(root)
|
||||
# Every numeric CSV value must be finite. Empty diagnostic fields stay unknown.
|
||||
for path in files:
|
||||
if path.suffix=='.csv':
|
||||
with path.open() as f:
|
||||
for row in csv.DictReader(f):
|
||||
for value in row.values():
|
||||
if value in (None,''):continue
|
||||
try:number=float(value)
|
||||
except ValueError:continue
|
||||
assert np.isfinite(number),(path,value)
|
||||
checksums={str(p.relative_to(root)):hashlib.sha256(p.read_bytes()).hexdigest() for p in files}
|
||||
write_json(root/'checksums-functional.json',checksums)
|
||||
selected=[*checksums,'checksums-functional.json','audit-validation.json','archive-files.json']
|
||||
write_json(root/'archive-files.json',selected)
|
||||
write_json(root/'audit-validation.json',{'passed':True,'validated_unix':time.time(),'checksummed_files':len(checksums),
|
||||
'proof_recomputed_from_retained_weights':True,'heldout_nll_recomputed_from_saved_arrays':True,
|
||||
'weights_present_and_retained':True,'archive_excludes_weights_and_tokenizer':True})
|
||||
print(json.dumps({'passed':True,'checksummed_files':len(checksums),'weights_retained':True}))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""One allocation-reduction attempt, with a similarly optimized baseline.
|
||||
|
||||
Does not change weights or run the network again. Checks bitwise equality to the
|
||||
network-tested factorization on every captured real Q/K input before timing.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import time
|
||||
import numpy as np
|
||||
from .model import StreamingLM
|
||||
from .gain_factorization import consolidate,normalized,LAYERS
|
||||
from .run import save_csv,write_json
|
||||
|
||||
|
||||
def baseline_inplace(q,k,a,b):
|
||||
q=normalized(q);q*=a
|
||||
k=normalized(k);k*=b
|
||||
return q,k
|
||||
|
||||
|
||||
def factored_inplace(q,k,a,b,r,combined,move='q_to_k'):
|
||||
q,k=normalized(q),normalized(k)
|
||||
if move=='q_to_k':
|
||||
q[...,:r]*=a[:r];k*=combined
|
||||
else:
|
||||
q*=combined;k[...,:r]*=b[:r]
|
||||
return q,k
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);a=p.parse_args();root=a.root
|
||||
ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text());remaining=int(7200-ledger['spent_seconds']-20)
|
||||
if remaining<60:raise TimeoutError('Cumulative budget insufficient')
|
||||
start=time.monotonic();result={'state':'running'}
|
||||
try:
|
||||
cfg=json.loads((root/'model_config.json').read_text());lm=StreamingLM(root/'weights',cfg,seconds=min(remaining,300))
|
||||
r=64;data=np.load(root/'baseline-activations.npz');checks=[]
|
||||
for layer in LAYERS:
|
||||
lm.guard();pfx=f'layers.{layer}.self_attn.'
|
||||
ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight')
|
||||
q,k=data[f'qraw_{layer}'],data[f'kraw_{layer}']
|
||||
for move in ('q_to_k','k_to_q'):
|
||||
combined=np.concatenate([(bg if move=='q_to_k' else ag)[:r],ag[r:]*bg[r:]])
|
||||
expected=consolidate(q,k,ag,bg,r,move)
|
||||
actual=factored_inplace(q,k,ag,bg,r,combined,move)
|
||||
for x,y in zip(expected,actual):np.testing.assert_array_equal(x,y)
|
||||
checks.append({'layer':layer,'direction':move,'query_elements':int(q.size),'key_elements':int(k.size),'bitwise_equal':True})
|
||||
q,k=data['qraw_3'][:1],data['kraw_3'][:1];pfx='layers.3.self_attn.'
|
||||
ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight')
|
||||
combined=np.concatenate([bg[:r],ag[r:]*bg[r:]])
|
||||
funcs={'baseline_legacy':lambda:(normalized(q)*ag,normalized(k)*bg),
|
||||
'baseline_inplace':lambda:baseline_inplace(q,k,ag,bg),
|
||||
'factored_inplace':lambda:factored_inplace(q,k,ag,bg,r,combined)}
|
||||
for _ in range(3):
|
||||
for fn in funcs.values():fn()
|
||||
timing=[];names=list(funcs)
|
||||
for repeat in range(30):
|
||||
order=names[repeat%3:]+names[:repeat%3]
|
||||
for name in order:
|
||||
lm.guard();t=time.perf_counter()
|
||||
for _ in range(50):funcs[name]()
|
||||
timing.append({'repeat':repeat,'implementation':name,'seconds_per_call':(time.perf_counter()-t)/50,'calls_per_block':50})
|
||||
save_csv(root/'gain-factorization-optimized-timing.csv',timing)
|
||||
values={name:np.array([row['seconds_per_call'] for row in timing if row['implementation']==name]) for name in names}
|
||||
ratio=values['baseline_inplace']/values['factored_inplace']
|
||||
medians={name:float(np.median(v)) for name,v in values.items()}
|
||||
result.update(state='completed',bitwise_checks=checks,medians_seconds=medians,
|
||||
paired_baseline_over_factored_median=float(np.median(ratio)),
|
||||
paired_ratio_p10=float(np.quantile(ratio,.1)),paired_ratio_p90=float(np.quantile(ratio,.9)),
|
||||
fraction_blocks_factored_faster=float(np.mean(ratio>1)),
|
||||
descriptive_local_gain=bool(np.median(ratio)>1.05 and np.mean(ratio>1)>=.8 and np.quantile(ratio,.1)>1),
|
||||
claim_limit='Hot-array CPU normalization microbenchmark against equally allocation-optimized baseline; no full-model speedup or independent statistical significance claim.')
|
||||
except Exception as exc:
|
||||
result.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise
|
||||
finally:
|
||||
elapsed=time.monotonic()-start;result.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
||||
ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.benchmark_factorization','seconds':elapsed});write_json(ledger_path,ledger)
|
||||
write_json(root/'gain-factorization-optimized-summary.json',result)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Architecture-derived Q/K gain consolidation on non-RoPE dimensions only.
|
||||
|
||||
Fixed layer3 interventions; no fitting, layer selection or learned alignment.
|
||||
Run only after baseline/candidate pilot finishes, within the same cumulative budget.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import time
|
||||
import numpy as np
|
||||
from .model import StreamingLM,norm,rope
|
||||
from .run import probability_metrics,write_json,save_csv
|
||||
|
||||
LAYERS=[3,7,11,15,19,23]
|
||||
|
||||
|
||||
def normalized(x):
|
||||
return x/np.sqrt(np.mean(x*x,axis=-1,keepdims=True)+np.float32(1e-6))
|
||||
|
||||
|
||||
def consolidate(q,k,a,b,r,move,joint=None):
|
||||
q,k=normalized(q),normalized(k)
|
||||
c=a[r:]*b[r:] if joint is None else joint
|
||||
if move=='q_to_k':
|
||||
q=np.concatenate([q[...,:r]*a[:r],q[...,r:]],axis=-1)
|
||||
k=np.concatenate([k[...,:r]*b[:r],k[...,r:]*c],axis=-1)
|
||||
else:
|
||||
q=np.concatenate([q[...,:r]*a[:r],q[...,r:]*c],axis=-1)
|
||||
k=np.concatenate([k[...,:r]*b[:r],k[...,r:]],axis=-1)
|
||||
return q,k
|
||||
|
||||
|
||||
class FactoredLM(StreamingLM):
|
||||
def configure(self,layer,move):
|
||||
self.factor_layer=layer;self.move=move
|
||||
p=f'layers.{layer}.self_attn.'
|
||||
self.a=1+self.small(p+'q_norm.weight');self.b=1+self.small(p+'k_norm.weight')
|
||||
self.r=int(self.config['head_dim']*self.config['rope_parameters']['partial_rotary_factor'])
|
||||
self.joint=self.a[self.r:]*self.b[self.r:]
|
||||
|
||||
def normalization(self,x,name):
|
||||
p=f'layers.{self.factor_layer}.self_attn.'
|
||||
if name not in (p+'q_norm',p+'k_norm'):return super().normalization(x,name)
|
||||
# This path does not multiply the unit-gain tail, unlike a full-vector mask.
|
||||
unit=x/np.sqrt(np.mean(x*x,axis=-1,keepdims=True)+self.eps)
|
||||
isq=name==p+'q_norm';head=self.a if isq else self.b
|
||||
skip=(isq and self.move=='q_to_k') or (not isq and self.move=='k_to_q')
|
||||
tail=unit[...,self.r:] if skip else unit[...,self.r:]*self.joint
|
||||
return np.concatenate([unit[...,:self.r]*head[:self.r],tail],axis=-1)
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);a=p.parse_args();root=a.root
|
||||
# Protocol is fixed before inspecting functional measurements.
|
||||
protocol={'fixed_intervention_layer':3,'directions':['q_to_k','k_to_q'],'diagnostic_layers':LAYERS,'no_selection':True,'no_training':True,'precision':'FP32 product; no BF16 requantization','quality_gate':{'mean_nll_delta':.01,'sequence_p95_nll_delta':.05}}
|
||||
protocol_path=root/'gain-factorization-protocol.json'
|
||||
if protocol_path.exists():
|
||||
old=json.loads(protocol_path.read_text());assert all(old[k]==v for k,v in protocol.items()), 'Protocol differs from preregistration'
|
||||
else:write_json(protocol_path,{**protocol,'registered_unix':time.time()})
|
||||
ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text())
|
||||
remaining=int(7200-ledger['spent_seconds']-30)
|
||||
if remaining<60:raise TimeoutError('No remaining cumulative budget for deterministic identity verification')
|
||||
start=time.monotonic()
|
||||
report={'state':'running','records':[],'interventions':[]}
|
||||
try:
|
||||
cfg=json.loads((root/'model_config.json').read_text());c=cfg['text_config'];r=int(c['head_dim']*c['rope_parameters']['partial_rotary_factor'])
|
||||
assert c['rms_norm_eps']==1e-6
|
||||
assert c['head_dim']==256 and r==64 and c['num_attention_heads']==8 and c['num_key_value_heads']==2
|
||||
lm=FactoredLM(root/'weights',cfg,seconds=remaining)
|
||||
data=np.load(root/'baseline-activations.npz');samples=json.loads((root/'samples-tokenized.json').read_text())['samples']
|
||||
rp=c['rope_parameters'];repeats=c['num_attention_heads']//c['num_key_value_heads']
|
||||
records=[]
|
||||
for layer in LAYERS:
|
||||
lm.guard()
|
||||
pfx=f'layers.{layer}.self_attn.'
|
||||
aw=lm.small(pfx+'q_norm.weight');bw=lm.small(pfx+'k_norm.weight');ag,bg=1+aw,1+bw
|
||||
q,k=data[f'qraw_{layer}'],data[f'kraw_{layer}']
|
||||
qr,kr=rope(norm(q,aw),rp['rope_theta'],rp['partial_rotary_factor']),rope(norm(k,bw),rp['rope_theta'],rp['partial_rotary_factor'])
|
||||
ref=np.einsum('bthd,bshd->bhts',qr,np.repeat(kr,repeats,axis=2))/np.float32(np.sqrt(c['head_dim']))
|
||||
mask=np.triu(np.ones(ref.shape[-2:],dtype=bool),1)
|
||||
def probs(s):
|
||||
s=np.where(mask,-np.inf,s);e=np.exp(s-s.max(-1,keepdims=True));return e/e.sum(-1,keepdims=True)
|
||||
pref=probs(ref)
|
||||
for move in ('q_to_k','k_to_q'):
|
||||
qq,kk=consolidate(q,k,ag,bg,r,move)
|
||||
qq,kk=rope(qq,rp['rope_theta'],rp['partial_rotary_factor']),rope(kk,rp['rope_theta'],rp['partial_rotary_factor'])
|
||||
changed=np.einsum('bthd,bshd->bhts',qq,np.repeat(kk,repeats,axis=2))/np.float32(np.sqrt(c['head_dim']))
|
||||
pp=probs(changed)
|
||||
for bi,s in enumerate(samples):
|
||||
diff=changed[bi].astype(np.float64)-ref[bi]
|
||||
records.append({'layer':layer,'direction':move,'sample_id':s['id'],'split':s['split'],
|
||||
'score_max_abs_error':float(np.abs(diff).max()),'score_relative_l2':float(np.linalg.norm(diff)/max(np.linalg.norm(ref[bi].astype(np.float64)),1e-12)),
|
||||
'probability_max_abs_error':float(np.abs(pp[bi]-pref[bi]).max()),**probability_metrics(pref[bi:bi+1],pp[bi:bi+1])})
|
||||
save_csv(root/'gain-factorization-score-metrics.csv',records)
|
||||
held=np.array([i for i,s in enumerate(samples) if s['split']=='heldout']);ids=np.array([s['token_ids'] for s in samples],dtype=np.int64)
|
||||
base=np.load(root/'baseline-heldout-token-nll.npy').mean(1)
|
||||
for move in ('q_to_k','k_to_q'):
|
||||
lm.configure(3,move)
|
||||
h,_=lm.forward(ids[held])
|
||||
nll=lm.nll(h,ids[held]);np.save(root/f'gain-factorization-{move}-token-nll.npy',nll)
|
||||
delta=nll.mean(1)-base
|
||||
report['interventions'].append({'layer':3,'direction':move,'sequence_nll_deltas':delta.tolist(),'mean_nll_delta':float(delta.mean()),'sequence_p95_delta':float(np.quantile(delta,.95)),'passed':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05)})
|
||||
write_json(root/'gain-factorization-summary.json',report)
|
||||
# Local normalization timing uses precomputed gains for both implementations.
|
||||
q,k=data['qraw_3'][:1],data['kraw_3'][:1];pfx='layers.3.self_attn.'
|
||||
ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight')
|
||||
def baseline():return normalized(q)*ag,normalized(k)*bg
|
||||
joint=ag[r:]*bg[r:]
|
||||
def factored():return consolidate(q,k,ag,bg,r,'q_to_k',joint=joint)
|
||||
for _ in range(3):baseline();factored()
|
||||
timing=[]
|
||||
for repeat in range(10):
|
||||
for label,fn in (('baseline',baseline),('factored',factored)) if repeat%2==0 else (('factored',factored),('baseline',baseline)):
|
||||
lm.guard();t=time.perf_counter()
|
||||
for _ in range(20):fn()
|
||||
timing.append({'repeat':repeat,'implementation':label,'seconds_per_call':(time.perf_counter()-t)/20})
|
||||
save_csv(root/'gain-factorization-kernel-timing.csv',timing)
|
||||
report.update(state='completed',maximum_score_abs_error=max(x['score_max_abs_error'] for x in records),maximum_probability_abs_error=max(x['probability_max_abs_error'] for x in records),
|
||||
bidirectional_heldout_pass=all(x['passed'] for x in report['interventions']),
|
||||
elementwise_gain_multiplies_saved_per_token_layer={'q_to_k':1536,'k_to_q':384},
|
||||
original_bf16_payload_bytes=1024,factored_lossless_mixed_payload_bytes=1024,
|
||||
fp32_cached_gain_payload_before=2048,fp32_cached_gain_payload_after=1280,
|
||||
claim_limit='Exact real-arithmetic identity only on non-RoPE coordinates; empirical FP32/NLL checks on tiny text pilot. Not evidence that two existing full layers interchange, not model-wide speedup.')
|
||||
except Exception as exc:
|
||||
report.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise
|
||||
finally:
|
||||
elapsed=time.monotonic()-start
|
||||
report.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
||||
ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.gain_factorization','seconds':elapsed});write_json(ledger_path,ledger)
|
||||
write_json(root/'gain-factorization-summary.json',report)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
+7
-2
@@ -6,6 +6,7 @@ No cached decode, padding, images, generation, fused kernels or autograd.
|
||||
Full-sequence gated delta recurrence replaces the algebraically equivalent
|
||||
chunk algorithm; both are independently compared in remote tests.
|
||||
"""
|
||||
import ctypes
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -87,6 +88,7 @@ class StreamingLM:
|
||||
self.qk_inputs = {}
|
||||
self.swap_observation = None
|
||||
self.swap_pressure_windows = 0
|
||||
self.malloc_trim = getattr(ctypes.CDLL(None), 'malloc_trim', None)
|
||||
|
||||
def guard(self):
|
||||
if time.monotonic() > self.deadline:
|
||||
@@ -104,8 +106,9 @@ class StreamingLM:
|
||||
elapsed = now-self.swap_observation[0]
|
||||
rate = (count-self.swap_observation[1])*4096/elapsed
|
||||
self.swap_pressure_windows = self.swap_pressure_windows+1 if rate > 4*1024**2 else 0
|
||||
if self.swap_pressure_windows >= 3:
|
||||
raise MemoryError('Sustained host swap IO above 4 MiB/s in three observation windows')
|
||||
own_swap_kib = int(status.get('VmSwap', '0 kB').split()[0])
|
||||
if self.swap_pressure_windows >= 3 and (available < 384*1024 or own_swap_kib > 128*1024):
|
||||
raise MemoryError(f'Sustained host swap pressure plus constrained memory: available={available} KiB, process_swap={own_swap_kib} KiB')
|
||||
self.swap_observation = (now,count)
|
||||
if rss_kib > 512*1024 or available < 256*1024:
|
||||
raise MemoryError(f'Soft memory stop: RSS {rss_kib} KiB, available {available} KiB')
|
||||
@@ -130,6 +133,8 @@ class StreamingLM:
|
||||
result[:, start:end] = flat @ w.T
|
||||
del w
|
||||
t.evict()
|
||||
if self.malloc_trim is not None:
|
||||
self.malloc_trim(0) # Release only this process's freed temporary heap pages.
|
||||
return result.reshape(*x.shape[:-1], outdim)
|
||||
|
||||
def embedding(self, ids):
|
||||
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Find repeated learned scalar MLP functions on real hidden inputs, no training.
|
||||
|
||||
Four fixed layers; calibration-only selection; one frozen pair; independent heldout
|
||||
functional test and two actual network substitutions. No scale fitting or R.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import time
|
||||
import numpy as np
|
||||
from .model import StreamingLM,silu
|
||||
from .run import write_json,save_csv
|
||||
|
||||
LAYERS=[5,6,20,21]
|
||||
|
||||
|
||||
def pair_metrics(x,y):
|
||||
x,y=x.astype(np.float64),y.astype(np.float64)
|
||||
xx,yy=float(x@x),float(y@y);delta=float((x-y)@(x-y))
|
||||
xc,yc=x-x.mean(),y-y.mean()
|
||||
return {'relative_l2_a_to_b':float(np.sqrt(delta/max(xx,1e-30))),
|
||||
'relative_l2_b_to_a':float(np.sqrt(delta/max(yy,1e-30))),
|
||||
'cosine':float(x@y/max(np.sqrt(xx*yy),1e-30)),
|
||||
'pearson':float(xc@yc/max(np.linalg.norm(xc)*np.linalg.norm(yc),1e-30)),
|
||||
'rms_a':float(np.sqrt(xx/x.size)),'rms_b':float(np.sqrt(yy/y.size)),
|
||||
'std_over_rms_a':float(np.std(x)/max(np.sqrt(xx/x.size),1e-30)),
|
||||
'std_over_rms_b':float(np.std(y)/max(np.sqrt(yy/y.size),1e-30)),
|
||||
'effective_energy_tokens_a':float(xx*xx/max(float((x**4).sum()),1e-30)),
|
||||
'effective_energy_tokens_b':float(yy*yy/max(float((y**4).sum()),1e-30))}
|
||||
|
||||
|
||||
class NeuronReuseLM(StreamingLM):
|
||||
def configure(self,layer,target,donor):self.reused_layer,self.target,self.donor=layer,target,donor
|
||||
def mlp(self,h,layer):
|
||||
p=f'layers.{layer}.mlp.'
|
||||
z=silu(self.linear(h,p+'gate_proj'))*self.linear(h,p+'up_proj')
|
||||
if layer==self.reused_layer:z[...,self.target]=z[...,self.donor]
|
||||
return self.linear(z,p+'down_proj')
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);a=p.parse_args();root=a.root
|
||||
protocol={'layers':LAYERS,'unit':'post_silu_gate_times_up_scalar_function; keep each output column',
|
||||
'thresholds':[.05,.10,.20],'maximum_directional_relative_l2':.20,'cosine_min':.98,'pearson_min':.95,
|
||||
'rms_floor':'at least within-layer median calibration neuron RMS','std_over_rms_min':.25,
|
||||
'effective_energy_tokens_min':16,'max_selected_pairs':1,'selection':'smallest worst directional calibration error; then layer/id',
|
||||
'heldout_per_text_error_max':.35,'heldout_quality_mean_nll_delta_max':.01,'heldout_quality_sequence_p95_delta_max':.05,
|
||||
'training':False,'scale_fitting':False,'cost_required':False}
|
||||
protocol_path=root/'neuron-reuse-protocol.json'
|
||||
if protocol_path.exists():
|
||||
old=json.loads(protocol_path.read_text());assert all(old[k]==v for k,v in protocol.items())
|
||||
else:write_json(protocol_path,{**protocol,'registered_unix':time.time()})
|
||||
ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text());remaining=int(7200-ledger['spent_seconds']-30)
|
||||
if remaining<120:raise TimeoutError('Remaining cumulative budget insufficient')
|
||||
started=time.monotonic();result={'state':'running','selected':None,'function_pass':False,'interventions':[]}
|
||||
try:
|
||||
cfg=json.loads((root/'model_config.json').read_text());lm=NeuronReuseLM(root/'weights',cfg,seconds=remaining)
|
||||
data=np.load(root/'baseline-activations.npz');samples=json.loads((root/'samples-tokenized.json').read_text())['samples']
|
||||
cal=np.array([i for i,s in enumerate(samples) if s['split']=='calibration']);held=np.array([i for i,s in enumerate(samples) if s['split']=='heldout'])
|
||||
activations={};summaries=[];best=None
|
||||
for layer in LAYERS:
|
||||
h=data[f'mlp_{layer}'];pfx=f'layers.{layer}.mlp.'
|
||||
z=silu(lm.linear(h,pfx+'gate_proj'))*lm.linear(h,pfx+'up_proj');activations[layer]=z
|
||||
# Profiles use all calibration tokens; never use heldout in selection.
|
||||
x=np.ascontiguousarray(z[cal].reshape(-1,z.shape[-1]).T,dtype=np.float64)
|
||||
n=x.shape[1];normsq=(x*x).sum(1);means=x.mean(1);variance=np.maximum(normsq-n*means**2,0)
|
||||
rms=np.sqrt(normsq/n);floor=float(np.median(rms));effective=normsq**2/np.maximum((x**4).sum(1),1e-30)
|
||||
active=(rms>=floor)&(np.sqrt(variance/np.maximum(normsq,1e-30))>=.25)&(effective>=16)
|
||||
counts={str(t):0 for t in protocol['thresholds']};tested=0;layerbest=None
|
||||
for start in range(0,x.shape[0],256):
|
||||
lm.guard();end=min(start+256,x.shape[0]);dot=x[start:end]@x.T
|
||||
delta=np.maximum(normsq[start:end,None]+normsq[None,:]-2*dot,0)
|
||||
error=np.sqrt(delta/np.maximum(np.minimum(normsq[start:end,None],normsq[None,:]),1e-30))
|
||||
cosine=dot/np.maximum(np.sqrt(normsq[start:end,None]*normsq[None,:]),1e-30)
|
||||
pearson=(dot-n*means[start:end,None]*means[None,:])/np.maximum(np.sqrt(variance[start:end,None]*variance[None,:]),1e-30)
|
||||
unique=np.arange(start,end)[:,None]<np.arange(x.shape[0])[None,:]
|
||||
allowed=unique&active[start:end,None]&active[None,:]&(cosine>=.98)&(pearson>=.95)
|
||||
tested+=int((unique&active[start:end,None]&active[None,:]).sum())
|
||||
for threshold in protocol['thresholds']:counts[str(threshold)]+=int((allowed&(error<=threshold)).sum())
|
||||
mask=allowed&(error<=.20)
|
||||
if mask.any():
|
||||
score=np.where(mask,error,np.inf);row,col=np.unravel_index(np.argmin(score),score.shape)
|
||||
candidate={'layer':layer,'a':int(start+row),'b':int(col),'calibration_worst_relative_l2':float(score[row,col])}
|
||||
key=lambda c:(c['calibration_worst_relative_l2'],c['layer'],c['a'],c['b'])
|
||||
if layerbest is None or key(candidate)<key(layerbest):layerbest=candidate
|
||||
if best is None or key(candidate)<key(best):best=candidate
|
||||
summaries.append({'layer':layer,'neurons':int(x.shape[0]),'eligible_neurons':int(active.sum()),'eligible_unique_pairs_checked':tested,'rms_floor':floor,'calibration_pair_counts':counts,'best':layerbest})
|
||||
write_json(root/'neuron-reuse-search-summary.json',summaries)
|
||||
print(json.dumps(summaries[-1]),flush=True)
|
||||
np.savez(root/'neuron-reuse-activations.npz',**{f'layer{k}':v for k,v in activations.items()})
|
||||
write_json(root/'neuron-reuse-selection.json',{'selected':best,'selection_uses_heldout':False})
|
||||
result['selected']=best
|
||||
if best is not None:
|
||||
layer,i,j=best['layer'],best['a'],best['b'];z=activations[layer]
|
||||
metrics=[]
|
||||
for split,index in [('calibration',cal),('heldout',held)]:
|
||||
metrics.append({'split':split,'scope':'pooled',**pair_metrics(z[index,:,i].reshape(-1),z[index,:,j].reshape(-1))})
|
||||
for idx in index:metrics.append({'split':split,'scope':samples[idx]['id'],**pair_metrics(z[idx,:,i],z[idx,:,j])})
|
||||
save_csv(root/'neuron-reuse-function-metrics.csv',metrics)
|
||||
pooled=next(r for r in metrics if r['split']=='heldout' and r['scope']=='pooled')
|
||||
floor=next(r['rms_floor'] for r in summaries if r['layer']==layer)
|
||||
function_pass=(max(pooled['relative_l2_a_to_b'],pooled['relative_l2_b_to_a'])<=.20 and pooled['cosine']>=.98 and pooled['pearson']>=.95 and
|
||||
min(pooled['std_over_rms_a'],pooled['std_over_rms_b'])>=.25 and min(pooled['effective_energy_tokens_a'],pooled['effective_energy_tokens_b'])>=16 and
|
||||
min(pooled['rms_a'],pooled['rms_b'])>=floor and
|
||||
all(max(r['relative_l2_a_to_b'],r['relative_l2_b_to_a'])<=.35 for r in metrics if r['split']=='heldout' and r['scope']!='pooled'))
|
||||
result.update(function_pass=bool(function_pass),heldout_function_metrics=pooled)
|
||||
# Show parameter similarity separately from actual functional similarity.
|
||||
wd=[]
|
||||
for role in ('gate_proj','up_proj'):
|
||||
t=lm.tensors[f'model.language_model.layers.{layer}.mlp.{role}.weight'];d=t.shape[1]
|
||||
wa,wb=t.read(slice(i*d,(i+1)*d)),t.read(slice(j*d,(j+1)*d))
|
||||
wd.append({'role':role,'weight_cosine':float(wa.astype(np.float64)@wb/max(np.linalg.norm(wa.astype(np.float64))*np.linalg.norm(wb.astype(np.float64)),1e-30))});t.evict()
|
||||
result['weight_diagnostics']=wd
|
||||
if function_pass:
|
||||
ids=np.array([s['token_ids'] for s in samples],dtype=np.int64);base=np.load(root/'baseline-heldout-token-nll.npy').mean(1)
|
||||
for target,donor in ((i,j),(j,i)):
|
||||
lm.configure(layer,target,donor);h,_=lm.forward(ids[held]);nll=lm.nll(h,ids[held])
|
||||
np.save(root/f'neuron-reuse-{target}-uses-{donor}-token-nll.npy',nll)
|
||||
delta=nll.mean(1)-base
|
||||
result['interventions'].append({'target':target,'donor':donor,'sequence_nll_deltas':delta.tolist(),'mean_nll_delta':float(delta.mean()),'sequence_p95_delta':float(np.quantile(delta,.95)),'passed':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05)})
|
||||
write_json(root/'neuron-reuse-summary.json',result)
|
||||
# No fallback after looking at heldout.
|
||||
result.update(state='completed',empirical_repeated_function_positive=bool(result['function_pass'] and len(result['interventions'])==2 and all(r['passed'] for r in result['interventions'])),
|
||||
claim_limit='Distinct learned scalar MLP subfunctions on the specified short-text distribution; not whole-layer duplication, global equivalence or broad task-capability proof. No performance requirement.')
|
||||
except Exception as exc:result.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise
|
||||
finally:
|
||||
elapsed=time.monotonic()-started;result.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
||||
ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.neuron_reuse','seconds':elapsed});write_json(ledger_path,ledger)
|
||||
write_json(root/'neuron-reuse-summary.json',result)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,108 @@
|
||||
"""Existing learned RMSNorm modules: uniform function bound + independent probes.
|
||||
|
||||
Deterministic exhaustive selection by a mathematical all-input error bound, not
|
||||
calibration/heldout fitting. Same role, shape and eps; one pair only.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import resource
|
||||
import time
|
||||
import numpy as np
|
||||
from .model import StreamingLM,norm
|
||||
from .run import write_json,save_csv,error_rows
|
||||
|
||||
ROLES=['input_layernorm','post_attention_layernorm']
|
||||
|
||||
|
||||
def uniform_bounds(a,b):
|
||||
if np.any(a==0) or np.any(b==0):return None
|
||||
diff=np.abs(a.astype(np.float64)-b.astype(np.float64))
|
||||
return float(np.max(diff/np.abs(a))),float(np.max(diff/np.abs(b)))
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);args=p.parse_args();root=args.root
|
||||
protocol={'roles':ROLES,'layers':list(range(24)),'same_role_only':True,'same_eps':1e-6,
|
||||
'uniform_worst_error_limit':.20,'empirical_function_worst_relative_error_limit':.05,
|
||||
'selection':'smallest maximum two-direction uniform bound, then role/layer; no input data used',
|
||||
'minimum_gain_standard_deviation':.001,'max_selected_pairs':1,'heldout_nll_mean_delta_limit':.01,
|
||||
'heldout_nll_sequence_p95_delta_limit':.05,'training':False,'cost_required':False}
|
||||
path=root/'norm-repetition-protocol.json'
|
||||
if path.exists():
|
||||
existing=json.loads(path.read_text());assert all(existing[k]==v for k,v in protocol.items())
|
||||
else:write_json(path,{**protocol,'registered_unix':time.time()})
|
||||
ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text());remaining=int(7200-ledger['spent_seconds']-30)
|
||||
if remaining<120:raise TimeoutError('Insufficient cumulative computation budget')
|
||||
started=time.monotonic();result={'state':'running','selected':None,'interventions':[]}
|
||||
try:
|
||||
cfg=json.loads((root/'model_config.json').read_text());lm=StreamingLM(root/'weights',cfg,seconds=remaining)
|
||||
assert lm.eps==1e-6
|
||||
candidates=[];gains={}
|
||||
for role in ROLES:
|
||||
for layer in range(24):gains[(role,layer)]=1+lm.small(f'layers.{layer}.{role}.weight')
|
||||
for i in range(24):
|
||||
for j in range(i+1,24):
|
||||
a,b=gains[(role,i)],gains[(role,j)]
|
||||
bounds=uniform_bounds(a,b)
|
||||
if bounds is None:continue
|
||||
eligible=min(float(np.std(a)),float(np.std(b)))>=.001
|
||||
candidates.append({'role':role,'a':i,'b':j,'a_to_b_uniform_bound':bounds[0],'b_to_a_uniform_bound':bounds[1],
|
||||
'worst_uniform_bound':max(bounds),'eligible_nonconstant_gains':eligible,
|
||||
'gain_std_a':float(np.std(a)),'gain_std_b':float(np.std(b)),
|
||||
'gain_min_a':float(np.min(a)),'gain_min_b':float(np.min(b))})
|
||||
save_csv(root/'norm-repetition-uniform-bounds.csv',candidates)
|
||||
feasible=[c for c in candidates if c['eligible_nonconstant_gains'] and c['worst_uniform_bound']<=.20]
|
||||
counts={str(t):sum(c['eligible_nonconstant_gains'] and c['worst_uniform_bound']<=t for c in candidates) for t in (.05,.10,.20)}
|
||||
selected=sorted(feasible,key=lambda c:(c['worst_uniform_bound'],c['role'],c['a'],c['b']))[:1]
|
||||
write_json(root/'norm-repetition-selection.json',{'selected':selected,'uniform_bound_counts':counts,'selection_reads_no_input_data':True})
|
||||
result['uniform_bound_counts']=counts
|
||||
if selected:
|
||||
c=selected[0];result['selected']=c
|
||||
# Selection is frozen before reading tokenized inputs or their activations.
|
||||
samples=json.loads((root/'samples-tokenized.json').read_text())['samples'];ids=np.array([s['token_ids'] for s in samples],dtype=np.int64)
|
||||
held=np.array([i for i,s in enumerate(samples) if s['split']=='heldout'])
|
||||
kind='attn' if c['role']=='input_layernorm' else 'mlp'
|
||||
captures=[(kind,c['a']),(kind,c['b'])]
|
||||
h,saved=lm.forward(ids,capture=captures)
|
||||
# The untouched recomputation must reproduce the saved real baseline.
|
||||
original=np.load(root/'baseline-activations.npz')['final']
|
||||
np.testing.assert_array_equal(h,original)
|
||||
np.savez(root/'norm-repetition-input-profiles.npz',**saved)
|
||||
ga,gb=gains[(c['role'],c['a'])],gains[(c['role'],c['b'])]
|
||||
rows=[]
|
||||
for source in (c['a'],c['b']):
|
||||
n=saved[f'{kind}_{source}'];g=gains[(c['role'],source)]
|
||||
# n = u * gamma_source. Reconstruct common normalized input u;
|
||||
# source gains are nonzero. Additional rounding is ~FP32 only.
|
||||
unit=n/g
|
||||
fa,fb=unit*ga,unit*gb
|
||||
rec=error_rows(fa,fb,samples,{'source_layer':source,'role':c['role'],'a':c['a'],'b':c['b']})
|
||||
rows.extend(rec)
|
||||
save_csv(root/'norm-repetition-function-metrics.csv',rows)
|
||||
heldrows=[r for r in rows if r['split']=='heldout']
|
||||
function_pass=all(max(r['relative_l2'],r['reverse_relative_l2'])<=.05 for r in heldrows)
|
||||
result.update(uniform_function_similarity_proven=True,heldout_function_pass=bool(function_pass),
|
||||
worst_heldout_relative_l2=max(max(r['relative_l2'],r['reverse_relative_l2']) for r in heldrows),
|
||||
worst_heldout_cosine=min(r['cosine'] for r in heldrows),untouched_full_forward_bitwise_matches_baseline=True)
|
||||
if function_pass:
|
||||
baseline=np.load(root/'baseline-heldout-token-nll.npy').mean(1)
|
||||
for target,donor in ((c['a'],c['b']),(c['b'],c['a'])):
|
||||
lm.norm_replacement=(f'layers.{target}.{c["role"]}',f'layers.{donor}.{c["role"]}')
|
||||
changed,_=lm.forward(ids[held]);nll=lm.nll(changed,ids[held]);np.save(root/f'norm-repetition-{target}-uses-{donor}-token-nll.npy',nll)
|
||||
delta=nll.mean(1)-baseline
|
||||
result['interventions'].append({'target':target,'donor':donor,'sequence_nll_deltas':delta.tolist(),
|
||||
'mean_nll_delta':float(delta.mean()),'sequence_p95_nll_delta':float(np.quantile(delta,.95)),
|
||||
'passed_quality':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05)})
|
||||
write_json(root/'norm-repetition-summary.json',result)
|
||||
result['learned_function_repetition_positive']=bool(function_pass)
|
||||
result['bidirectional_network_quality_pass']=bool(len(result['interventions'])==2 and all(x['passed_quality'] for x in result['interventions']))
|
||||
result.update(state='completed',claim_limit='Existing learned RMSNorm module functions with identical eps and shape; uniform real-arithmetic relative bound and tiny real-text probes. Does not establish attention/MLP-body duplication or broad task capability.')
|
||||
except Exception as exc:result.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise
|
||||
finally:
|
||||
elapsed=time.monotonic()-started;result.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
|
||||
ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.norm_repetition','seconds':elapsed});write_json(ledger_path,ledger)
|
||||
write_json(root/'norm-repetition-summary.json',result)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Package code-independent scientific outputs, retain large activation caches externally.
|
||||
|
||||
The experiment directory remains immutable and complete. The delivery directory
|
||||
contains all figure source tables/NLL arrays plus hashes/locations for three large
|
||||
reproducible activation caches. Never deletes model weights or cache files.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from .run import write_json
|
||||
|
||||
CACHES={'baseline-activations.npz','neuron-reuse-activations.npz','norm-repetition-input-profiles.npz'}
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);p.add_argument('--output',required=True,type=Path);a=p.parse_args()
|
||||
root,out=a.root.resolve(),a.output.resolve();assert root!=out
|
||||
out.mkdir(parents=True,exist_ok=True)
|
||||
full=json.loads((root/'checksums-functional.json').read_text())
|
||||
files=json.loads((root/'archive-files.json').read_text())
|
||||
for name in files:
|
||||
src=root/name
|
||||
if name in CACHES:continue
|
||||
dest=out/name;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,dest)
|
||||
external=[]
|
||||
for name in sorted(CACHES):
|
||||
src=root/name;digest=hashlib.sha256(src.read_bytes()).hexdigest();assert digest==full[name]
|
||||
external.append({'name':name,'server_path':str(src),'bytes':src.stat().st_size,'sha256':digest,'retained':True,'reproducible_by':'functional.run / functional.neuron_reuse / functional.norm_repetition'})
|
||||
write_json(out/'external-activation-cache.json',{'host':'8.211.154.134','files':external,'not_model_weights':True,
|
||||
'reason':'Large intermediate caches retained on the authorized server; all figure source tables and token NLL arrays are delivered in Git.'})
|
||||
shutil.copy2(root/'checksums-functional.json',out/'checksums-full-server.json')
|
||||
(out/'DELIVERY.md').write_text('# Scientific delivery and retained caches\n\nAll figure CSV/JSON source data, per-token NLL, protocols, reports, metadata and historical logs are included.\nThree large reproducible intermediate activation caches remain on 8.211.154.134 and in the original full local archive;\n`external-activation-cache.json` records exact locations, sizes and SHA256. No weight or cache was deleted.\nThe original full experiment manifest is `checksums-full-server.json`; the delivery-only manifest is\n`checksums-functional.json`. No model checkpoint or tokenizer payload is in this Git delivery.\n')
|
||||
validation=json.loads((out/'audit-validation.json').read_text());validation.update(delivery_excludes_external_activation_caches=True,full_experiment_checksums='checksums-full-server.json',figure_source_tables_and_token_nll_complete=True)
|
||||
write_json(out/'audit-validation.json',validation)
|
||||
checks={}
|
||||
for file in sorted(out.rglob('*')):
|
||||
if file.is_file() and file.name not in ('checksums-functional.json','audit-validation.json','archive-files.json'):
|
||||
checks[str(file.relative_to(out))]=hashlib.sha256(file.read_bytes()).hexdigest()
|
||||
write_json(out/'checksums-functional.json',checks)
|
||||
write_json(out/'archive-files.json',[*checks,'checksums-functional.json','audit-validation.json','archive-files.json'])
|
||||
print(json.dumps({'delivery_files':len(checks)+3,'checksummed_files':len(checks),'external_caches':len(external),'weights_and_caches_retained':True}))
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
+20
-8
@@ -14,14 +14,26 @@ def main():
|
||||
samples,ids=tokenize(a.root)
|
||||
start=time.time()
|
||||
lm=StreamingLM(a.root/'weights',json.loads((a.root/'model_config.json').read_text()),seconds=1200)
|
||||
def progress(layer):print(json.dumps({'preflight_layer':layer,'seconds':time.time()-start}),flush=True)
|
||||
h,_=lm.forward(ids[:1,:8],progress=progress)
|
||||
nll=lm.nll(h,ids[:1,:8])
|
||||
if not np.isfinite(nll).all():raise FloatingPointError('Nonfinite NLL')
|
||||
write_json(a.root/'preflight.json',{'passed':True,'sample_id':samples[0]['id'],'tokens':8,
|
||||
'elapsed_seconds':time.time()-start,'peak_rss_kib':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
|
||||
'mean_nll':float(nll.mean()),'token_nll':nll.tolist(),
|
||||
'limitation':'Real-weight capacity check, not an independent correctness oracle or quality benchmark'})
|
||||
snapshots=[]
|
||||
result={'passed':False,'sample_id':samples[0]['id'],'tokens':8}
|
||||
def progress(layer):
|
||||
status=dict(line.split(':',1) for line in Path('/proc/self/status').read_text().splitlines() if ':' in line)
|
||||
snapshots.append({'layer':layer,'seconds':time.time()-start,'rss_kib':int(status['VmRSS'].split()[0]),
|
||||
'process_swap_kib':int(status.get('VmSwap','0 kB').split()[0]),
|
||||
'available_kib':int(next(line.split()[1] for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemAvailable:')))})
|
||||
print(json.dumps(snapshots[-1]),flush=True)
|
||||
try:
|
||||
h,_=lm.forward(ids[:1,:8],progress=progress)
|
||||
nll=lm.nll(h,ids[:1,:8])
|
||||
if not np.isfinite(nll).all():raise FloatingPointError('Nonfinite NLL')
|
||||
result.update(passed=True,mean_nll=float(nll.mean()),token_nll=nll.tolist())
|
||||
except Exception as exc:
|
||||
result.update(error_type=type(exc).__name__,error=str(exc))
|
||||
raise
|
||||
finally:
|
||||
result.update(elapsed_seconds=time.time()-start,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
|
||||
resource_snapshots=snapshots,limitation='Real-weight capacity check, not an independent correctness oracle or quality benchmark')
|
||||
write_json(a.root/'preflight.json',result)
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
|
||||
@@ -14,7 +14,7 @@ import numpy as np
|
||||
from .model import StreamingLM, delta_rule, rope, norm
|
||||
|
||||
EXPECTED_COMMIT = 'fd3cf0140de03f7619ac95b5cfed05b711ff9e22'
|
||||
NAMES = {'l2norm', 'torch_chunk_gated_delta_rule', 'torch_recurrent_gated_delta_rule',
|
||||
NAMES = {'apply_mask_to_padding_states', 'l2norm', 'torch_chunk_gated_delta_rule', 'torch_recurrent_gated_delta_rule',
|
||||
'causal_conv1d_fn', 'Qwen3_5RMSNormGated', 'Qwen3_5GatedDeltaNet',
|
||||
'rotate_half', 'apply_rotary_pos_emb', 'repeat_kv', 'eager_attention_forward',
|
||||
'Qwen3_5Attention', 'Qwen3_5MLP', 'Qwen3_5RMSNorm', 'Qwen3_5DecoderLayer',
|
||||
@@ -30,7 +30,7 @@ def official_namespace(source):
|
||||
if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in NAMES:
|
||||
for child in ast.walk(node):
|
||||
if isinstance(child, (ast.FunctionDef, ast.ClassDef)):
|
||||
child.decorator_list = []
|
||||
child.decorator_list = [d for d in child.decorator_list if isinstance(d, ast.Name) and d.id in ('staticmethod','classmethod')]
|
||||
nodes.append(node)
|
||||
assert {x.name for x in nodes} == NAMES
|
||||
tree = ast.Module(body=[ast.ImportFrom(module='__future__',names=[ast.alias(name='annotations')],level=0),*nodes],type_ignores=[])
|
||||
@@ -134,7 +134,7 @@ def main():
|
||||
actual,_=lm.forward(ids)
|
||||
check('complete_two_layer_text_forward',actual,expected)
|
||||
result={'passed':True,'reference_commit':EXPECTED_COMMIT,'source_sha256':hashlib.sha256(a.source.read_bytes()).hexdigest(),
|
||||
'torch':torch.__version__,'numpy':np.__version__,'tests':evidence,'limitations':'FP32 CPU semantics on tiny synthetic configurations; not full-checkpoint BF16 kernel equivalence'}
|
||||
'torch':torch.__version__,'numpy':np.__version__,'implementation_sha256':hashlib.sha256(Path(__file__).with_name('model.py').read_bytes()).hexdigest(),'tests':evidence,'limitations':'FP32 CPU semantics on tiny synthetic configurations; not full-checkpoint BF16 kernel equivalence'}
|
||||
a.output.write_text(json.dumps(result,indent=2))
|
||||
print(json.dumps({'passed':True,'checks':len(evidence),'maximum_absolute_discrepancy':max(x['max_abs_error'] for x in evidence)}))
|
||||
|
||||
|
||||
+64
-9
@@ -23,9 +23,9 @@ def main():
|
||||
ax.bar(np.arange(len(c)),[max(v['maximum_symmetric_relative_error'],1e-8) for v in c],color=['#238b45' if v['passed'] else '#9e9e9e' for v in c])
|
||||
ax.axhline(.05,color='#b2182b',linestyle='--',label='Relative L2 screen = 0.05; other gates also apply')
|
||||
ax.set_yscale('log');ax.set_xticks(np.arange(len(c)),[v['id'] for v in c],rotation=45,ha='right')
|
||||
ax.set_ylabel('Worst calibration symmetric relative L2')
|
||||
ax.set_ylabel('Worst calibration directional relative L2')
|
||||
ax.set_title('Fixed candidates; same-input function comparison\n4 calibration texts, 32 tokens; no training')
|
||||
ax.legend(loc='upper left',bbox_to_anchor=(0,1.01),fontsize=8);fig.tight_layout();save(fig,'candidate_function_errors')
|
||||
ax.legend(loc='upper left',bbox_to_anchor=(0,-.38),fontsize=8);fig.tight_layout();save(fig,'candidate_function_errors')
|
||||
if (root/'interventions.json').exists():
|
||||
interventions=json.loads((root/'interventions.json').read_text())
|
||||
fig,ax=plt.subplots(figsize=(8,4.5))
|
||||
@@ -36,13 +36,68 @@ def main():
|
||||
ax.set_xticks(np.arange(4),['heldout zh1','heldout zh2','heldout en1','heldout en2'])
|
||||
ax.set_ylabel('Token NLL change (nat/token)');ax.set_title('Single-site interventions; frozen candidate\n4 heldout texts, 32 tokens; limited pilot evidence')
|
||||
ax.legend(loc='upper left',bbox_to_anchor=(0,-.18),fontsize=8);fig.tight_layout();save(fig,'heldout_intervention_nll')
|
||||
text=['# CPU functional reuse pilot','',f'- State: {status["state"]}',f'- Positive bidirectional pilot: {status.get("positive_bidirectional_quality",False)}',
|
||||
f'- Elapsed analysis seconds: {status.get("elapsed_seconds")}',f'- Peak RSS KiB: {status.get("peak_rss_kib")}',
|
||||
'- Original BF16 weights decoded into FP32 CPU operators, no training.',
|
||||
'- Calibration and heldout samples are distinct but very small, short and experiment-authored.',
|
||||
'- A positive result is limited to these empirical tests, not global equivalence or GPU speedup.',
|
||||
'- Weight retention: keep until explicit user deletion request; no automatic cleanup.','']
|
||||
if status['state']!='completed':text+=['## Stop evidence','',f'{status.get("error_type")}: {status.get("error")}','']
|
||||
norm_path=root/'norm-repetition-summary.json'
|
||||
repetition=json.loads(norm_path.read_text()) if norm_path.exists() else {}
|
||||
if (root/'norm-repetition-function-metrics.csv').exists():
|
||||
with (root/'norm-repetition-function-metrics.csv').open() as f:rows=list(csv.DictReader(f))
|
||||
sample_ids=list(dict.fromkeys(row['sample_id'] for row in rows))
|
||||
fig,ax=plt.subplots(figsize=(9,4.8))
|
||||
for source in sorted({row['source_layer'] for row in rows},key=int):
|
||||
rr=[r for r in rows if r['source_layer']==source]
|
||||
ax.plot(range(len(rr)),[max(float(r['relative_l2']),float(r['reverse_relative_l2'])) for r in rr],'o-',label=f'Input captured at layer {source}')
|
||||
ax.axhline(.05,color='#b2182b',ls='--',label='Heldout empirical screen = 0.05')
|
||||
chosen=repetition.get('selected')
|
||||
if chosen:ax.axhline(chosen['worst_uniform_bound'],color='#6a51a3',ls=':',label='Proven uniform all-input bound')
|
||||
ax.axvline(3.5,color='gray',linewidth=.8)
|
||||
ax.set_xticks(range(len(sample_ids)),sample_ids,rotation=30,ha='right');ax.set_ylabel('Worst of two directional relative L2 errors')
|
||||
ax.set_title('Learned input RMSNorm at layer indices 12/13: same-input comparison\nCalibration (left) / independent heldout texts (right), 32 tokens each')
|
||||
ax.legend(loc='upper left',bbox_to_anchor=(0,-.25),fontsize=8);fig.tight_layout();save(fig,'learned_norm_function_errors')
|
||||
if repetition.get('interventions'):
|
||||
fig,ax=plt.subplots(figsize=(8,4.5))
|
||||
for v in repetition['interventions']:
|
||||
ax.plot(range(4),v['sequence_nll_deltas'],'o-',label=f'Layer {v["target"]} uses {v["donor"]}; mean={v["mean_nll_delta"]:.5g}')
|
||||
ax.axhline(0,color='gray',linewidth=.8);ax.set_xticks(range(4),['zh1','zh2','en1','en2'])
|
||||
ax.set_xlabel('Independent heldout text (32 tokens)');ax.set_ylabel('Mean token NLL change (nat/token)')
|
||||
ax.set_title('Bidirectional single-site substitution of existing learned RMSNorm modules')
|
||||
ax.legend(loc='upper left',bbox_to_anchor=(0,-.18),fontsize=8);fig.tight_layout();save(fig,'learned_norm_network_substitution')
|
||||
if (root/'gain-factorization-score-metrics.csv').exists():
|
||||
with (root/'gain-factorization-score-metrics.csv').open() as f:rr=list(csv.DictReader(f))
|
||||
layers=sorted({int(v['layer']) for v in rr});fig,axes=plt.subplots(1,2,figsize=(10,4))
|
||||
for ax,key,label in zip(axes,['score_max_abs_error','probability_max_abs_error'],['Maximum absolute score error','Maximum absolute probability error']):
|
||||
for direction in ('q_to_k','k_to_q'):
|
||||
ax.plot(layers,[max(float(v[key]) for v in rr if int(v['layer'])==layer and v['direction']==direction) for layer in layers],'o-',label=direction)
|
||||
ax.set_xlabel('Full-attention layer index');ax.set_xticks(layers);ax.set_ylabel(label);ax.ticklabel_format(axis='y',style='sci',scilimits=(0,0));ax.legend(loc='upper left',fontsize=8)
|
||||
fig.suptitle('Supplement: algebraic reparameterization, separate from learned-function repetition');fig.tight_layout();save(fig,'algebraic_gain_identity')
|
||||
neuron_path=root/'neuron-reuse-summary.json'
|
||||
neuron=json.loads(neuron_path.read_text()) if neuron_path.exists() else {}
|
||||
text=['# CPU learned-function repetition pilot','',
|
||||
'Primary question: do two distinct trained parts implement similar functions? Cost is not a success condition.',
|
||||
f'- Existing learned RMSNorm function positive: {repetition.get("learned_function_repetition_positive",False)}',
|
||||
f'- Bidirectional network quality screen passed: {repetition.get("bidirectional_network_quality_pass",False)}',
|
||||
f'- Learned scalar MLP neuron search positive: {neuron.get("empirical_repeated_function_positive",False)}',
|
||||
f'- Original 11 candidate run state: {status["state"]}; selected: {status.get("selected_candidate",[])}',
|
||||
'- All model weights are retained on the server until the user explicitly requests deletion.','']
|
||||
if repetition.get('selected'):
|
||||
c=repetition['selected'];text += ['## Existing learned module evidence','',
|
||||
f'Role: {c["role"]}; layers {c["a"]} and {c["b"]}; each maps 2048-dimensional input to output.',
|
||||
f'Uniform real-arithmetic relative-error bound, first-to-second: {c["a_to_b_uniform_bound"]:.12g}; reverse: {c["b_to_a_uniform_bound"]:.12g}.',
|
||||
f'Maximum heldout directional output relative L2: {repetition.get("worst_heldout_relative_l2")}; minimum cosine: {repetition.get("worst_heldout_cosine")}.',
|
||||
'Same epsilon and nonzero gains: the common RMS denominator cancels. For any nonzero input x,',
|
||||
'`||f_a(x)-f_b(x)|| / ||f_a(x)|| <= max_j |a_j-b_j|/|a_j|`; swap a/b for the reverse direction.',
|
||||
'Selection used only the tight all-input bound across 552 same-role pairs, before inspecting inputs.',
|
||||
'Real probes use four calibration and four heldout experiment-authored texts, 32 tokens each, and both source layers.',
|
||||
'These are learned, nonconstant gain vectors. The claim concerns RMSNorm modules; it does not establish attention/MLP-body duplication.','']
|
||||
for v in repetition.get('interventions',[]):text += [f'- Layer {v["target"]} uses {v["donor"]}: mean heldout NLL delta {v["mean_nll_delta"]:.12g}; sequence P95 delta {v["sequence_p95_nll_delta"]:.12g}; screen pass={v["passed_quality"]}.']
|
||||
text += ['', '## Negative results and separate algebraic result','',
|
||||
'The original whole-attention, whole-MLP and within-layer Q/K norm candidates did not pass their gates.',
|
||||
'The scalar neuron search excludes low-RMS/near-constant profiles and selects at most one pair without heldout feedback; consult its search summary for coverage.',
|
||||
'Q/K non-RoPE gain consolidation is a separately verified real-arithmetic identity, not evidence that two existing learned modules duplicate functions.',
|
||||
'Earlier local CPU timing records are preserved for provenance; performance is outside the current success criterion.','',
|
||||
'## Limits and provenance','',
|
||||
'Original BF16 weights were decoded into streaming FP32 CPU operators; no training, scale fitting, R fitting or GPU.',
|
||||
'The numerical reference includes 22 independent official Torch comparisons. Early adapter errors and two resource-guard stops are preserved as historical logs.',
|
||||
'The persistent computation ledger includes a conservative 180-second reserve for early checks; no budget reset after that reserve.',
|
||||
'The text sample is tiny and short. Network NLL checks are not broad task-accuracy validation.','']
|
||||
(root/'FUNCTIONAL_REPORT.md').write_text('\n'.join(text))
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
"""Measure an explicit two-component alias storage layout, not inference speed.
|
||||
|
||||
Payloads remain on the server under weights/ and must not be committed or cleaned
|
||||
without the user's explicit deletion request. Manifest/byte/hash evidence may be archived.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
from pilot.tensors import load_tensors,decode
|
||||
from .run import write_json
|
||||
|
||||
|
||||
def payload(t,path):
|
||||
digest=hashlib.sha256()
|
||||
with path.open('wb') as f:
|
||||
for start in range(0,t.raw.size,262144):
|
||||
b=t.raw[start:start+262144].tobytes();f.write(b);digest.update(b)
|
||||
t.evict()
|
||||
return {'file':path.name,'dtype':t.dtype,'shape':list(t.shape),'bytes':path.stat().st_size,'sha256':digest.hexdigest()}
|
||||
|
||||
|
||||
def main():
|
||||
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);a=p.parse_args();root=a.root
|
||||
selected=json.loads((root/'selection.json').read_text())['selected']
|
||||
if not selected:
|
||||
write_json(root/'storage-evidence.json',{'measured':False,'reason':'No calibration candidate selected'});return
|
||||
c=selected[0]
|
||||
if c['kind']!='qk_norm':
|
||||
write_json(root/'storage-evidence.json',{'measured':False,'reason':'Only two-gain alias layout implemented in this bounded pilot; larger-module savings remain theoretical'});return
|
||||
tensors=load_tensors(root/'weights')
|
||||
prefix=f'model.language_model.layers.{c["i"]}.self_attn.'
|
||||
names=[prefix+'q_norm.weight',prefix+'k_norm.weight']
|
||||
# Original pair plus two alternative layouts; each is counted independently.
|
||||
path=root/'component-layout'/'weights';path.mkdir(parents=True,exist_ok=True)
|
||||
original={'format':'two_component_raw_v1','tensors':{}}
|
||||
for short,name in zip(('q','k'),names):original['tensors'][name]=payload(tensors[name],path/f'{short}.bin')
|
||||
original_bytes=json.dumps(original,separators=(',',':'),sort_keys=True).encode()
|
||||
(path/'original-layout.json').write_bytes(original_bytes)
|
||||
evidence=[]
|
||||
for target,donor in ((names[0],names[1]),(names[1],names[0])):
|
||||
entry=original['tensors'][donor]
|
||||
layout={'format':'two_component_raw_v1','tensors':{donor:entry},'aliases':{target:donor}}
|
||||
encoded=json.dumps(layout,separators=(',',':'),sort_keys=True).encode()
|
||||
file=path/('share-'+('k' if donor==names[1] else 'q')+'.json');file.write_bytes(encoded)
|
||||
t=tensors[donor]
|
||||
raw=np.fromfile(path/entry['file'],dtype=t.raw.dtype)
|
||||
reconstructed=decode(raw,t.dtype).reshape(t.shape)
|
||||
np.testing.assert_array_equal(reconstructed,t.read().reshape(t.shape));t.evict()
|
||||
before=sum(e['bytes'] for e in original['tensors'].values())+len(original_bytes)
|
||||
after=entry['bytes']+len(encoded)
|
||||
evidence.append({'target':target,'donor':donor,'original_payload_bytes':sum(e['bytes'] for e in original['tensors'].values()),
|
||||
'shared_payload_bytes':entry['bytes'],'original_descriptor_bytes':len(original_bytes),
|
||||
'shared_descriptor_bytes':len(encoded),'original_total_bytes':before,'shared_total_bytes':after,
|
||||
'net_saved_bytes':before-after,'shared_descriptor_sha256':hashlib.sha256(encoded).hexdigest(),
|
||||
'decoded_donor_exactly_matches_intervention_gain':True})
|
||||
write_json(root/'storage-evidence.json',{'measured':True,'candidate':c['id'],'layouts':evidence,
|
||||
'claim_limit':'Measured standalone component layout including descriptors, not an integrated compressed checkpoint, latency gain or full-model RSS reduction. Two sharing directions are alternatives and must not be added.'})
|
||||
|
||||
|
||||
if __name__=='__main__':main()
|
||||
+26
-4
@@ -22,13 +22,32 @@ def main():
|
||||
state.update(kwargs,supervisor_elapsed_seconds=time.monotonic()-start)
|
||||
write_json(root/'supervisor-status.json',state)
|
||||
def execute(module,args,timeout,log):
|
||||
with (root/log).open('w') as out:
|
||||
r=subprocess.run([sys.executable,'-B','-m',module,*args],stdout=out,stderr=subprocess.STDOUT,timeout=timeout)
|
||||
ledger_path=root/'compute-budget-ledger.json'
|
||||
ledger=json.loads(ledger_path.read_text()) if ledger_path.exists() else {'spent_seconds':0.,'calls':[]}
|
||||
timeout=min(timeout,int(7200-ledger['spent_seconds']))
|
||||
if timeout<1:raise TimeoutError('Cumulative two-hour compute budget exhausted')
|
||||
t=time.monotonic()
|
||||
try:
|
||||
with (root/log).open('w') as out:
|
||||
r=subprocess.run([sys.executable,'-B','-m',module,*args],stdout=out,stderr=subprocess.STDOUT,timeout=timeout)
|
||||
finally:
|
||||
elapsed=time.monotonic()-t
|
||||
ledger['spent_seconds']+=elapsed
|
||||
ledger['calls'].append({'module':module,'seconds':elapsed})
|
||||
write_json(ledger_path,ledger)
|
||||
if r.returncode:raise RuntimeError(f'{module} failed: exit {r.returncode}, see {log}')
|
||||
status()
|
||||
try:
|
||||
while not all(importlib.util.find_spec(name) for name in ('torch','tokenizers')):
|
||||
while True:
|
||||
ready=False
|
||||
if all(importlib.util.find_spec(name) for name in ('torch','tokenizers')):
|
||||
try:
|
||||
r=subprocess.run([sys.executable,'-B','-c','import torch, tokenizers; torch.empty(1)'],capture_output=True,timeout=30)
|
||||
ready=r.returncode==0
|
||||
except subprocess.TimeoutExpired:pass
|
||||
if ready:break
|
||||
if time.monotonic()-start>3600:raise TimeoutError('Binary reference dependencies not ready within one hour')
|
||||
status()
|
||||
time.sleep(15)
|
||||
compute_start=time.monotonic()
|
||||
status(state='running',phase='reference_check')
|
||||
@@ -47,15 +66,18 @@ def main():
|
||||
except (OSError,ValueError,KeyError):ready=False
|
||||
if ready:break
|
||||
if time.monotonic()-start>a.download_wait_seconds:raise TimeoutError('Verified weight download not ready before wait deadline')
|
||||
status()
|
||||
time.sleep(15)
|
||||
status(state='running',phase='real_weight_preflight')
|
||||
compute_start=time.monotonic()
|
||||
execute('functional.preflight',['--root',str(root)],1200,'preflight.log')
|
||||
preflight_seconds=time.monotonic()-compute_start
|
||||
remaining=int(7200-reference_seconds-preflight_seconds-60)
|
||||
remaining=int(7200-json.loads((root/'compute-budget-ledger.json').read_text())['spent_seconds']-60)
|
||||
if remaining<60:raise TimeoutError('No remaining bounded functional computation budget')
|
||||
status(phase='functional_analysis',preflight_seconds=preflight_seconds,remaining_seconds=remaining)
|
||||
execute('functional.run',['--root',str(root),'--seconds',str(remaining)],remaining+15,'functional-run.log')
|
||||
status(phase='storage_accounting')
|
||||
execute('functional.storage',['--root',str(root)],120,'storage-accounting.log')
|
||||
status(phase='report')
|
||||
execute('functional.report',['--root',str(root)],120,'functional-report.log')
|
||||
status(state='completed',phase='done')
|
||||
|
||||
Reference in New Issue
Block a user