Validate repeated learned RMSNorm functions with CPU evidence

This commit is contained in:
2026-11-09 14:14:59 +08:00
parent 573cbedd98
commit 6d1acbc26a
108 changed files with 6526 additions and 33 deletions
+24 -1
View File
@@ -56,7 +56,8 @@ search extra candidates, mutate Git or delete weights. After reference checks, i
8-token real text capacity check, then the bounded functional pilot. Reference, preflight
and functional compute together are capped at two hours with a reporting reserve;
download wait is recorded separately. Projection memory is checked, RSS soft stop is
512 MiB, available-system-memory floor 256 MiB; sustained host swap IO also stops the run.
512 MiB, available-system-memory floor 256 MiB; sustained host swap IO combined with low available memory or >128 MiB process swap also stops the run.
A persistent ledger counts computation across restarts; early checks/preflight receive a conservative 180-second reserve.
`preflight.py` is a capacity smoke check, not a semantic oracle. `run.py` records exact
sample/token IDs, fixed configuration, source hashes, activations, per-text/per-head metrics,
@@ -66,3 +67,25 @@ saved data to produce PNG/PDF/SVG figures without reopening weights.
No generation throughput is measured in this short-text pilot. Any parameter-tying storage
benefit must be accounted for separately, including metadata. Neither same-input similarity
nor favorable tiny heldout NLL demonstrates broad mutual interchangeability or GPU speedup.
## Current primary result and completed extensions
The latest user goal is learned-function repetition, independent of costs. Existing
input RMSNorm modules at layer indices 12/13 have a proven worst two-direction real-arithmetic
error bound of 11.3402%, worst observed heldout relative error 4.5285%, minimum cosine 0.999487,
and small bidirectional heldout NLL changes. See [findings](../docs/CPU_FUNCTIONAL_FINDINGS.md).
- `neuron_reuse.py`: fixed four-layer scalar MLP function search, nontrivial activity
gates, calibration-only selection and independent heldout/substitution logic; no pair passed.
- `norm_repetition.py`: same-role learned RMSNorm bound across 552 pairs, one deterministic
selection independent of any input samples, real probes and two full-network substitutions.
- `gain_factorization.py`: architecture-derived identity on non-RoPE coordinates; a separate
reparameterization result, not a learned-function repetition example.
- `benchmark_factorization.py`: historical one-attempt CPU allocation optimization and
bitwise equivalence check. Performance is no longer a success condition.
- `storage.py`: explicit component layout accounting; the original candidate run selected
no pair. This does not affect the primary function-repetition conclusion.
Do not expand the final positive claim to full attention or MLP bodies. Retain the
fixed checkpoint until the user explicitly requests deletion. Full tests and audit are
run remotely; no local build or model execution is required to inspect the archived results.
+86
View File
@@ -0,0 +1,86 @@
"""Audit saved functional evidence and archive only outputs, never checkpoint weights."""
import argparse
import csv
import hashlib
import json
from pathlib import Path
import platform
import subprocess
import time
import numpy as np
from .model import StreamingLM
from .norm_repetition import uniform_bounds
from .run import write_json
EXCLUDE={'tokenizer.json','reference.py','checksums-functional.json','audit-validation.json','archive-files.json'}
def archive_files(root):
files=[p for p in root.iterdir() if p.is_file() and p.name not in EXCLUDE and not p.name.endswith(('.partial','.safetensors'))]
files += [p for p in (root/'figures').glob('*') if p.is_file()]
return sorted(files)
def main():
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);a=p.parse_args();root=a.root
manifest=json.loads((root/'manifest.json').read_text())
assert manifest['revision']=='15852e8c16360a2fea060d615a32b45270f8a8fc'
for f in manifest['files']:
if f['name'].endswith('.safetensors'):
path=root/'weights'/f['name']
assert path.is_file() and path.stat().st_size==f['bytes']
assert f['sha256']==f['upstream_sha256']
retention=json.loads((root/'retention-policy.json').read_text());assert retention['automatic_cleanup'] is False
reference=json.loads((root/'reference-check.json').read_text())
assert reference['passed'] and len(reference['tests'])==22
assert reference['implementation_sha256']==hashlib.sha256(Path('functional/model.py').read_bytes()).hexdigest()
tests=(root/'final-tests.txt').read_text();assert 'Ran 12 tests' in tests and '\nOK\n' in tests
result=json.loads((root/'norm-repetition-summary.json').read_text());assert result['state']=='completed'
assert result['learned_function_repetition_positive'] and result['bidirectional_network_quality_pass']
cfg=json.loads((root/'model_config.json').read_text());lm=StreamingLM(root/'weights',cfg,seconds=60)
c=result['selected'];ga=1+lm.small(f'layers.{c["a"]}.{c["role"]}.weight');gb=1+lm.small(f'layers.{c["b"]}.{c["role"]}.weight')
bounds=uniform_bounds(ga,gb)
np.testing.assert_allclose(bounds,[c['a_to_b_uniform_bound'],c['b_to_a_uniform_bound']],atol=1e-14,rtol=1e-14)
with (root/'norm-repetition-uniform-bounds.csv').open() as f:assert len(list(csv.DictReader(f)))==552
baseline=np.load(root/'baseline-heldout-token-nll.npy');assert baseline.shape==(4,31)
for rec in result['interventions']:
nll=np.load(root/f'norm-repetition-{rec["target"]}-uses-{rec["donor"]}-token-nll.npy')
delta=nll.mean(1)-baseline.mean(1)
np.testing.assert_allclose(delta,rec['sequence_nll_deltas'],atol=1e-13,rtol=1e-13)
assert abs(delta.mean()-rec['mean_nll_delta'])<1e-13
with (root/'norm-repetition-function-metrics.csv').open() as f:rows=list(csv.DictReader(f))
held=[r for r in rows if r['split']=='heldout'];assert len(held)==8
worst=max(max(float(r['relative_l2']),float(r['reverse_relative_l2'])) for r in held)
assert abs(worst-result['worst_heldout_relative_l2'])<1e-14
neuron=json.loads((root/'neuron-reuse-search-summary.json').read_text());assert sum(x['eligible_unique_pairs_checked'] for x in neuron)==7450139
ledger=json.loads((root/'compute-budget-ledger.json').read_text());assert ledger['spent_seconds']+600<7200
environment={'python':platform.python_version(),'platform':platform.platform(),'numpy':np.__version__,
'packages':subprocess.run([str(Path(__import__('sys').executable)),'-m','pip','freeze'],capture_output=True,text=True,check=True).stdout.splitlines(),
'weights_retained':True,'no_automatic_cleanup':True,'ledger_seconds_including_initial_reserve':ledger['spent_seconds'],
'final_tests_report_audit_additional_budget_upper_reserve_seconds':600,
'historical_error_logs':'Early reference adapter errors and preflight pressure stops; latest summaries/tests are authoritative.'}
write_json(root/'functional-environment.json',environment)
source_paths=[*Path('functional').glob('*.py'),Path('functional/reference.json'),Path('pilot/tensors.py'),Path('tests/test_functional.py')]
write_json(root/'final-source-hashes.json',{str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source_paths)})
files=archive_files(root)
# Every numeric CSV value must be finite. Empty diagnostic fields stay unknown.
for path in files:
if path.suffix=='.csv':
with path.open() as f:
for row in csv.DictReader(f):
for value in row.values():
if value in (None,''):continue
try:number=float(value)
except ValueError:continue
assert np.isfinite(number),(path,value)
checksums={str(p.relative_to(root)):hashlib.sha256(p.read_bytes()).hexdigest() for p in files}
write_json(root/'checksums-functional.json',checksums)
selected=[*checksums,'checksums-functional.json','audit-validation.json','archive-files.json']
write_json(root/'archive-files.json',selected)
write_json(root/'audit-validation.json',{'passed':True,'validated_unix':time.time(),'checksummed_files':len(checksums),
'proof_recomputed_from_retained_weights':True,'heldout_nll_recomputed_from_saved_arrays':True,
'weights_present_and_retained':True,'archive_excludes_weights_and_tokenizer':True})
print(json.dumps({'passed':True,'checksummed_files':len(checksums),'weights_retained':True}))
if __name__=='__main__':main()
+83
View File
@@ -0,0 +1,83 @@
"""One allocation-reduction attempt, with a similarly optimized baseline.
Does not change weights or run the network again. Checks bitwise equality to the
network-tested factorization on every captured real Q/K input before timing.
"""
import argparse
import json
from pathlib import Path
import resource
import time
import numpy as np
from .model import StreamingLM
from .gain_factorization import consolidate,normalized,LAYERS
from .run import save_csv,write_json
def baseline_inplace(q,k,a,b):
q=normalized(q);q*=a
k=normalized(k);k*=b
return q,k
def factored_inplace(q,k,a,b,r,combined,move='q_to_k'):
q,k=normalized(q),normalized(k)
if move=='q_to_k':
q[...,:r]*=a[:r];k*=combined
else:
q*=combined;k[...,:r]*=b[:r]
return q,k
def main():
p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);a=p.parse_args();root=a.root
ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text());remaining=int(7200-ledger['spent_seconds']-20)
if remaining<60:raise TimeoutError('Cumulative budget insufficient')
start=time.monotonic();result={'state':'running'}
try:
cfg=json.loads((root/'model_config.json').read_text());lm=StreamingLM(root/'weights',cfg,seconds=min(remaining,300))
r=64;data=np.load(root/'baseline-activations.npz');checks=[]
for layer in LAYERS:
lm.guard();pfx=f'layers.{layer}.self_attn.'
ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight')
q,k=data[f'qraw_{layer}'],data[f'kraw_{layer}']
for move in ('q_to_k','k_to_q'):
combined=np.concatenate([(bg if move=='q_to_k' else ag)[:r],ag[r:]*bg[r:]])
expected=consolidate(q,k,ag,bg,r,move)
actual=factored_inplace(q,k,ag,bg,r,combined,move)
for x,y in zip(expected,actual):np.testing.assert_array_equal(x,y)
checks.append({'layer':layer,'direction':move,'query_elements':int(q.size),'key_elements':int(k.size),'bitwise_equal':True})
q,k=data['qraw_3'][:1],data['kraw_3'][:1];pfx='layers.3.self_attn.'
ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight')
combined=np.concatenate([bg[:r],ag[r:]*bg[r:]])
funcs={'baseline_legacy':lambda:(normalized(q)*ag,normalized(k)*bg),
'baseline_inplace':lambda:baseline_inplace(q,k,ag,bg),
'factored_inplace':lambda:factored_inplace(q,k,ag,bg,r,combined)}
for _ in range(3):
for fn in funcs.values():fn()
timing=[];names=list(funcs)
for repeat in range(30):
order=names[repeat%3:]+names[:repeat%3]
for name in order:
lm.guard();t=time.perf_counter()
for _ in range(50):funcs[name]()
timing.append({'repeat':repeat,'implementation':name,'seconds_per_call':(time.perf_counter()-t)/50,'calls_per_block':50})
save_csv(root/'gain-factorization-optimized-timing.csv',timing)
values={name:np.array([row['seconds_per_call'] for row in timing if row['implementation']==name]) for name in names}
ratio=values['baseline_inplace']/values['factored_inplace']
medians={name:float(np.median(v)) for name,v in values.items()}
result.update(state='completed',bitwise_checks=checks,medians_seconds=medians,
paired_baseline_over_factored_median=float(np.median(ratio)),
paired_ratio_p10=float(np.quantile(ratio,.1)),paired_ratio_p90=float(np.quantile(ratio,.9)),
fraction_blocks_factored_faster=float(np.mean(ratio>1)),
descriptive_local_gain=bool(np.median(ratio)>1.05 and np.mean(ratio>1)>=.8 and np.quantile(ratio,.1)>1),
claim_limit='Hot-array CPU normalization microbenchmark against equally allocation-optimized baseline; no full-model speedup or independent statistical significance claim.')
except Exception as exc:
result.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise
finally:
elapsed=time.monotonic()-start;result.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.benchmark_factorization','seconds':elapsed});write_json(ledger_path,ledger)
write_json(root/'gain-factorization-optimized-summary.json',result)
if __name__=='__main__':main()
+135
View File
@@ -0,0 +1,135 @@
"""Architecture-derived Q/K gain consolidation on non-RoPE dimensions only.
Fixed layer3 interventions; no fitting, layer selection or learned alignment.
Run only after baseline/candidate pilot finishes, within the same cumulative budget.
"""
import argparse
import csv
import json
from pathlib import Path
import resource
import time
import numpy as np
from .model import StreamingLM,norm,rope
from .run import probability_metrics,write_json,save_csv
LAYERS=[3,7,11,15,19,23]
def normalized(x):
return x/np.sqrt(np.mean(x*x,axis=-1,keepdims=True)+np.float32(1e-6))
def consolidate(q,k,a,b,r,move,joint=None):
q,k=normalized(q),normalized(k)
c=a[r:]*b[r:] if joint is None else joint
if move=='q_to_k':
q=np.concatenate([q[...,:r]*a[:r],q[...,r:]],axis=-1)
k=np.concatenate([k[...,:r]*b[:r],k[...,r:]*c],axis=-1)
else:
q=np.concatenate([q[...,:r]*a[:r],q[...,r:]*c],axis=-1)
k=np.concatenate([k[...,:r]*b[:r],k[...,r:]],axis=-1)
return q,k
class FactoredLM(StreamingLM):
def configure(self,layer,move):
self.factor_layer=layer;self.move=move
p=f'layers.{layer}.self_attn.'
self.a=1+self.small(p+'q_norm.weight');self.b=1+self.small(p+'k_norm.weight')
self.r=int(self.config['head_dim']*self.config['rope_parameters']['partial_rotary_factor'])
self.joint=self.a[self.r:]*self.b[self.r:]
def normalization(self,x,name):
p=f'layers.{self.factor_layer}.self_attn.'
if name not in (p+'q_norm',p+'k_norm'):return super().normalization(x,name)
# This path does not multiply the unit-gain tail, unlike a full-vector mask.
unit=x/np.sqrt(np.mean(x*x,axis=-1,keepdims=True)+self.eps)
isq=name==p+'q_norm';head=self.a if isq else self.b
skip=(isq and self.move=='q_to_k') or (not isq and self.move=='k_to_q')
tail=unit[...,self.r:] if skip else unit[...,self.r:]*self.joint
return np.concatenate([unit[...,:self.r]*head[:self.r],tail],axis=-1)
def main():
p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);a=p.parse_args();root=a.root
# Protocol is fixed before inspecting functional measurements.
protocol={'fixed_intervention_layer':3,'directions':['q_to_k','k_to_q'],'diagnostic_layers':LAYERS,'no_selection':True,'no_training':True,'precision':'FP32 product; no BF16 requantization','quality_gate':{'mean_nll_delta':.01,'sequence_p95_nll_delta':.05}}
protocol_path=root/'gain-factorization-protocol.json'
if protocol_path.exists():
old=json.loads(protocol_path.read_text());assert all(old[k]==v for k,v in protocol.items()), 'Protocol differs from preregistration'
else:write_json(protocol_path,{**protocol,'registered_unix':time.time()})
ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text())
remaining=int(7200-ledger['spent_seconds']-30)
if remaining<60:raise TimeoutError('No remaining cumulative budget for deterministic identity verification')
start=time.monotonic()
report={'state':'running','records':[],'interventions':[]}
try:
cfg=json.loads((root/'model_config.json').read_text());c=cfg['text_config'];r=int(c['head_dim']*c['rope_parameters']['partial_rotary_factor'])
assert c['rms_norm_eps']==1e-6
assert c['head_dim']==256 and r==64 and c['num_attention_heads']==8 and c['num_key_value_heads']==2
lm=FactoredLM(root/'weights',cfg,seconds=remaining)
data=np.load(root/'baseline-activations.npz');samples=json.loads((root/'samples-tokenized.json').read_text())['samples']
rp=c['rope_parameters'];repeats=c['num_attention_heads']//c['num_key_value_heads']
records=[]
for layer in LAYERS:
lm.guard()
pfx=f'layers.{layer}.self_attn.'
aw=lm.small(pfx+'q_norm.weight');bw=lm.small(pfx+'k_norm.weight');ag,bg=1+aw,1+bw
q,k=data[f'qraw_{layer}'],data[f'kraw_{layer}']
qr,kr=rope(norm(q,aw),rp['rope_theta'],rp['partial_rotary_factor']),rope(norm(k,bw),rp['rope_theta'],rp['partial_rotary_factor'])
ref=np.einsum('bthd,bshd->bhts',qr,np.repeat(kr,repeats,axis=2))/np.float32(np.sqrt(c['head_dim']))
mask=np.triu(np.ones(ref.shape[-2:],dtype=bool),1)
def probs(s):
s=np.where(mask,-np.inf,s);e=np.exp(s-s.max(-1,keepdims=True));return e/e.sum(-1,keepdims=True)
pref=probs(ref)
for move in ('q_to_k','k_to_q'):
qq,kk=consolidate(q,k,ag,bg,r,move)
qq,kk=rope(qq,rp['rope_theta'],rp['partial_rotary_factor']),rope(kk,rp['rope_theta'],rp['partial_rotary_factor'])
changed=np.einsum('bthd,bshd->bhts',qq,np.repeat(kk,repeats,axis=2))/np.float32(np.sqrt(c['head_dim']))
pp=probs(changed)
for bi,s in enumerate(samples):
diff=changed[bi].astype(np.float64)-ref[bi]
records.append({'layer':layer,'direction':move,'sample_id':s['id'],'split':s['split'],
'score_max_abs_error':float(np.abs(diff).max()),'score_relative_l2':float(np.linalg.norm(diff)/max(np.linalg.norm(ref[bi].astype(np.float64)),1e-12)),
'probability_max_abs_error':float(np.abs(pp[bi]-pref[bi]).max()),**probability_metrics(pref[bi:bi+1],pp[bi:bi+1])})
save_csv(root/'gain-factorization-score-metrics.csv',records)
held=np.array([i for i,s in enumerate(samples) if s['split']=='heldout']);ids=np.array([s['token_ids'] for s in samples],dtype=np.int64)
base=np.load(root/'baseline-heldout-token-nll.npy').mean(1)
for move in ('q_to_k','k_to_q'):
lm.configure(3,move)
h,_=lm.forward(ids[held])
nll=lm.nll(h,ids[held]);np.save(root/f'gain-factorization-{move}-token-nll.npy',nll)
delta=nll.mean(1)-base
report['interventions'].append({'layer':3,'direction':move,'sequence_nll_deltas':delta.tolist(),'mean_nll_delta':float(delta.mean()),'sequence_p95_delta':float(np.quantile(delta,.95)),'passed':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05)})
write_json(root/'gain-factorization-summary.json',report)
# Local normalization timing uses precomputed gains for both implementations.
q,k=data['qraw_3'][:1],data['kraw_3'][:1];pfx='layers.3.self_attn.'
ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight')
def baseline():return normalized(q)*ag,normalized(k)*bg
joint=ag[r:]*bg[r:]
def factored():return consolidate(q,k,ag,bg,r,'q_to_k',joint=joint)
for _ in range(3):baseline();factored()
timing=[]
for repeat in range(10):
for label,fn in (('baseline',baseline),('factored',factored)) if repeat%2==0 else (('factored',factored),('baseline',baseline)):
lm.guard();t=time.perf_counter()
for _ in range(20):fn()
timing.append({'repeat':repeat,'implementation':label,'seconds_per_call':(time.perf_counter()-t)/20})
save_csv(root/'gain-factorization-kernel-timing.csv',timing)
report.update(state='completed',maximum_score_abs_error=max(x['score_max_abs_error'] for x in records),maximum_probability_abs_error=max(x['probability_max_abs_error'] for x in records),
bidirectional_heldout_pass=all(x['passed'] for x in report['interventions']),
elementwise_gain_multiplies_saved_per_token_layer={'q_to_k':1536,'k_to_q':384},
original_bf16_payload_bytes=1024,factored_lossless_mixed_payload_bytes=1024,
fp32_cached_gain_payload_before=2048,fp32_cached_gain_payload_after=1280,
claim_limit='Exact real-arithmetic identity only on non-RoPE coordinates; empirical FP32/NLL checks on tiny text pilot. Not evidence that two existing full layers interchange, not model-wide speedup.')
except Exception as exc:
report.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise
finally:
elapsed=time.monotonic()-start
report.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.gain_factorization','seconds':elapsed});write_json(ledger_path,ledger)
write_json(root/'gain-factorization-summary.json',report)
if __name__=='__main__':main()
+7 -2
View File
@@ -6,6 +6,7 @@ No cached decode, padding, images, generation, fused kernels or autograd.
Full-sequence gated delta recurrence replaces the algebraically equivalent
chunk algorithm; both are independently compared in remote tests.
"""
import ctypes
import json
import time
from pathlib import Path
@@ -87,6 +88,7 @@ class StreamingLM:
self.qk_inputs = {}
self.swap_observation = None
self.swap_pressure_windows = 0
self.malloc_trim = getattr(ctypes.CDLL(None), 'malloc_trim', None)
def guard(self):
if time.monotonic() > self.deadline:
@@ -104,8 +106,9 @@ class StreamingLM:
elapsed = now-self.swap_observation[0]
rate = (count-self.swap_observation[1])*4096/elapsed
self.swap_pressure_windows = self.swap_pressure_windows+1 if rate > 4*1024**2 else 0
if self.swap_pressure_windows >= 3:
raise MemoryError('Sustained host swap IO above 4 MiB/s in three observation windows')
own_swap_kib = int(status.get('VmSwap', '0 kB').split()[0])
if self.swap_pressure_windows >= 3 and (available < 384*1024 or own_swap_kib > 128*1024):
raise MemoryError(f'Sustained host swap pressure plus constrained memory: available={available} KiB, process_swap={own_swap_kib} KiB')
self.swap_observation = (now,count)
if rss_kib > 512*1024 or available < 256*1024:
raise MemoryError(f'Soft memory stop: RSS {rss_kib} KiB, available {available} KiB')
@@ -130,6 +133,8 @@ class StreamingLM:
result[:, start:end] = flat @ w.T
del w
t.evict()
if self.malloc_trim is not None:
self.malloc_trim(0) # Release only this process's freed temporary heap pages.
return result.reshape(*x.shape[:-1], outdim)
def embedding(self, ids):
+134
View File
@@ -0,0 +1,134 @@
"""Find repeated learned scalar MLP functions on real hidden inputs, no training.
Four fixed layers; calibration-only selection; one frozen pair; independent heldout
functional test and two actual network substitutions. No scale fitting or R.
"""
import argparse
import hashlib
import json
from pathlib import Path
import resource
import time
import numpy as np
from .model import StreamingLM,silu
from .run import write_json,save_csv
LAYERS=[5,6,20,21]
def pair_metrics(x,y):
x,y=x.astype(np.float64),y.astype(np.float64)
xx,yy=float(x@x),float(y@y);delta=float((x-y)@(x-y))
xc,yc=x-x.mean(),y-y.mean()
return {'relative_l2_a_to_b':float(np.sqrt(delta/max(xx,1e-30))),
'relative_l2_b_to_a':float(np.sqrt(delta/max(yy,1e-30))),
'cosine':float(x@y/max(np.sqrt(xx*yy),1e-30)),
'pearson':float(xc@yc/max(np.linalg.norm(xc)*np.linalg.norm(yc),1e-30)),
'rms_a':float(np.sqrt(xx/x.size)),'rms_b':float(np.sqrt(yy/y.size)),
'std_over_rms_a':float(np.std(x)/max(np.sqrt(xx/x.size),1e-30)),
'std_over_rms_b':float(np.std(y)/max(np.sqrt(yy/y.size),1e-30)),
'effective_energy_tokens_a':float(xx*xx/max(float((x**4).sum()),1e-30)),
'effective_energy_tokens_b':float(yy*yy/max(float((y**4).sum()),1e-30))}
class NeuronReuseLM(StreamingLM):
def configure(self,layer,target,donor):self.reused_layer,self.target,self.donor=layer,target,donor
def mlp(self,h,layer):
p=f'layers.{layer}.mlp.'
z=silu(self.linear(h,p+'gate_proj'))*self.linear(h,p+'up_proj')
if layer==self.reused_layer:z[...,self.target]=z[...,self.donor]
return self.linear(z,p+'down_proj')
def main():
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);a=p.parse_args();root=a.root
protocol={'layers':LAYERS,'unit':'post_silu_gate_times_up_scalar_function; keep each output column',
'thresholds':[.05,.10,.20],'maximum_directional_relative_l2':.20,'cosine_min':.98,'pearson_min':.95,
'rms_floor':'at least within-layer median calibration neuron RMS','std_over_rms_min':.25,
'effective_energy_tokens_min':16,'max_selected_pairs':1,'selection':'smallest worst directional calibration error; then layer/id',
'heldout_per_text_error_max':.35,'heldout_quality_mean_nll_delta_max':.01,'heldout_quality_sequence_p95_delta_max':.05,
'training':False,'scale_fitting':False,'cost_required':False}
protocol_path=root/'neuron-reuse-protocol.json'
if protocol_path.exists():
old=json.loads(protocol_path.read_text());assert all(old[k]==v for k,v in protocol.items())
else:write_json(protocol_path,{**protocol,'registered_unix':time.time()})
ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text());remaining=int(7200-ledger['spent_seconds']-30)
if remaining<120:raise TimeoutError('Remaining cumulative budget insufficient')
started=time.monotonic();result={'state':'running','selected':None,'function_pass':False,'interventions':[]}
try:
cfg=json.loads((root/'model_config.json').read_text());lm=NeuronReuseLM(root/'weights',cfg,seconds=remaining)
data=np.load(root/'baseline-activations.npz');samples=json.loads((root/'samples-tokenized.json').read_text())['samples']
cal=np.array([i for i,s in enumerate(samples) if s['split']=='calibration']);held=np.array([i for i,s in enumerate(samples) if s['split']=='heldout'])
activations={};summaries=[];best=None
for layer in LAYERS:
h=data[f'mlp_{layer}'];pfx=f'layers.{layer}.mlp.'
z=silu(lm.linear(h,pfx+'gate_proj'))*lm.linear(h,pfx+'up_proj');activations[layer]=z
# Profiles use all calibration tokens; never use heldout in selection.
x=np.ascontiguousarray(z[cal].reshape(-1,z.shape[-1]).T,dtype=np.float64)
n=x.shape[1];normsq=(x*x).sum(1);means=x.mean(1);variance=np.maximum(normsq-n*means**2,0)
rms=np.sqrt(normsq/n);floor=float(np.median(rms));effective=normsq**2/np.maximum((x**4).sum(1),1e-30)
active=(rms>=floor)&(np.sqrt(variance/np.maximum(normsq,1e-30))>=.25)&(effective>=16)
counts={str(t):0 for t in protocol['thresholds']};tested=0;layerbest=None
for start in range(0,x.shape[0],256):
lm.guard();end=min(start+256,x.shape[0]);dot=x[start:end]@x.T
delta=np.maximum(normsq[start:end,None]+normsq[None,:]-2*dot,0)
error=np.sqrt(delta/np.maximum(np.minimum(normsq[start:end,None],normsq[None,:]),1e-30))
cosine=dot/np.maximum(np.sqrt(normsq[start:end,None]*normsq[None,:]),1e-30)
pearson=(dot-n*means[start:end,None]*means[None,:])/np.maximum(np.sqrt(variance[start:end,None]*variance[None,:]),1e-30)
unique=np.arange(start,end)[:,None]<np.arange(x.shape[0])[None,:]
allowed=unique&active[start:end,None]&active[None,:]&(cosine>=.98)&(pearson>=.95)
tested+=int((unique&active[start:end,None]&active[None,:]).sum())
for threshold in protocol['thresholds']:counts[str(threshold)]+=int((allowed&(error<=threshold)).sum())
mask=allowed&(error<=.20)
if mask.any():
score=np.where(mask,error,np.inf);row,col=np.unravel_index(np.argmin(score),score.shape)
candidate={'layer':layer,'a':int(start+row),'b':int(col),'calibration_worst_relative_l2':float(score[row,col])}
key=lambda c:(c['calibration_worst_relative_l2'],c['layer'],c['a'],c['b'])
if layerbest is None or key(candidate)<key(layerbest):layerbest=candidate
if best is None or key(candidate)<key(best):best=candidate
summaries.append({'layer':layer,'neurons':int(x.shape[0]),'eligible_neurons':int(active.sum()),'eligible_unique_pairs_checked':tested,'rms_floor':floor,'calibration_pair_counts':counts,'best':layerbest})
write_json(root/'neuron-reuse-search-summary.json',summaries)
print(json.dumps(summaries[-1]),flush=True)
np.savez(root/'neuron-reuse-activations.npz',**{f'layer{k}':v for k,v in activations.items()})
write_json(root/'neuron-reuse-selection.json',{'selected':best,'selection_uses_heldout':False})
result['selected']=best
if best is not None:
layer,i,j=best['layer'],best['a'],best['b'];z=activations[layer]
metrics=[]
for split,index in [('calibration',cal),('heldout',held)]:
metrics.append({'split':split,'scope':'pooled',**pair_metrics(z[index,:,i].reshape(-1),z[index,:,j].reshape(-1))})
for idx in index:metrics.append({'split':split,'scope':samples[idx]['id'],**pair_metrics(z[idx,:,i],z[idx,:,j])})
save_csv(root/'neuron-reuse-function-metrics.csv',metrics)
pooled=next(r for r in metrics if r['split']=='heldout' and r['scope']=='pooled')
floor=next(r['rms_floor'] for r in summaries if r['layer']==layer)
function_pass=(max(pooled['relative_l2_a_to_b'],pooled['relative_l2_b_to_a'])<=.20 and pooled['cosine']>=.98 and pooled['pearson']>=.95 and
min(pooled['std_over_rms_a'],pooled['std_over_rms_b'])>=.25 and min(pooled['effective_energy_tokens_a'],pooled['effective_energy_tokens_b'])>=16 and
min(pooled['rms_a'],pooled['rms_b'])>=floor and
all(max(r['relative_l2_a_to_b'],r['relative_l2_b_to_a'])<=.35 for r in metrics if r['split']=='heldout' and r['scope']!='pooled'))
result.update(function_pass=bool(function_pass),heldout_function_metrics=pooled)
# Show parameter similarity separately from actual functional similarity.
wd=[]
for role in ('gate_proj','up_proj'):
t=lm.tensors[f'model.language_model.layers.{layer}.mlp.{role}.weight'];d=t.shape[1]
wa,wb=t.read(slice(i*d,(i+1)*d)),t.read(slice(j*d,(j+1)*d))
wd.append({'role':role,'weight_cosine':float(wa.astype(np.float64)@wb/max(np.linalg.norm(wa.astype(np.float64))*np.linalg.norm(wb.astype(np.float64)),1e-30))});t.evict()
result['weight_diagnostics']=wd
if function_pass:
ids=np.array([s['token_ids'] for s in samples],dtype=np.int64);base=np.load(root/'baseline-heldout-token-nll.npy').mean(1)
for target,donor in ((i,j),(j,i)):
lm.configure(layer,target,donor);h,_=lm.forward(ids[held]);nll=lm.nll(h,ids[held])
np.save(root/f'neuron-reuse-{target}-uses-{donor}-token-nll.npy',nll)
delta=nll.mean(1)-base
result['interventions'].append({'target':target,'donor':donor,'sequence_nll_deltas':delta.tolist(),'mean_nll_delta':float(delta.mean()),'sequence_p95_delta':float(np.quantile(delta,.95)),'passed':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05)})
write_json(root/'neuron-reuse-summary.json',result)
# No fallback after looking at heldout.
result.update(state='completed',empirical_repeated_function_positive=bool(result['function_pass'] and len(result['interventions'])==2 and all(r['passed'] for r in result['interventions'])),
claim_limit='Distinct learned scalar MLP subfunctions on the specified short-text distribution; not whole-layer duplication, global equivalence or broad task-capability proof. No performance requirement.')
except Exception as exc:result.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise
finally:
elapsed=time.monotonic()-started;result.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.neuron_reuse','seconds':elapsed});write_json(ledger_path,ledger)
write_json(root/'neuron-reuse-summary.json',result)
if __name__=='__main__':main()
+108
View File
@@ -0,0 +1,108 @@
"""Existing learned RMSNorm modules: uniform function bound + independent probes.
Deterministic exhaustive selection by a mathematical all-input error bound, not
calibration/heldout fitting. Same role, shape and eps; one pair only.
"""
import argparse
import json
from pathlib import Path
import resource
import time
import numpy as np
from .model import StreamingLM,norm
from .run import write_json,save_csv,error_rows
ROLES=['input_layernorm','post_attention_layernorm']
def uniform_bounds(a,b):
if np.any(a==0) or np.any(b==0):return None
diff=np.abs(a.astype(np.float64)-b.astype(np.float64))
return float(np.max(diff/np.abs(a))),float(np.max(diff/np.abs(b)))
def main():
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);args=p.parse_args();root=args.root
protocol={'roles':ROLES,'layers':list(range(24)),'same_role_only':True,'same_eps':1e-6,
'uniform_worst_error_limit':.20,'empirical_function_worst_relative_error_limit':.05,
'selection':'smallest maximum two-direction uniform bound, then role/layer; no input data used',
'minimum_gain_standard_deviation':.001,'max_selected_pairs':1,'heldout_nll_mean_delta_limit':.01,
'heldout_nll_sequence_p95_delta_limit':.05,'training':False,'cost_required':False}
path=root/'norm-repetition-protocol.json'
if path.exists():
existing=json.loads(path.read_text());assert all(existing[k]==v for k,v in protocol.items())
else:write_json(path,{**protocol,'registered_unix':time.time()})
ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text());remaining=int(7200-ledger['spent_seconds']-30)
if remaining<120:raise TimeoutError('Insufficient cumulative computation budget')
started=time.monotonic();result={'state':'running','selected':None,'interventions':[]}
try:
cfg=json.loads((root/'model_config.json').read_text());lm=StreamingLM(root/'weights',cfg,seconds=remaining)
assert lm.eps==1e-6
candidates=[];gains={}
for role in ROLES:
for layer in range(24):gains[(role,layer)]=1+lm.small(f'layers.{layer}.{role}.weight')
for i in range(24):
for j in range(i+1,24):
a,b=gains[(role,i)],gains[(role,j)]
bounds=uniform_bounds(a,b)
if bounds is None:continue
eligible=min(float(np.std(a)),float(np.std(b)))>=.001
candidates.append({'role':role,'a':i,'b':j,'a_to_b_uniform_bound':bounds[0],'b_to_a_uniform_bound':bounds[1],
'worst_uniform_bound':max(bounds),'eligible_nonconstant_gains':eligible,
'gain_std_a':float(np.std(a)),'gain_std_b':float(np.std(b)),
'gain_min_a':float(np.min(a)),'gain_min_b':float(np.min(b))})
save_csv(root/'norm-repetition-uniform-bounds.csv',candidates)
feasible=[c for c in candidates if c['eligible_nonconstant_gains'] and c['worst_uniform_bound']<=.20]
counts={str(t):sum(c['eligible_nonconstant_gains'] and c['worst_uniform_bound']<=t for c in candidates) for t in (.05,.10,.20)}
selected=sorted(feasible,key=lambda c:(c['worst_uniform_bound'],c['role'],c['a'],c['b']))[:1]
write_json(root/'norm-repetition-selection.json',{'selected':selected,'uniform_bound_counts':counts,'selection_reads_no_input_data':True})
result['uniform_bound_counts']=counts
if selected:
c=selected[0];result['selected']=c
# Selection is frozen before reading tokenized inputs or their activations.
samples=json.loads((root/'samples-tokenized.json').read_text())['samples'];ids=np.array([s['token_ids'] for s in samples],dtype=np.int64)
held=np.array([i for i,s in enumerate(samples) if s['split']=='heldout'])
kind='attn' if c['role']=='input_layernorm' else 'mlp'
captures=[(kind,c['a']),(kind,c['b'])]
h,saved=lm.forward(ids,capture=captures)
# The untouched recomputation must reproduce the saved real baseline.
original=np.load(root/'baseline-activations.npz')['final']
np.testing.assert_array_equal(h,original)
np.savez(root/'norm-repetition-input-profiles.npz',**saved)
ga,gb=gains[(c['role'],c['a'])],gains[(c['role'],c['b'])]
rows=[]
for source in (c['a'],c['b']):
n=saved[f'{kind}_{source}'];g=gains[(c['role'],source)]
# n = u * gamma_source. Reconstruct common normalized input u;
# source gains are nonzero. Additional rounding is ~FP32 only.
unit=n/g
fa,fb=unit*ga,unit*gb
rec=error_rows(fa,fb,samples,{'source_layer':source,'role':c['role'],'a':c['a'],'b':c['b']})
rows.extend(rec)
save_csv(root/'norm-repetition-function-metrics.csv',rows)
heldrows=[r for r in rows if r['split']=='heldout']
function_pass=all(max(r['relative_l2'],r['reverse_relative_l2'])<=.05 for r in heldrows)
result.update(uniform_function_similarity_proven=True,heldout_function_pass=bool(function_pass),
worst_heldout_relative_l2=max(max(r['relative_l2'],r['reverse_relative_l2']) for r in heldrows),
worst_heldout_cosine=min(r['cosine'] for r in heldrows),untouched_full_forward_bitwise_matches_baseline=True)
if function_pass:
baseline=np.load(root/'baseline-heldout-token-nll.npy').mean(1)
for target,donor in ((c['a'],c['b']),(c['b'],c['a'])):
lm.norm_replacement=(f'layers.{target}.{c["role"]}',f'layers.{donor}.{c["role"]}')
changed,_=lm.forward(ids[held]);nll=lm.nll(changed,ids[held]);np.save(root/f'norm-repetition-{target}-uses-{donor}-token-nll.npy',nll)
delta=nll.mean(1)-baseline
result['interventions'].append({'target':target,'donor':donor,'sequence_nll_deltas':delta.tolist(),
'mean_nll_delta':float(delta.mean()),'sequence_p95_nll_delta':float(np.quantile(delta,.95)),
'passed_quality':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05)})
write_json(root/'norm-repetition-summary.json',result)
result['learned_function_repetition_positive']=bool(function_pass)
result['bidirectional_network_quality_pass']=bool(len(result['interventions'])==2 and all(x['passed_quality'] for x in result['interventions']))
result.update(state='completed',claim_limit='Existing learned RMSNorm module functions with identical eps and shape; uniform real-arithmetic relative bound and tiny real-text probes. Does not establish attention/MLP-body duplication or broad task capability.')
except Exception as exc:result.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise
finally:
elapsed=time.monotonic()-started;result.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss)
ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.norm_repetition','seconds':elapsed});write_json(ledger_path,ledger)
write_json(root/'norm-repetition-summary.json',result)
if __name__=='__main__':main()
+46
View File
@@ -0,0 +1,46 @@
"""Package code-independent scientific outputs, retain large activation caches externally.
The experiment directory remains immutable and complete. The delivery directory
contains all figure source tables/NLL arrays plus hashes/locations for three large
reproducible activation caches. Never deletes model weights or cache files.
"""
import argparse
import hashlib
import json
from pathlib import Path
import shutil
from .run import write_json
CACHES={'baseline-activations.npz','neuron-reuse-activations.npz','norm-repetition-input-profiles.npz'}
def main():
p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);p.add_argument('--output',required=True,type=Path);a=p.parse_args()
root,out=a.root.resolve(),a.output.resolve();assert root!=out
out.mkdir(parents=True,exist_ok=True)
full=json.loads((root/'checksums-functional.json').read_text())
files=json.loads((root/'archive-files.json').read_text())
for name in files:
src=root/name
if name in CACHES:continue
dest=out/name;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(src,dest)
external=[]
for name in sorted(CACHES):
src=root/name;digest=hashlib.sha256(src.read_bytes()).hexdigest();assert digest==full[name]
external.append({'name':name,'server_path':str(src),'bytes':src.stat().st_size,'sha256':digest,'retained':True,'reproducible_by':'functional.run / functional.neuron_reuse / functional.norm_repetition'})
write_json(out/'external-activation-cache.json',{'host':'8.211.154.134','files':external,'not_model_weights':True,
'reason':'Large intermediate caches retained on the authorized server; all figure source tables and token NLL arrays are delivered in Git.'})
shutil.copy2(root/'checksums-functional.json',out/'checksums-full-server.json')
(out/'DELIVERY.md').write_text('# Scientific delivery and retained caches\n\nAll figure CSV/JSON source data, per-token NLL, protocols, reports, metadata and historical logs are included.\nThree large reproducible intermediate activation caches remain on 8.211.154.134 and in the original full local archive;\n`external-activation-cache.json` records exact locations, sizes and SHA256. No weight or cache was deleted.\nThe original full experiment manifest is `checksums-full-server.json`; the delivery-only manifest is\n`checksums-functional.json`. No model checkpoint or tokenizer payload is in this Git delivery.\n')
validation=json.loads((out/'audit-validation.json').read_text());validation.update(delivery_excludes_external_activation_caches=True,full_experiment_checksums='checksums-full-server.json',figure_source_tables_and_token_nll_complete=True)
write_json(out/'audit-validation.json',validation)
checks={}
for file in sorted(out.rglob('*')):
if file.is_file() and file.name not in ('checksums-functional.json','audit-validation.json','archive-files.json'):
checks[str(file.relative_to(out))]=hashlib.sha256(file.read_bytes()).hexdigest()
write_json(out/'checksums-functional.json',checks)
write_json(out/'archive-files.json',[*checks,'checksums-functional.json','audit-validation.json','archive-files.json'])
print(json.dumps({'delivery_files':len(checks)+3,'checksummed_files':len(checks),'external_caches':len(external),'weights_and_caches_retained':True}))
if __name__=='__main__':main()
+20 -8
View File
@@ -14,14 +14,26 @@ def main():
samples,ids=tokenize(a.root)
start=time.time()
lm=StreamingLM(a.root/'weights',json.loads((a.root/'model_config.json').read_text()),seconds=1200)
def progress(layer):print(json.dumps({'preflight_layer':layer,'seconds':time.time()-start}),flush=True)
h,_=lm.forward(ids[:1,:8],progress=progress)
nll=lm.nll(h,ids[:1,:8])
if not np.isfinite(nll).all():raise FloatingPointError('Nonfinite NLL')
write_json(a.root/'preflight.json',{'passed':True,'sample_id':samples[0]['id'],'tokens':8,
'elapsed_seconds':time.time()-start,'peak_rss_kib':resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
'mean_nll':float(nll.mean()),'token_nll':nll.tolist(),
'limitation':'Real-weight capacity check, not an independent correctness oracle or quality benchmark'})
snapshots=[]
result={'passed':False,'sample_id':samples[0]['id'],'tokens':8}
def progress(layer):
status=dict(line.split(':',1) for line in Path('/proc/self/status').read_text().splitlines() if ':' in line)
snapshots.append({'layer':layer,'seconds':time.time()-start,'rss_kib':int(status['VmRSS'].split()[0]),
'process_swap_kib':int(status.get('VmSwap','0 kB').split()[0]),
'available_kib':int(next(line.split()[1] for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemAvailable:')))})
print(json.dumps(snapshots[-1]),flush=True)
try:
h,_=lm.forward(ids[:1,:8],progress=progress)
nll=lm.nll(h,ids[:1,:8])
if not np.isfinite(nll).all():raise FloatingPointError('Nonfinite NLL')
result.update(passed=True,mean_nll=float(nll.mean()),token_nll=nll.tolist())
except Exception as exc:
result.update(error_type=type(exc).__name__,error=str(exc))
raise
finally:
result.update(elapsed_seconds=time.time()-start,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss,
resource_snapshots=snapshots,limitation='Real-weight capacity check, not an independent correctness oracle or quality benchmark')
write_json(a.root/'preflight.json',result)
if __name__=='__main__':main()
+3 -3
View File
@@ -14,7 +14,7 @@ import numpy as np
from .model import StreamingLM, delta_rule, rope, norm
EXPECTED_COMMIT = 'fd3cf0140de03f7619ac95b5cfed05b711ff9e22'
NAMES = {'l2norm', 'torch_chunk_gated_delta_rule', 'torch_recurrent_gated_delta_rule',
NAMES = {'apply_mask_to_padding_states', 'l2norm', 'torch_chunk_gated_delta_rule', 'torch_recurrent_gated_delta_rule',
'causal_conv1d_fn', 'Qwen3_5RMSNormGated', 'Qwen3_5GatedDeltaNet',
'rotate_half', 'apply_rotary_pos_emb', 'repeat_kv', 'eager_attention_forward',
'Qwen3_5Attention', 'Qwen3_5MLP', 'Qwen3_5RMSNorm', 'Qwen3_5DecoderLayer',
@@ -30,7 +30,7 @@ def official_namespace(source):
if isinstance(node, (ast.FunctionDef, ast.ClassDef)) and node.name in NAMES:
for child in ast.walk(node):
if isinstance(child, (ast.FunctionDef, ast.ClassDef)):
child.decorator_list = []
child.decorator_list = [d for d in child.decorator_list if isinstance(d, ast.Name) and d.id in ('staticmethod','classmethod')]
nodes.append(node)
assert {x.name for x in nodes} == NAMES
tree = ast.Module(body=[ast.ImportFrom(module='__future__',names=[ast.alias(name='annotations')],level=0),*nodes],type_ignores=[])
@@ -134,7 +134,7 @@ def main():
actual,_=lm.forward(ids)
check('complete_two_layer_text_forward',actual,expected)
result={'passed':True,'reference_commit':EXPECTED_COMMIT,'source_sha256':hashlib.sha256(a.source.read_bytes()).hexdigest(),
'torch':torch.__version__,'numpy':np.__version__,'tests':evidence,'limitations':'FP32 CPU semantics on tiny synthetic configurations; not full-checkpoint BF16 kernel equivalence'}
'torch':torch.__version__,'numpy':np.__version__,'implementation_sha256':hashlib.sha256(Path(__file__).with_name('model.py').read_bytes()).hexdigest(),'tests':evidence,'limitations':'FP32 CPU semantics on tiny synthetic configurations; not full-checkpoint BF16 kernel equivalence'}
a.output.write_text(json.dumps(result,indent=2))
print(json.dumps({'passed':True,'checks':len(evidence),'maximum_absolute_discrepancy':max(x['max_abs_error'] for x in evidence)}))
+64 -9
View File
@@ -23,9 +23,9 @@ def main():
ax.bar(np.arange(len(c)),[max(v['maximum_symmetric_relative_error'],1e-8) for v in c],color=['#238b45' if v['passed'] else '#9e9e9e' for v in c])
ax.axhline(.05,color='#b2182b',linestyle='--',label='Relative L2 screen = 0.05; other gates also apply')
ax.set_yscale('log');ax.set_xticks(np.arange(len(c)),[v['id'] for v in c],rotation=45,ha='right')
ax.set_ylabel('Worst calibration symmetric relative L2')
ax.set_ylabel('Worst calibration directional relative L2')
ax.set_title('Fixed candidates; same-input function comparison\n4 calibration texts, 32 tokens; no training')
ax.legend(loc='upper left',bbox_to_anchor=(0,1.01),fontsize=8);fig.tight_layout();save(fig,'candidate_function_errors')
ax.legend(loc='upper left',bbox_to_anchor=(0,-.38),fontsize=8);fig.tight_layout();save(fig,'candidate_function_errors')
if (root/'interventions.json').exists():
interventions=json.loads((root/'interventions.json').read_text())
fig,ax=plt.subplots(figsize=(8,4.5))
@@ -36,13 +36,68 @@ def main():
ax.set_xticks(np.arange(4),['heldout zh1','heldout zh2','heldout en1','heldout en2'])
ax.set_ylabel('Token NLL change (nat/token)');ax.set_title('Single-site interventions; frozen candidate\n4 heldout texts, 32 tokens; limited pilot evidence')
ax.legend(loc='upper left',bbox_to_anchor=(0,-.18),fontsize=8);fig.tight_layout();save(fig,'heldout_intervention_nll')
text=['# CPU functional reuse pilot','',f'- State: {status["state"]}',f'- Positive bidirectional pilot: {status.get("positive_bidirectional_quality",False)}',
f'- Elapsed analysis seconds: {status.get("elapsed_seconds")}',f'- Peak RSS KiB: {status.get("peak_rss_kib")}',
'- Original BF16 weights decoded into FP32 CPU operators, no training.',
'- Calibration and heldout samples are distinct but very small, short and experiment-authored.',
'- A positive result is limited to these empirical tests, not global equivalence or GPU speedup.',
'- Weight retention: keep until explicit user deletion request; no automatic cleanup.','']
if status['state']!='completed':text+=['## Stop evidence','',f'{status.get("error_type")}: {status.get("error")}','']
norm_path=root/'norm-repetition-summary.json'
repetition=json.loads(norm_path.read_text()) if norm_path.exists() else {}
if (root/'norm-repetition-function-metrics.csv').exists():
with (root/'norm-repetition-function-metrics.csv').open() as f:rows=list(csv.DictReader(f))
sample_ids=list(dict.fromkeys(row['sample_id'] for row in rows))
fig,ax=plt.subplots(figsize=(9,4.8))
for source in sorted({row['source_layer'] for row in rows},key=int):
rr=[r for r in rows if r['source_layer']==source]
ax.plot(range(len(rr)),[max(float(r['relative_l2']),float(r['reverse_relative_l2'])) for r in rr],'o-',label=f'Input captured at layer {source}')
ax.axhline(.05,color='#b2182b',ls='--',label='Heldout empirical screen = 0.05')
chosen=repetition.get('selected')
if chosen:ax.axhline(chosen['worst_uniform_bound'],color='#6a51a3',ls=':',label='Proven uniform all-input bound')
ax.axvline(3.5,color='gray',linewidth=.8)
ax.set_xticks(range(len(sample_ids)),sample_ids,rotation=30,ha='right');ax.set_ylabel('Worst of two directional relative L2 errors')
ax.set_title('Learned input RMSNorm at layer indices 12/13: same-input comparison\nCalibration (left) / independent heldout texts (right), 32 tokens each')
ax.legend(loc='upper left',bbox_to_anchor=(0,-.25),fontsize=8);fig.tight_layout();save(fig,'learned_norm_function_errors')
if repetition.get('interventions'):
fig,ax=plt.subplots(figsize=(8,4.5))
for v in repetition['interventions']:
ax.plot(range(4),v['sequence_nll_deltas'],'o-',label=f'Layer {v["target"]} uses {v["donor"]}; mean={v["mean_nll_delta"]:.5g}')
ax.axhline(0,color='gray',linewidth=.8);ax.set_xticks(range(4),['zh1','zh2','en1','en2'])
ax.set_xlabel('Independent heldout text (32 tokens)');ax.set_ylabel('Mean token NLL change (nat/token)')
ax.set_title('Bidirectional single-site substitution of existing learned RMSNorm modules')
ax.legend(loc='upper left',bbox_to_anchor=(0,-.18),fontsize=8);fig.tight_layout();save(fig,'learned_norm_network_substitution')
if (root/'gain-factorization-score-metrics.csv').exists():
with (root/'gain-factorization-score-metrics.csv').open() as f:rr=list(csv.DictReader(f))
layers=sorted({int(v['layer']) for v in rr});fig,axes=plt.subplots(1,2,figsize=(10,4))
for ax,key,label in zip(axes,['score_max_abs_error','probability_max_abs_error'],['Maximum absolute score error','Maximum absolute probability error']):
for direction in ('q_to_k','k_to_q'):
ax.plot(layers,[max(float(v[key]) for v in rr if int(v['layer'])==layer and v['direction']==direction) for layer in layers],'o-',label=direction)
ax.set_xlabel('Full-attention layer index');ax.set_xticks(layers);ax.set_ylabel(label);ax.ticklabel_format(axis='y',style='sci',scilimits=(0,0));ax.legend(loc='upper left',fontsize=8)
fig.suptitle('Supplement: algebraic reparameterization, separate from learned-function repetition');fig.tight_layout();save(fig,'algebraic_gain_identity')
neuron_path=root/'neuron-reuse-summary.json'
neuron=json.loads(neuron_path.read_text()) if neuron_path.exists() else {}
text=['# CPU learned-function repetition pilot','',
'Primary question: do two distinct trained parts implement similar functions? Cost is not a success condition.',
f'- Existing learned RMSNorm function positive: {repetition.get("learned_function_repetition_positive",False)}',
f'- Bidirectional network quality screen passed: {repetition.get("bidirectional_network_quality_pass",False)}',
f'- Learned scalar MLP neuron search positive: {neuron.get("empirical_repeated_function_positive",False)}',
f'- Original 11 candidate run state: {status["state"]}; selected: {status.get("selected_candidate",[])}',
'- All model weights are retained on the server until the user explicitly requests deletion.','']
if repetition.get('selected'):
c=repetition['selected'];text += ['## Existing learned module evidence','',
f'Role: {c["role"]}; layers {c["a"]} and {c["b"]}; each maps 2048-dimensional input to output.',
f'Uniform real-arithmetic relative-error bound, first-to-second: {c["a_to_b_uniform_bound"]:.12g}; reverse: {c["b_to_a_uniform_bound"]:.12g}.',
f'Maximum heldout directional output relative L2: {repetition.get("worst_heldout_relative_l2")}; minimum cosine: {repetition.get("worst_heldout_cosine")}.',
'Same epsilon and nonzero gains: the common RMS denominator cancels. For any nonzero input x,',
'`||f_a(x)-f_b(x)|| / ||f_a(x)|| <= max_j |a_j-b_j|/|a_j|`; swap a/b for the reverse direction.',
'Selection used only the tight all-input bound across 552 same-role pairs, before inspecting inputs.',
'Real probes use four calibration and four heldout experiment-authored texts, 32 tokens each, and both source layers.',
'These are learned, nonconstant gain vectors. The claim concerns RMSNorm modules; it does not establish attention/MLP-body duplication.','']
for v in repetition.get('interventions',[]):text += [f'- Layer {v["target"]} uses {v["donor"]}: mean heldout NLL delta {v["mean_nll_delta"]:.12g}; sequence P95 delta {v["sequence_p95_nll_delta"]:.12g}; screen pass={v["passed_quality"]}.']
text += ['', '## Negative results and separate algebraic result','',
'The original whole-attention, whole-MLP and within-layer Q/K norm candidates did not pass their gates.',
'The scalar neuron search excludes low-RMS/near-constant profiles and selects at most one pair without heldout feedback; consult its search summary for coverage.',
'Q/K non-RoPE gain consolidation is a separately verified real-arithmetic identity, not evidence that two existing learned modules duplicate functions.',
'Earlier local CPU timing records are preserved for provenance; performance is outside the current success criterion.','',
'## Limits and provenance','',
'Original BF16 weights were decoded into streaming FP32 CPU operators; no training, scale fitting, R fitting or GPU.',
'The numerical reference includes 22 independent official Torch comparisons. Early adapter errors and two resource-guard stops are preserved as historical logs.',
'The persistent computation ledger includes a conservative 180-second reserve for early checks; no budget reset after that reserve.',
'The text sample is tiny and short. Network NLL checks are not broad task-accuracy validation.','']
(root/'FUNCTIONAL_REPORT.md').write_text('\n'.join(text))
+62
View File
@@ -0,0 +1,62 @@
"""Measure an explicit two-component alias storage layout, not inference speed.
Payloads remain on the server under weights/ and must not be committed or cleaned
without the user's explicit deletion request. Manifest/byte/hash evidence may be archived.
"""
import argparse
import hashlib
import json
from pathlib import Path
import numpy as np
from pilot.tensors import load_tensors,decode
from .run import write_json
def payload(t,path):
digest=hashlib.sha256()
with path.open('wb') as f:
for start in range(0,t.raw.size,262144):
b=t.raw[start:start+262144].tobytes();f.write(b);digest.update(b)
t.evict()
return {'file':path.name,'dtype':t.dtype,'shape':list(t.shape),'bytes':path.stat().st_size,'sha256':digest.hexdigest()}
def main():
p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);a=p.parse_args();root=a.root
selected=json.loads((root/'selection.json').read_text())['selected']
if not selected:
write_json(root/'storage-evidence.json',{'measured':False,'reason':'No calibration candidate selected'});return
c=selected[0]
if c['kind']!='qk_norm':
write_json(root/'storage-evidence.json',{'measured':False,'reason':'Only two-gain alias layout implemented in this bounded pilot; larger-module savings remain theoretical'});return
tensors=load_tensors(root/'weights')
prefix=f'model.language_model.layers.{c["i"]}.self_attn.'
names=[prefix+'q_norm.weight',prefix+'k_norm.weight']
# Original pair plus two alternative layouts; each is counted independently.
path=root/'component-layout'/'weights';path.mkdir(parents=True,exist_ok=True)
original={'format':'two_component_raw_v1','tensors':{}}
for short,name in zip(('q','k'),names):original['tensors'][name]=payload(tensors[name],path/f'{short}.bin')
original_bytes=json.dumps(original,separators=(',',':'),sort_keys=True).encode()
(path/'original-layout.json').write_bytes(original_bytes)
evidence=[]
for target,donor in ((names[0],names[1]),(names[1],names[0])):
entry=original['tensors'][donor]
layout={'format':'two_component_raw_v1','tensors':{donor:entry},'aliases':{target:donor}}
encoded=json.dumps(layout,separators=(',',':'),sort_keys=True).encode()
file=path/('share-'+('k' if donor==names[1] else 'q')+'.json');file.write_bytes(encoded)
t=tensors[donor]
raw=np.fromfile(path/entry['file'],dtype=t.raw.dtype)
reconstructed=decode(raw,t.dtype).reshape(t.shape)
np.testing.assert_array_equal(reconstructed,t.read().reshape(t.shape));t.evict()
before=sum(e['bytes'] for e in original['tensors'].values())+len(original_bytes)
after=entry['bytes']+len(encoded)
evidence.append({'target':target,'donor':donor,'original_payload_bytes':sum(e['bytes'] for e in original['tensors'].values()),
'shared_payload_bytes':entry['bytes'],'original_descriptor_bytes':len(original_bytes),
'shared_descriptor_bytes':len(encoded),'original_total_bytes':before,'shared_total_bytes':after,
'net_saved_bytes':before-after,'shared_descriptor_sha256':hashlib.sha256(encoded).hexdigest(),
'decoded_donor_exactly_matches_intervention_gain':True})
write_json(root/'storage-evidence.json',{'measured':True,'candidate':c['id'],'layouts':evidence,
'claim_limit':'Measured standalone component layout including descriptors, not an integrated compressed checkpoint, latency gain or full-model RSS reduction. Two sharing directions are alternatives and must not be added.'})
if __name__=='__main__':main()
+26 -4
View File
@@ -22,13 +22,32 @@ def main():
state.update(kwargs,supervisor_elapsed_seconds=time.monotonic()-start)
write_json(root/'supervisor-status.json',state)
def execute(module,args,timeout,log):
with (root/log).open('w') as out:
r=subprocess.run([sys.executable,'-B','-m',module,*args],stdout=out,stderr=subprocess.STDOUT,timeout=timeout)
ledger_path=root/'compute-budget-ledger.json'
ledger=json.loads(ledger_path.read_text()) if ledger_path.exists() else {'spent_seconds':0.,'calls':[]}
timeout=min(timeout,int(7200-ledger['spent_seconds']))
if timeout<1:raise TimeoutError('Cumulative two-hour compute budget exhausted')
t=time.monotonic()
try:
with (root/log).open('w') as out:
r=subprocess.run([sys.executable,'-B','-m',module,*args],stdout=out,stderr=subprocess.STDOUT,timeout=timeout)
finally:
elapsed=time.monotonic()-t
ledger['spent_seconds']+=elapsed
ledger['calls'].append({'module':module,'seconds':elapsed})
write_json(ledger_path,ledger)
if r.returncode:raise RuntimeError(f'{module} failed: exit {r.returncode}, see {log}')
status()
try:
while not all(importlib.util.find_spec(name) for name in ('torch','tokenizers')):
while True:
ready=False
if all(importlib.util.find_spec(name) for name in ('torch','tokenizers')):
try:
r=subprocess.run([sys.executable,'-B','-c','import torch, tokenizers; torch.empty(1)'],capture_output=True,timeout=30)
ready=r.returncode==0
except subprocess.TimeoutExpired:pass
if ready:break
if time.monotonic()-start>3600:raise TimeoutError('Binary reference dependencies not ready within one hour')
status()
time.sleep(15)
compute_start=time.monotonic()
status(state='running',phase='reference_check')
@@ -47,15 +66,18 @@ def main():
except (OSError,ValueError,KeyError):ready=False
if ready:break
if time.monotonic()-start>a.download_wait_seconds:raise TimeoutError('Verified weight download not ready before wait deadline')
status()
time.sleep(15)
status(state='running',phase='real_weight_preflight')
compute_start=time.monotonic()
execute('functional.preflight',['--root',str(root)],1200,'preflight.log')
preflight_seconds=time.monotonic()-compute_start
remaining=int(7200-reference_seconds-preflight_seconds-60)
remaining=int(7200-json.loads((root/'compute-budget-ledger.json').read_text())['spent_seconds']-60)
if remaining<60:raise TimeoutError('No remaining bounded functional computation budget')
status(phase='functional_analysis',preflight_seconds=preflight_seconds,remaining_seconds=remaining)
execute('functional.run',['--root',str(root),'--seconds',str(remaining)],remaining+15,'functional-run.log')
status(phase='storage_accounting')
execute('functional.storage',['--root',str(root)],120,'storage-accounting.log')
status(phase='report')
execute('functional.report',['--root',str(root)],120,'functional-report.log')
status(state='completed',phase='done')