Validate repeated learned RMSNorm functions with CPU evidence

This commit is contained in:
2026-11-09 14:14:59 +08:00
parent 573cbedd98
commit 6d1acbc26a
108 changed files with 6526 additions and 33 deletions
+57
View File
@@ -59,4 +59,61 @@ class FunctionalTests(unittest.TestCase):
np.testing.assert_array_equal(norm(x,weight,centered=False),np.zeros_like(x))
def test_nonrotary_gain_identity_and_rotary_negative_control(self):
from functional.model import rope
from functional.gain_factorization import consolidate
rng=np.random.default_rng(19)
q=rng.standard_normal((2,7,8,256),dtype=np.float32)
k=rng.standard_normal((2,7,2,256),dtype=np.float32)
a=rng.uniform(.5,1.5,256).astype(np.float32)
b=rng.uniform(.5,1.5,256).astype(np.float32)
def scores(q,k):return np.einsum('bthd,bshd->bhts',rope(q),np.repeat(rope(k),4,axis=2))/16
reference=scores(norm(q,a,centered=False),norm(k,b,centered=False))
for direction in ('q_to_k','k_to_q'):
q1,k1=consolidate(q,k,a,b,64,direction)
np.testing.assert_allclose(scores(q1,k1),reference,atol=3e-6,rtol=3e-5)
# Moving gains through rotated coordinates is not a valid general identity.
q1,k1=consolidate(q,k,a,b,0,'q_to_k')
self.assertGreater(float(np.max(np.abs(scores(q1,k1)-reference))),1e-3)
def test_neuron_function_metrics_and_targeted_replacement(self):
from functional.neuron_reuse import pair_metrics,NeuronReuseLM
x=np.linspace(-2,2,128,dtype=np.float64)
m=pair_metrics(x,x*1.05)
self.assertAlmostEqual(m['relative_l2_a_to_b'],.05)
self.assertAlmostEqual(m['pearson'],1.)
self.assertGreater(m['effective_energy_tokens_a'],16)
dead=pair_metrics(np.zeros(128),np.zeros(128))
self.assertEqual(dead['rms_a'],0.)
class Tiny(NeuronReuseLM):
def __init__(self):
self.arrays={'gate_proj':np.array([[1,0],[0,2],[1,1]],dtype=np.float32),
'up_proj':np.array([[2,0],[0,1],[1,-1]],dtype=np.float32),
'down_proj':np.array([[1,2,3],[-1,1,0]],dtype=np.float32)}
def linear(self,h,name,rows=256):return h@self.arrays[name.rsplit('.',1)[-1]].T
lm=Tiny();lm.configure(5,0,1)
h=np.array([[[1,2],[-1,3]]],dtype=np.float32)
z=silu(h@lm.arrays['gate_proj'].T)*(h@lm.arrays['up_proj'].T)
z[...,0]=z[...,1]
np.testing.assert_allclose(lm.mlp(h,5),z@lm.arrays['down_proj'].T)
original=silu(h@lm.arrays['gate_proj'].T)*(h@lm.arrays['up_proj'].T)
np.testing.assert_allclose(lm.mlp(h,6),original@lm.arrays['down_proj'].T)
def test_uniform_rmsnorm_function_bound_is_tight(self):
from functional.norm_repetition import uniform_bounds
a=np.array([1.2,.8,1.5,1.1],dtype=np.float64)
b=np.array([1.22,.78,1.45,1.13],dtype=np.float64)
ab,ba=uniform_bounds(a,b)
basis=np.eye(4)
# Common RMS denominator cancels; basis inputs attain the exact maximum.
ya,yb=basis*a,basis*b
ea=np.linalg.norm(ya-yb,axis=-1)/np.linalg.norm(ya,axis=-1)
eb=np.linalg.norm(ya-yb,axis=-1)/np.linalg.norm(yb,axis=-1)
self.assertAlmostEqual(float(ea.max()),ab)
self.assertAlmostEqual(float(eb.max()),ba)
self.assertIsNone(uniform_bounds(np.array([0.,1.]),np.ones(2)))
if __name__=='__main__':unittest.main()