Validate repeated learned RMSNorm functions with CPU evidence

This commit is contained in:
2026-11-09 14:14:59 +08:00
parent 573cbedd98
commit 6d1acbc26a
108 changed files with 6526 additions and 33 deletions
@@ -0,0 +1,388 @@
<?xml version="1.0" encoding="utf-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns:xlink="http://www.w3.org/1999/xlink" width="640.424062pt" height="312.347793pt" viewBox="0 0 640.424062 312.347793" xmlns="http://www.w3.org/2000/svg" version="1.1">
<metadata>
<rdf:RDF xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:cc="http://creativecommons.org/ns#" xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
<cc:Work>
<dc:type rdf:resource="http://purl.org/dc/dcmitype/StillImage"/>
<dc:date>2026-11-09T14:10:55.472730</dc:date>
<dc:format>image/svg+xml</dc:format>
<dc:creator>
<cc:Agent>
<dc:title>Matplotlib v3.11.2, https://matplotlib.org/</dc:title>
</cc:Agent>
</dc:creator>
</cc:Work>
</rdf:RDF>
</metadata>
<defs>
<style type="text/css">*{stroke-linejoin: round; stroke-linecap: butt}</style>
</defs>
<g id="figure_1">
<g id="patch_1">
<path d="M 0 312.347793
L 640.424062 312.347793
L 640.424062 0
L 0 0
z
" style="fill: #ffffff"/>
</g>
<g id="axes_1">
<g id="patch_2">
<path d="M 50.466406 208.18075
L 633.224062 208.18075
L 633.224062 37.922578
L 50.466406 37.922578
z
" style="fill: #ffffff"/>
</g>
<g id="matplotlib.axis_1">
<g id="xtick_1">
<g id="line2d_1">
<defs>
<path id="mb7bbc15d82" d="M 0 0
L 0 3.5
" style="stroke: #000000; stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#mb7bbc15d82" x="76.955391" y="208.18075" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_1">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(41.486676 241.545565) rotate(-30)">cal-zh-1</text>
</g>
</g>
<g id="xtick_2">
<g id="line2d_2">
<g>
<use xlink:href="#mb7bbc15d82" x="152.638203" y="208.18075" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_2">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(117.169489 241.545565) rotate(-30)">cal-zh-2</text>
</g>
</g>
<g id="xtick_3">
<g id="line2d_3">
<g>
<use xlink:href="#mb7bbc15d82" x="228.321016" y="208.18075" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_3">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(192.068819 241.997909) rotate(-30)">cal-en-1</text>
</g>
</g>
<g id="xtick_4">
<g id="line2d_4">
<g>
<use xlink:href="#mb7bbc15d82" x="304.003828" y="208.18075" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_4">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(267.751631 241.997909) rotate(-30)">cal-en-2</text>
</g>
</g>
<g id="xtick_5">
<g id="line2d_5">
<g>
<use xlink:href="#mb7bbc15d82" x="379.686641" y="208.18075" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_5">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(340.062357 243.944784) rotate(-30)">test-zh-1</text>
</g>
</g>
<g id="xtick_6">
<g id="line2d_6">
<g>
<use xlink:href="#mb7bbc15d82" x="455.369453" y="208.18075" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_6">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(415.74517 243.944784) rotate(-30)">test-zh-2</text>
</g>
</g>
<g id="xtick_7">
<g id="line2d_7">
<g>
<use xlink:href="#mb7bbc15d82" x="531.052266" y="208.18075" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_7">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(490.6445 244.396451) rotate(-30)">test-en-1</text>
</g>
</g>
<g id="xtick_8">
<g id="line2d_8">
<g>
<use xlink:href="#mb7bbc15d82" x="606.735078" y="208.18075" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_8">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(566.327312 244.396451) rotate(-30)">test-en-2</text>
</g>
</g>
</g>
<g id="matplotlib.axis_2">
<g id="ytick_1">
<g id="line2d_9">
<defs>
<path id="ma0713fe695" d="M 0 0
L -3.5 0
" style="stroke: #000000; stroke-width: 0.8"/>
</defs>
<g>
<use xlink:href="#ma0713fe695" x="50.466406" y="204.137497" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_9">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: end" x="43.466406" y="207.936325" transform="rotate(-0 43.466406 207.936325)">0.04</text>
</g>
</g>
<g id="ytick_2">
<g id="line2d_10">
<g>
<use xlink:href="#ma0713fe695" x="50.466406" y="182.54738" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_10">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: end" x="43.466406" y="186.346208" transform="rotate(-0 43.466406 186.346208)">0.05</text>
</g>
</g>
<g id="ytick_3">
<g id="line2d_11">
<g>
<use xlink:href="#ma0713fe695" x="50.466406" y="160.957263" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_11">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: end" x="43.466406" y="164.756091" transform="rotate(-0 43.466406 164.756091)">0.06</text>
</g>
</g>
<g id="ytick_4">
<g id="line2d_12">
<g>
<use xlink:href="#ma0713fe695" x="50.466406" y="139.367146" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_12">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: end" x="43.466406" y="143.165974" transform="rotate(-0 43.466406 143.165974)">0.07</text>
</g>
</g>
<g id="ytick_5">
<g id="line2d_13">
<g>
<use xlink:href="#ma0713fe695" x="50.466406" y="117.777029" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_13">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: end" x="43.466406" y="121.575857" transform="rotate(-0 43.466406 121.575857)">0.08</text>
</g>
</g>
<g id="ytick_6">
<g id="line2d_14">
<g>
<use xlink:href="#ma0713fe695" x="50.466406" y="96.186911" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_14">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: end" x="43.466406" y="99.98574" transform="rotate(-0 43.466406 99.98574)">0.09</text>
</g>
</g>
<g id="ytick_7">
<g id="line2d_15">
<g>
<use xlink:href="#ma0713fe695" x="50.466406" y="74.596794" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_15">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: end" x="43.466406" y="78.395623" transform="rotate(-0 43.466406 78.395623)">0.10</text>
</g>
</g>
<g id="ytick_8">
<g id="line2d_16">
<g>
<use xlink:href="#ma0713fe695" x="50.466406" y="53.006677" style="stroke: #000000; stroke-width: 0.8"/>
</g>
</g>
<g id="text_16">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: end" x="43.466406" y="56.805505" transform="rotate(-0 43.466406 56.805505)">0.11</text>
</g>
</g>
<g id="text_17">
<text style="font-size: 10px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: middle" x="14.798438" y="123.051664" transform="rotate(-90 14.798438 123.051664)">Worst of two directional relative L2 errors</text>
</g>
</g>
<g id="line2d_17">
<path d="M 76.955391 197.451805
L 152.638203 192.877368
L 228.321016 193.610655
L 304.003828 199.099794
L 379.686641 196.376861
L 455.369453 192.726888
L 531.052266 194.835109
L 606.735078 198.558148
" clip-path="url(#pc5e8d93515)" style="fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square"/>
<defs>
<path id="m8708773af9" d="M 0 3
C 0.795609 3 1.55874 2.683901 2.12132 2.12132
C 2.683901 1.55874 3 0.795609 3 0
C 3 -0.795609 2.683901 -1.55874 2.12132 -2.12132
C 1.55874 -2.683901 0.795609 -3 0 -3
C -0.795609 -3 -1.55874 -2.683901 -2.12132 -2.12132
C -2.683901 -1.55874 -3 -0.795609 -3 0
C -3 0.795609 -2.683901 1.55874 -2.12132 2.12132
C -1.55874 2.683901 -0.795609 3 0 3
z
" style="stroke: #1f77b4"/>
</defs>
<g clip-path="url(#pc5e8d93515)">
<use xlink:href="#m8708773af9" x="76.955391" y="197.451805" style="fill: #1f77b4; stroke: #1f77b4"/>
<use xlink:href="#m8708773af9" x="152.638203" y="192.877368" style="fill: #1f77b4; stroke: #1f77b4"/>
<use xlink:href="#m8708773af9" x="228.321016" y="193.610655" style="fill: #1f77b4; stroke: #1f77b4"/>
<use xlink:href="#m8708773af9" x="304.003828" y="199.099794" style="fill: #1f77b4; stroke: #1f77b4"/>
<use xlink:href="#m8708773af9" x="379.686641" y="196.376861" style="fill: #1f77b4; stroke: #1f77b4"/>
<use xlink:href="#m8708773af9" x="455.369453" y="192.726888" style="fill: #1f77b4; stroke: #1f77b4"/>
<use xlink:href="#m8708773af9" x="531.052266" y="194.835109" style="fill: #1f77b4; stroke: #1f77b4"/>
<use xlink:href="#m8708773af9" x="606.735078" y="198.558148" style="fill: #1f77b4; stroke: #1f77b4"/>
</g>
</g>
<g id="line2d_18">
<path d="M 76.955391 198.934893
L 152.638203 194.123521
L 228.321016 195.5305
L 304.003828 200.441742
L 379.686641 197.894474
L 455.369453 194.413382
L 531.052266 196.725517
L 606.735078 199.872446
" clip-path="url(#pc5e8d93515)" style="fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: square"/>
<defs>
<path id="m5a845d07a0" d="M 0 3
C 0.795609 3 1.55874 2.683901 2.12132 2.12132
C 2.683901 1.55874 3 0.795609 3 0
C 3 -0.795609 2.683901 -1.55874 2.12132 -2.12132
C 1.55874 -2.683901 0.795609 -3 0 -3
C -0.795609 -3 -1.55874 -2.683901 -2.12132 -2.12132
C -2.683901 -1.55874 -3 -0.795609 -3 0
C -3 0.795609 -2.683901 1.55874 -2.12132 2.12132
C -1.55874 2.683901 -0.795609 3 0 3
z
" style="stroke: #ff7f0e"/>
</defs>
<g clip-path="url(#pc5e8d93515)">
<use xlink:href="#m5a845d07a0" x="76.955391" y="198.934893" style="fill: #ff7f0e; stroke: #ff7f0e"/>
<use xlink:href="#m5a845d07a0" x="152.638203" y="194.123521" style="fill: #ff7f0e; stroke: #ff7f0e"/>
<use xlink:href="#m5a845d07a0" x="228.321016" y="195.5305" style="fill: #ff7f0e; stroke: #ff7f0e"/>
<use xlink:href="#m5a845d07a0" x="304.003828" y="200.441742" style="fill: #ff7f0e; stroke: #ff7f0e"/>
<use xlink:href="#m5a845d07a0" x="379.686641" y="197.894474" style="fill: #ff7f0e; stroke: #ff7f0e"/>
<use xlink:href="#m5a845d07a0" x="455.369453" y="194.413382" style="fill: #ff7f0e; stroke: #ff7f0e"/>
<use xlink:href="#m5a845d07a0" x="531.052266" y="196.725517" style="fill: #ff7f0e; stroke: #ff7f0e"/>
<use xlink:href="#m5a845d07a0" x="606.735078" y="199.872446" style="fill: #ff7f0e; stroke: #ff7f0e"/>
</g>
</g>
<g id="line2d_19">
<path d="M 50.466406 182.54738
L 633.224062 182.54738
" clip-path="url(#pc5e8d93515)" style="fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #b2182b; stroke-width: 1.5"/>
</g>
<g id="line2d_20">
<path d="M 50.466406 45.661586
L 633.224062 45.661586
" clip-path="url(#pc5e8d93515)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #6a51a3; stroke-width: 1.5"/>
</g>
<g id="line2d_21">
<path d="M 341.845234 208.18075
L 341.845234 37.922578
" clip-path="url(#pc5e8d93515)" style="fill: none; stroke: #808080; stroke-width: 0.8; stroke-linecap: square"/>
</g>
<g id="patch_3">
<path d="M 50.466406 208.18075
L 50.466406 37.922578
" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_4">
<path d="M 633.224062 208.18075
L 633.224062 37.922578
" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_5">
<path d="M 50.466406 208.18075
L 633.224063 208.18075
" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="patch_6">
<path d="M 50.466406 37.922578
L 633.224063 37.922578
" style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/>
</g>
<g id="text_18">
<text style="font-size: 12px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(126.937422 17.519297)">Learned input RMSNorm at layer indices 12/13: same-input comparison</text>
<text style="font-size: 12px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif" transform="translate(137.227422 31.922578)">Calibration (left) / independent heldout texts (right), 32 tokens each</text>
</g>
<g id="legend_1">
<g id="patch_7">
<path d="M 56.066406 305.147793
L 211.338906 305.147793
Q 212.938906 305.147793 212.938906 303.547793
L 212.938906 256.345293
Q 212.938906 254.745293 211.338906 254.745293
L 56.066406 254.745293
Q 54.466406 254.745293 54.466406 256.345293
L 54.466406 303.547793
Q 54.466406 305.147793 56.066406 305.147793
z
" style="fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter"/>
</g>
<g id="line2d_22">
<path d="M 57.666406 261.224043
L 65.666406 261.224043
L 73.666406 261.224043
" style="fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square"/>
<g>
<use xlink:href="#m8708773af9" x="65.666406" y="261.224043" style="fill: #1f77b4; stroke: #1f77b4"/>
</g>
</g>
<g id="text_19">
<text style="font-size: 8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start" x="80.066406" y="264.024043" transform="rotate(-0 80.066406 264.024043)">Input captured at layer 12</text>
</g>
<g id="line2d_23">
<path d="M 57.666406 273.224668
L 65.666406 273.224668
L 73.666406 273.224668
" style="fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: square"/>
<g>
<use xlink:href="#m5a845d07a0" x="65.666406" y="273.224668" style="fill: #ff7f0e; stroke: #ff7f0e"/>
</g>
</g>
<g id="text_20">
<text style="font-size: 8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start" x="80.066406" y="276.024668" transform="rotate(-0 80.066406 276.024668)">Input captured at layer 13</text>
</g>
<g id="line2d_24">
<path d="M 57.666406 285.225293
L 65.666406 285.225293
L 73.666406 285.225293
" style="fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #b2182b; stroke-width: 1.5"/>
</g>
<g id="text_21">
<text style="font-size: 8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start" x="80.066406" y="288.025293" transform="rotate(-0 80.066406 288.025293)">Heldout empirical screen = 0.05</text>
</g>
<g id="line2d_25">
<path d="M 57.666406 297.225918
L 65.666406 297.225918
L 73.666406 297.225918
" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #6a51a3; stroke-width: 1.5"/>
</g>
<g id="text_22">
<text style="font-size: 8px; font-family: 'DejaVu Sans', 'Bitstream Vera Sans', 'Computer Modern Sans Serif', 'Lucida Grande', 'Verdana', 'Geneva', 'Lucid', 'Arial', 'Helvetica', 'Avant Garde', sans-serif; text-anchor: start" x="80.066406" y="300.025918" transform="rotate(-0 80.066406 300.025918)">Proven uniform all-input bound</text>
</g>
</g>
</g>
</g>
<defs>
<clipPath id="pc5e8d93515">
<rect x="50.466406" y="37.922578" width="582.757656" height="170.258172"/>
</clipPath>
</defs>
</svg>

After

Width:  |  Height:  |  Size: 19 KiB