diff --git a/.gitattributes b/.gitattributes index e8690fc..7cfdd5b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,4 @@ results/** -text # Generated CSV uses RFC 4180 line endings; Matplotlib SVG paths contain spaces. results/**/*.csv -whitespace results/**/*.svg -whitespace +results/**/*.log -whitespace diff --git a/.gitignore b/.gitignore index 88f76cf..25b9d85 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,8 @@ __pycache__/ weights/ *.partial *.log + +# Large reproducible functional activations remain on the experiment server. +results/cpu-qwen35-2b/baseline-activations.npz +results/cpu-qwen35-2b/neuron-reuse-activations.npz +results/cpu-qwen35-2b/norm-repetition-input-profiles.npz diff --git a/README.md b/README.md index 5ef778a..f155e90 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,15 @@ # Model parameter similarity experiments -Bounded, inference-free analysis of official unquantized model weights. The first experiment -is **Qwen3.5-2B**. Model weights are downloaded and analyzed on the authorized remote host, -then deleted after archived results pass validation. No model weights are committed here. +Research on repeated learned functions in **Qwen3.5-2B**: a completed static weight baseline +and a training-free CPU functional pilot. The CPU study found similar existing RMSNorm +functions at layer indices 12/13, with a uniform analytic error bound and heldout substitution checks. +Cost is not a success condition. The completed static pilot was inference-free and its +weights were deleted after archival; the functional pilot executes streaming FP32 text forwards. +The current CPU functional checkpoint is retained until the user explicitly requests deletion. +No model weights are committed here. +- [CPU learned-function repetition findings](docs/CPU_FUNCTIONAL_FINDINGS.md) +- [Current learned-function repetition criterion](docs/LEARNED_FUNCTION_REPETITION.md) - [CPU-first feasibility and validation](docs/CPU_FIRST_VALIDATION.md) - [Training-free functional reuse plan (proposed; not executed)](docs/TRAINING_FREE_FUNCTIONAL_PLAN.md) - [Primary-source literature and applicability](docs/FUNCTIONAL_REUSE_LITERATURE.md) @@ -19,9 +25,10 @@ then deleted after archived results pass validation. No model weights are commit - `pilot/run.py`: one-model supervisor with bounded download wait and analysis timeouts. - `tests/test_pilot.py`: synthetic numerical and 3D correctness tests. -The primary result is **verified net byte savings versus direct relative weight error**. -This does not demonstrate preserved task capability or faster inference. Different block -scales are alternative partitions and their savings must not be summed. +The static baseline reports **verified net byte savings versus direct relative weight error**. +The functional study separately reports same-input error, an all-input RMSNorm function bound +and tiny heldout NLL changes; these are not broad task-accuracy guarantees. Different static +block scales are alternative partitions and their savings must not be summed. Dependencies are pinned in `requirements.txt`; install only prebuilt wheels on the remote host. Full resolved versions and run provenance accompany each result. See the SOP for commands. diff --git a/docs/CPU_FIRST_VALIDATION.md b/docs/CPU_FIRST_VALIDATION.md index 9623d02..e342dd9 100644 --- a/docs/CPU_FIRST_VALIDATION.md +++ b/docs/CPU_FIRST_VALIDATION.md @@ -28,7 +28,7 @@ NumPy 2.5.4，OPENBLAS_NUM_THREADS=1、OMP_NUM_THREADS=1，远程 timeout 60 秒 使用受信任的算子实现及小数组正确性检查；模型仓库代码不直接执行。 先确认首个短样本能以受控内存运行。不用不正确的“简化 Qwen”产生看似有价值的结果。 2. **真实分布上的候选**：batch=1、32 tokens，校准和留出各 4 段真实文本。 - 按顺序加载、计算、释放每层，保留很小的入口激活；探测选定的 3 个 full-attention 层对和 2 个 MLP 对。 + 按顺序加载、计算、释放每层，保留很小的入口激活；探测预先固定的 3 个 full-attention 层对、2 个 MLP 对，以及 6 个 full-attention 层内的 Q/K 归一化模块对，共 11 对。 每对在两个来源层的隐藏输入上分别测试，记录 JS、top-k、完整分支输出误差。 3. **最小干预证据**：最多对 1 个候选做单处替换；冻结参数，在留出文本上看输出变化， 条件允许时流式计算 token NLL。词表输出头分块归约 logsumexp，避免一次物化大 logits。 @@ -48,8 +48,28 @@ NumPy 2.5.4，OPENBLAS_NUM_THREADS=1、OMP_NUM_THREADS=1，远程 timeout 60 秒 - 先实现最小正确路径；线性状态递推、长上下文和生成速度不塞入首轮。 - 总计算 120 分钟，阶段预算见功能方案；无候选、超时或数值/实现核对失败均停止。 - 容量探测已完成，真实模型阶段尚未执行。后续需要恢复本实验自有的固定 revision 权重， - 分析后按既有规则校验归档并删除；不增加模型规模或自动租用 GPU。 + 权重持续保留，只有用户明确要求删除时才清理；不增加模型规模或自动租用 GPU。 CPU 上“找到输出接近的候选”能支持继续投入 GPU； “在指定小样本中单处复用保持质量且节省明确资源”才是初步正例。 两者都不能替代大规模能力验证。CPU 时延不外推为 GPU 加速率。 + +## 当前权重保留规则 + +用户于 2026-11-09 明确要求：本轮模型权重保留，只有用户明确要求删除时才删除。 +试验完成、超时、失败、归档或结果校验均不得自动触发清理。该要求覆盖先前逐模型分析后删除的约定。 +上一轮已删除的历史记录保留；本轮恢复下载的权重受新规则保护。 + +## 本轮执行补充 + +实际实验分支为 `codex/cpu-reuse-experiment`。在读取真实权重/功能指标前，新增固定的 +6 个层内 Q/K RMSNorm 增益共享候选，以检查较小粒度的功能复用；不是看过留出结果后扩充搜索。 +最多选择一个候选，必须分别验证两个替换方向；完整模块失败不会被小模块通过所掩盖。 +归一化共享最多节省少量字节，不预设能显著减少计算。 +采集真实隐藏输入必须正确执行前置线性注意力；它不进入首轮复用候选，但其递推计算纳入官方参考对照。 + +## 用户最新目标修订 + +不再以成本/加速作为成功条件；主问题是现有学习部分的功能重复。 +新增一次固定四层的 MLP 神经元函数搜索、独立留出和双向替换， +见 [功能重复判据](LEARNED_FUNCTION_REPETITION.md)。解析增益合并单独归类，不冒充学习部分重复。 diff --git a/docs/CPU_FUNCTIONAL_FINDINGS.md b/docs/CPU_FUNCTIONAL_FINDINGS.md new file mode 100644 index 0000000..a96c82a --- /dev/null +++ b/docs/CPU_FUNCTIONAL_FINDINGS.md @@ -0,0 +1,86 @@ +# CPU 先导结论：存在已训练归一化模块的相似功能重复 + +本轮主目标按用户最新要求为功能重复的存在性，不以成本/加速为成功条件。 +全部计算在 2 核、约 1.7 GiB RAM 的 CPU 服务器完成，无训练、缩放拟合或旋转搜索。 +层编号均为从 0 开始的索引。 + +## 一个具体正例 + +固定官方 Qwen3.5-2B checkpoint 的两个独立已训练模块： + +- `model.language_model.layers.12.input_layernorm` +- `model.language_model.layers.13.input_layernorm` + +二者均为 2048→2048 的 RMSNorm，eps=1e-6，使用训练后的有效增益 gamma=1+weight。 +增益标准差分别约 0.1901、0.1685，最小值约 0.5547、0.5703；不是零函数或恒定增益。 + +| 证据 | 结果 | +|---|---| +| 全输入域数学界（相对于层12输出） | 输出相对 L2 误差≤10.1852% | +| 反方向全输入域数学界（相对于层13输出） | 输出相对 L2 误差≤11.3402% | +| 4 条留出文本、两个来源层输入的最坏双向误差 | 4.5285% | +| 相同留出比较中的最低余弦 | 0.999487 | +| 层12使用层13，留出平均 token NLL 增量 | +0.0000325651 nat/token | +| 层13使用层12，留出平均 token NLL 增量 | −0.0003639730 nat/token | + +两方向逐文本 NLL 增量 P95 分别约 0.00098464、0.00070972，均通过原定小样本质量门槛。 +所有数值来自保存的逐文本、逐 token 原始数据。留出为 4 条不同的实验原创短文本， +每条截取 32 tokens；校准另有 4 条。它们不是广泛任务能力评测。 + +## 为什么不只是“权重看起来接近” + +对相同 eps 和维数，f_a(x)=diag(a)x/sqrt(mean(x²)+eps)。任意非零实数输入满足： + +||f_a(x)-f_b(x)|| / ||f_a(x)|| ≤ max_j |a_j-b_j|/|a_j|。 + +共同归一化分母消去；坐标基向量可达到最大值，因而该界是精确的最坏情况。 +反向分母换成 |b_j|。本轮先穷举 24 层同角色 input/post-attention norm 的 552 对， +仅按最小双向最坏界选定一对，未使用任何输入样本作选择。≤20% 界有 18 对，≤10% 无配对； +本轮只验证最优的一对，不根据留出结果换选。 + +这个确定性界直接给出两个已训练函数整体接近的证据。真实输入上的误差更小， +与只关心目标输入分布的目标一致。浮点实测与实数数学界分别报告。 +重新采集输入时，未修改的完整前向逐 bit 复现了最初保存的基线输出。 +输入探测分别使用两个来源层，对相同输入同时应用两套增益，避免比较不同输入的输出。 +从已加增益的激活还原共同归一化输入时有额外 FP32 舍入，未将其称为 bitwise 等价。 + +结论限定为两个已有 RMSNorm 模块实现相似映射；不把共同包含 RMS 操作本身当作证据。 +二者学习增益不同且非恒定，输出非零。完整 attention/MLP 主体的重复仍未得到正例。 + +## 已完成的阴性搜索 + +1. 原定 3 对完整 attention、2 对完整 MLP、6 对层内 Q/K norm 均未通过预设门槛。 + attention 检索分布 JS 范围约 0.145～0.318，也未通过 0.01 的门槛； + 未进行头排列、连续坐标变换或训练对齐，不能外推为不存在其他可复用表示。 +2. 在层 5、6、20、21 各 6144 个 MLP scalar 子函数中，排除低活跃和近常量单元后， + 共精确筛查 7,450,139 个唯一同层配对。要求双向相对误差≤20%、余弦≥0.98、 + Pearson≥0.95，无通过者。没有针对留出集不断更换候选或放宽门槛。 +3. 这些结果只覆盖声明的模块、输入和阈值，不否定其他粒度/角色的功能重复。 + +## 单独归类的解析恒等式 + +Q/K 未参与 RoPE 的 192 个维度，其增益只以乘积影响点积，可合并到任一侧。 +96 组真实输入检查最大分数差 3.8147e-6、最大概率差 8.0466e-7；固定第3层两个方向 +留出 NLL 变化接近 FP32 舍入。详见 [推导](QK_GAIN_FACTORIZATION.md)。 +这是重参数化等价，不作为“两个已有学习部分功能重复”的正例。先前性能测试原始数据 +保留用于溯源，成本不属于当前结论的成功条件。 + +## 正确性、资源与保留规则 + +- 固定模型 revision：15852e8c16360a2fea060d615a32b45270f8a8fc，原始权重 SHA256 校验通过。 +- 流式读取原始 BF16，逐块转为 FP32，不同时载入全模型。 +- 与固定 Transformers 官方定义比较的 22 项检查通过，最大差约 5.4e-7。 +- 保留早期参考适配器错误、临时分区空间不足及两次过于保守的换页停止记录。 + 只归还自身临时堆页，未改变系统 swap 设置；后续有界前向完成。 +- 计算预算跨重启累计，初始保守计入 180 秒先前检查预留；具体时长及峰值见归档日志。 +- 当前权重按用户最新指令持续保留，只在用户明确要求删除时清理。 + +原始表、样本/token IDs、激活、NLL、证明界与图表见 `results/cpu-qwen35-2b/`。 +后续 GPU 阶段可增加长度、语言和任务来验证正例的稳健性；本轮不自动启动新模型或扩展搜索。 + +## Git 交付与保留缓存 + +全部图表源 CSV/JSON、逐 token NLL、证明界和日志已交付。三份较大的中间激活缓存 +保留在原服务器及本地完整归档，路径、大小、SHA256 见结果目录的 external-activation-cache.json。 +其完整原实验校验表为 checksums-full-server.json；Git交付包校验表为 checksums-functional.json。 +没有删除模型或缓存；这些缓存可由固定代码和样本重新生成，也可从服务器按哈希取回。 diff --git a/docs/LEARNED_FUNCTION_REPETITION.md b/docs/LEARNED_FUNCTION_REPETITION.md new file mode 100644 index 0000000..490d492 --- /dev/null +++ b/docs/LEARNED_FUNCTION_REPETITION.md @@ -0,0 +1,58 @@ +# 当前目标：已有学习部分是否重复实现相似函数 + +用户于 2026-11-09 明确：不以成本或加速作为成功条件，只关心相似功能重复是否存在。 +权重继续保留，只有用户明确要求删除才清理；不训练、不拟合旋转或缩放。 + +## 什么可以作为正例 + +两个不同的已训练部分，在同一真实输入上产生接近、且并非恒定/接近零的输出； +独立留出输入仍满足预先固定的门槛。实际双向替换只作补充验证。 +不能仅因替换一小部分后整模型 NLL 几乎不变，就认定它们实现相同函数。 + +Q/K 增益的解析合并证明同一计算的重参数化自由度，不等于两个现成学习部分功能重复。 +前一轮 11 对完整/归一化模块未通过门槛，也不能否定更小粒度的重复。 +不将两个模块都包含 SiLU、矩阵乘法等共同运算本身作为重复功能证据。 + +## 新增一次有界搜索：MLP 学习神经元 + +在已保存的第 5、6、20、21 层真实 MLP 入口输入上，研究 + +f_(l,j)(h) = SiLU(w_gate_(l,j)^T h) × (w_up_(l,j)^T h)。 + +每层 6,144 个神经元具有不同学习参数。比较同一层内两个 scalar 子函数， +不是比较来自不同输入的两个观测值，也不是把整层或整 tensor 称作重复。 +下游 down-projection 的列保持各自原样；替换发生在两个投影和 SiLU 之后的激活值处。 + +1. 4 条校准文本、4 条留出文本，每条 32 tokens，中文/英文各半，沿用先前冻结样本。 +2. 校准阶段只保留 RMS 至少为该层神经元中位数、std/RMS≥0.25、能量有效 token 数≥16 + 的非平凡单元。能量有效样本数定义为 (sum f²)² / sum f⁴，不是独立统计样本量。 +3. 对这些单元做分块穷举；实际配对数必须记录。不用随机 hidden 向量作主要证据。 +4. 预先固定门槛：双向直接相对 L2≤20%、余弦≥0.98、Pearson≥0.95；同时输出 5/10/20% + 三档校准配对数量。没有拟合线性缩放、偏置或 R。 +5. 全部层中仅按校准的最小最坏方向误差选择 1 对，冻结选择；无通过者停止。 +6. 留出集要求同样的汇总函数门槛及活跃度条件，逐文本双向误差≤35%。不根据留出失败换候选。 +7. 只有函数验证通过，才做两个方向的独立网络替换；留出平均 NLL 增量≤0.01 nat/token， + 逐文本增量 P95≤0.05。它不是主要的功能重复判据，防止因单元贡献很小而得到假正例。 + +本阶段使用先前两小时累计预算的剩余额度，复用现有隐藏输入；不增加模型、样本规模或成本测试。 +神经元层面的经验正例不证明完整层可以互换、不证明全输入空间等价，也不能推断独立任务能力。 +留出只有 4 条文本，结论必须限定到此短文本先导分布；后续扩展文本才能检验稳健性。 + +## 同角色学习 RMSNorm：确定性函数误差界 + +静态试验已经发现部分语言层 post-attention norm 的参数接近。新增一次独立检验： +比较 24 层各自的 input/post-attention RMSNorm（同角色内共 552 对）， +仅按训练后增益的全输入最坏误差界选 1 对，完全不依赖校准或留出输入。 +增益需非零且不是常量向量（std≥0.001），不把没有学习参数的共同归一化运算当作正例。 + +相同 eps、相同形状下，f_a(x)=diag(a)x/sqrt(mean(x²)+eps)。对任意非零 x， + +||f_a(x)-f_b(x)|| / ||f_a(x)|| ≤ max_j |a_j-b_j|/|a_j|。 + +反方向分母换为 |b_j|。该上界由相应坐标基向量达到，是实数算术下的精确最坏界。 +因此可先证明两个现成学习模块的函数整体接近，而不只比较原始参数 w；实际增益为 1+w。 +选择门槛：最坏双向界≤20%；同时记录≤5%、≤10%、≤20% 配对数量。只选最优界的一对。 +随后重新采集其实际输入，要求留出逐文本直接双向输出误差≤5%，再测两个方向网络替换。 +函数接近与替换后网络 NLL 分开报告；后者沿用已有 0.01/0.05 门槛，不决定函数存在性本身。 +选定后不根据留出结果换配对；不搜索视觉模块、不增加模型、不以成本为条件。 +归一化层正例不能推广为 attention 或 MLP 主体存在同样重复。 diff --git a/docs/QK_GAIN_FACTORIZATION.md b/docs/QK_GAIN_FACTORIZATION.md new file mode 100644 index 0000000..91fff58 --- /dev/null +++ b/docs/QK_GAIN_FACTORIZATION.md @@ -0,0 +1,46 @@ +# Q/K 未旋转维度的确定性增益合并（已数值验证，归为重参数化） + +这是从已核对架构直接推导的恒等式，不是看留出指标后扩大近邻搜索。 +原定 11 对候选保留原协议；本项是独立的解析对照，不训练、不求 R，也不选择层。 +预先固定完整网络干预层为第 3 层，分别验证两个移动方向；不根据数据换层。 + +## 恒等式与前提 + +以完成 RMS 单位化、尚未乘可学习增益的 q/k 为 u/v，Q/K 增益向量为 a/b。 +固定 checkpoint 的 norm 参数实际使用 a=1+w_Q，b=1+w_K。 +模型 head_dim=256，partial_rotary_factor=0.25：前 64 维旋转，后 192 维不旋转。 +逐头共享的增益向量与 GQA 结构已由官方代码确认。 + +对于后 192 维，任意 query/key 位置 p/s： + +sum_j (a_j u_pj)(b_j v_sj) = sum_j u_pj ((a_j b_j) v_sj) + = sum_j ((a_j b_j) u_pj) v_sj。 + +前 64 维保留原增益和位置旋转。因而完整 QK 点积在实数算术中相等，softmax、V、 +输出门控和 O 不变时，完整 attention 分支也是同一个函数。这个结论不依赖某一输入分布。 +浮点乘法次序改变，不能承诺逐 bit 相等；需报告实际误差并验证留出 NLL。 +不能把这一规则用于旋转维度，或未经核对的逐头独立增益架构。 + +## 验证与成本边界 + +1. 在已有真实 q/k 输入上，检查 6 个 full-attention 层的原始/两个合并方向分数、概率及输出。 +2. 固定第 3 层进行两个方向的独立网络替换，不叠加，使用原留出文本和质量阈值。 +3. 记录预先缓存增益后、相同 CPU 环境下原版与合并版局部归一化耗时；不外推 GPU 性能。 +4. 双精度数学推导和 FP32 实测分开。没有消除 Q/K 投影或 RMS 方差计算。 + +将 Q 的未旋转增益移入 K 时，每 token、每层少做 8×192=1,536 次逐元素增益乘法； +反方向少做 2×192=384 次。它们只是归一化里的小部分操作，不是整个模型的百分比加速。 +NumPy 切片/拼接可能抵消这点收益，必须报告测得的局部时延。 + +参数量从两个 256 维增益变为两个 64 维旋转增益及一个 192 维联合增益，即 512→320 个数。 +但原增益以 BF16 存储、联合乘积以 FP32 保留时，载荷仍是 1,024 字节，没有 checkpoint +净压缩；元数据还可能增加。FP32 运行时缓存增益的理论载荷 2,048→1,280 字节， +不等于已测得整个模型 RSS 降低。另行 BF16 量化乘积会增加误差，不属于本次恒等式验证。 + +这是同一注意力计算内部的重参数化与冗余消除，不是两个现成 attention 层能互换的证明。 +若得到正例，应将它作为小粒度确定性优化机会；较大模块互换仍由原候选试验单独回答。 + +## 完成状态 + +96 组真实输入分数/概率检查及固定第3层双向网络检查均通过。 +该结果独立归类；主目标的已训练函数重复正例见 [CPU 功能结论](CPU_FUNCTIONAL_FINDINGS.md)。 diff --git a/docs/SOP.md b/docs/SOP.md index 80689ab..8fb3b6e 100644 --- a/docs/SOP.md +++ b/docs/SOP.md @@ -121,3 +121,20 @@ validation. See the [bounded plan](TRAINING_FREE_FUNCTIONAL_PLAN.md) and [primary-source literature](FUNCTIONAL_REUSE_LITERATURE.md). No forward pass, new download, or new model is started by adopting this document. The original static pilot remains inference-free; the proposed follow-up requires a separately confirmed execution environment. + +## Current functional-run retention override + +On 2026-11-09 the user explicitly changed retention: keep the CPU functional pilot +checkpoint until the user explicitly requests deletion. Completion, timeout, failure +and archive validation do not authorize cleanup. The earlier pilot cleanup remains +a historical action; do not apply its automatic cleanup step to the current run. + +## CPU functional follow-up outcome + +The executed CPU study found a learned RMSNorm function-repetition example at text +layer indices 12/13. Use same-input function error and a uniform analytic bound as +primary evidence, with separate heldout network substitution checks. Costs are not +a success condition under the latest user instruction. See the +[executed findings](CPU_FUNCTIONAL_FINDINGS.md) and +[function-repetition criterion](LEARNED_FUNCTION_REPETITION.md). +Keep whole-module, neuron, normalization and algebraic-reparameterization results distinct. diff --git a/functional/README.md b/functional/README.md new file mode 100644 index 0000000..6aa8367 --- /dev/null +++ b/functional/README.md @@ -0,0 +1,91 @@ +# Bounded CPU functional pilot + +This is the execution implementation of the CPU-first plan, not the completed static +parameter pilot. Frozen Qwen3.5-2B weights are decoded to FP32 a projection block at a +time. CPU only, no autograd/optimization, no rotation fitting, no model-hub Python code. +The user explicitly requires retaining this checkpoint until they request deletion. + +## Fixed scope + +- Batch of eight distinct experiment-authored plain texts, first 32 tokens each: + four calibration and four heldout, two Chinese/two English in each split. +- Full attention pairs: (3,7), (11,15), (19,23). +- MLP pairs: (5,6), (20,21). +- Smaller-granularity Q/K RMSNorm pairs inside layers 3,7,11,15,19,23. + Added before observing real weights/functions for this pilot; these are small storage + opportunities, not a claim of substantial compute savings. +- At most one calibration-selected candidate, two separate substitution directions. + Selection prefers component size, then worst symmetric output error. No heldout-driven retry. +- Same-input relative L2 <=0.05 in both directions and token P95 <=0.1 in both directions; + attention additionally requires mean JS <=0.01 and top-5 overlap >=0.8. +- Heldout mean NLL increase <=0.01 nat/token and per-sequence P95 increase <=0.05. + These are pilot decision criteria, not mathematical capability guarantees. + +## Source and checks + +`reference.json` pins the official Transformers Qwen3.5 source commit, URL and hash. +`reference_check.py` extracts only named mathematical definitions from that exact file, +removes optional fused-kernel decorators, and compares the NumPy implementation against +Torch operators, full toy decoder layers and a two-layer text forward. Chunked delta and +recurrent delta implementations are both checked, including sequence padding boundaries. +The official source is Apache-2.0, copyright Qwen Team and HuggingFace; the reference file +is fetched for testing and is not bundled here. FP32 equivalence is checked, not equality +to every deployed BF16 kernel. + +`tests/test_functional.py` independently checks streamed projection/embedding/NLL, +causality, causal convolution, normalization conventions and basic metric identities. +These tests use tiny synthetic arrays. Real model evidence is produced only by `run.py`. + +## Remote execution + +Use the previously authorized CPU host and existing NumPy/Matplotlib environment. Install +only binary wheels: tokenizers==0.23.3 from HTTPS PyPI; torch==2.15.0+cpu from the official +CPU wheel index for independent reference checks. Use an experiment-owned on-disk TMPDIR: +the shared host's /tmp tmpfs was too small during the first wheel download. Do not change +system swap or shared temporary files. + +```sh +OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 venv/bin/python -B -m unittest tests.test_functional -v +OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 venv/bin/python -B -m functional.reference_check --source runs/cpu-qwen35-2b/reference.py --output runs/cpu-qwen35-2b/reference-check.json +OPENBLAS_NUM_THREADS=1 OMP_NUM_THREADS=1 venv/bin/python -B -m functional.supervise --root runs/cpu-qwen35-2b +``` + +The supervisor waits at most one hour for binary dependencies and four hours for the +already started pinned, verified checkpoint download. It does not install, retry downloads, +search extra candidates, mutate Git or delete weights. After reference checks, it runs one +8-token real text capacity check, then the bounded functional pilot. Reference, preflight +and functional compute together are capped at two hours with a reporting reserve; +download wait is recorded separately. Projection memory is checked, RSS soft stop is +512 MiB, available-system-memory floor 256 MiB; sustained host swap IO combined with low available memory or >128 MiB process swap also stops the run. +A persistent ledger counts computation across restarts; early checks/preflight receive a conservative 180-second reserve. + +`preflight.py` is a capacity smoke check, not a semantic oracle. `run.py` records exact +sample/token IDs, fixed configuration, source hashes, activations, per-text/per-head metrics, +selection, per-token NLL, intervention results and truthful stop coverage. `report.py` reads +saved data to produce PNG/PDF/SVG figures without reopening weights. + +No generation throughput is measured in this short-text pilot. Any parameter-tying storage +benefit must be accounted for separately, including metadata. Neither same-input similarity +nor favorable tiny heldout NLL demonstrates broad mutual interchangeability or GPU speedup. + +## Current primary result and completed extensions + +The latest user goal is learned-function repetition, independent of costs. Existing +input RMSNorm modules at layer indices 12/13 have a proven worst two-direction real-arithmetic +error bound of 11.3402%, worst observed heldout relative error 4.5285%, minimum cosine 0.999487, +and small bidirectional heldout NLL changes. See [findings](../docs/CPU_FUNCTIONAL_FINDINGS.md). + +- `neuron_reuse.py`: fixed four-layer scalar MLP function search, nontrivial activity + gates, calibration-only selection and independent heldout/substitution logic; no pair passed. +- `norm_repetition.py`: same-role learned RMSNorm bound across 552 pairs, one deterministic + selection independent of any input samples, real probes and two full-network substitutions. +- `gain_factorization.py`: architecture-derived identity on non-RoPE coordinates; a separate + reparameterization result, not a learned-function repetition example. +- `benchmark_factorization.py`: historical one-attempt CPU allocation optimization and + bitwise equivalence check. Performance is no longer a success condition. +- `storage.py`: explicit component layout accounting; the original candidate run selected + no pair. This does not affect the primary function-repetition conclusion. + +Do not expand the final positive claim to full attention or MLP bodies. Retain the +fixed checkpoint until the user explicitly requests deletion. Full tests and audit are +run remotely; no local build or model execution is required to inspect the archived results. diff --git a/functional/__init__.py b/functional/__init__.py new file mode 100644 index 0000000..faee6dc --- /dev/null +++ b/functional/__init__.py @@ -0,0 +1 @@ +"""CPU-only, training-free functional reuse pilot.""" diff --git a/functional/audit.py b/functional/audit.py new file mode 100644 index 0000000..79c9fdf --- /dev/null +++ b/functional/audit.py @@ -0,0 +1,86 @@ +"""Audit saved functional evidence and archive only outputs, never checkpoint weights.""" +import argparse +import csv +import hashlib +import json +from pathlib import Path +import platform +import subprocess +import time +import numpy as np +from .model import StreamingLM +from .norm_repetition import uniform_bounds +from .run import write_json + +EXCLUDE={'tokenizer.json','reference.py','checksums-functional.json','audit-validation.json','archive-files.json'} + + +def archive_files(root): + files=[p for p in root.iterdir() if p.is_file() and p.name not in EXCLUDE and not p.name.endswith(('.partial','.safetensors'))] + files += [p for p in (root/'figures').glob('*') if p.is_file()] + return sorted(files) + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--root',type=Path,required=True);a=p.parse_args();root=a.root + manifest=json.loads((root/'manifest.json').read_text()) + assert manifest['revision']=='15852e8c16360a2fea060d615a32b45270f8a8fc' + for f in manifest['files']: + if f['name'].endswith('.safetensors'): + path=root/'weights'/f['name'] + assert path.is_file() and path.stat().st_size==f['bytes'] + assert f['sha256']==f['upstream_sha256'] + retention=json.loads((root/'retention-policy.json').read_text());assert retention['automatic_cleanup'] is False + reference=json.loads((root/'reference-check.json').read_text()) + assert reference['passed'] and len(reference['tests'])==22 + assert reference['implementation_sha256']==hashlib.sha256(Path('functional/model.py').read_bytes()).hexdigest() + tests=(root/'final-tests.txt').read_text();assert 'Ran 12 tests' in tests and '\nOK\n' in tests + result=json.loads((root/'norm-repetition-summary.json').read_text());assert result['state']=='completed' + assert result['learned_function_repetition_positive'] and result['bidirectional_network_quality_pass'] + cfg=json.loads((root/'model_config.json').read_text());lm=StreamingLM(root/'weights',cfg,seconds=60) + c=result['selected'];ga=1+lm.small(f'layers.{c["a"]}.{c["role"]}.weight');gb=1+lm.small(f'layers.{c["b"]}.{c["role"]}.weight') + bounds=uniform_bounds(ga,gb) + np.testing.assert_allclose(bounds,[c['a_to_b_uniform_bound'],c['b_to_a_uniform_bound']],atol=1e-14,rtol=1e-14) + with (root/'norm-repetition-uniform-bounds.csv').open() as f:assert len(list(csv.DictReader(f)))==552 + baseline=np.load(root/'baseline-heldout-token-nll.npy');assert baseline.shape==(4,31) + for rec in result['interventions']: + nll=np.load(root/f'norm-repetition-{rec["target"]}-uses-{rec["donor"]}-token-nll.npy') + delta=nll.mean(1)-baseline.mean(1) + np.testing.assert_allclose(delta,rec['sequence_nll_deltas'],atol=1e-13,rtol=1e-13) + assert abs(delta.mean()-rec['mean_nll_delta'])<1e-13 + with (root/'norm-repetition-function-metrics.csv').open() as f:rows=list(csv.DictReader(f)) + held=[r for r in rows if r['split']=='heldout'];assert len(held)==8 + worst=max(max(float(r['relative_l2']),float(r['reverse_relative_l2'])) for r in held) + assert abs(worst-result['worst_heldout_relative_l2'])<1e-14 + neuron=json.loads((root/'neuron-reuse-search-summary.json').read_text());assert sum(x['eligible_unique_pairs_checked'] for x in neuron)==7450139 + ledger=json.loads((root/'compute-budget-ledger.json').read_text());assert ledger['spent_seconds']+600<7200 + environment={'python':platform.python_version(),'platform':platform.platform(),'numpy':np.__version__, + 'packages':subprocess.run([str(Path(__import__('sys').executable)),'-m','pip','freeze'],capture_output=True,text=True,check=True).stdout.splitlines(), + 'weights_retained':True,'no_automatic_cleanup':True,'ledger_seconds_including_initial_reserve':ledger['spent_seconds'], + 'final_tests_report_audit_additional_budget_upper_reserve_seconds':600, + 'historical_error_logs':'Early reference adapter errors and preflight pressure stops; latest summaries/tests are authoritative.'} + write_json(root/'functional-environment.json',environment) + source_paths=[*Path('functional').glob('*.py'),Path('functional/reference.json'),Path('pilot/tensors.py'),Path('tests/test_functional.py')] + write_json(root/'final-source-hashes.json',{str(p):hashlib.sha256(p.read_bytes()).hexdigest() for p in sorted(source_paths)}) + files=archive_files(root) + # Every numeric CSV value must be finite. Empty diagnostic fields stay unknown. + for path in files: + if path.suffix=='.csv': + with path.open() as f: + for row in csv.DictReader(f): + for value in row.values(): + if value in (None,''):continue + try:number=float(value) + except ValueError:continue + assert np.isfinite(number),(path,value) + checksums={str(p.relative_to(root)):hashlib.sha256(p.read_bytes()).hexdigest() for p in files} + write_json(root/'checksums-functional.json',checksums) + selected=[*checksums,'checksums-functional.json','audit-validation.json','archive-files.json'] + write_json(root/'archive-files.json',selected) + write_json(root/'audit-validation.json',{'passed':True,'validated_unix':time.time(),'checksummed_files':len(checksums), + 'proof_recomputed_from_retained_weights':True,'heldout_nll_recomputed_from_saved_arrays':True, + 'weights_present_and_retained':True,'archive_excludes_weights_and_tokenizer':True}) + print(json.dumps({'passed':True,'checksummed_files':len(checksums),'weights_retained':True})) + + +if __name__=='__main__':main() diff --git a/functional/benchmark_factorization.py b/functional/benchmark_factorization.py new file mode 100644 index 0000000..41dc357 --- /dev/null +++ b/functional/benchmark_factorization.py @@ -0,0 +1,83 @@ +"""One allocation-reduction attempt, with a similarly optimized baseline. + +Does not change weights or run the network again. Checks bitwise equality to the +network-tested factorization on every captured real Q/K input before timing. +""" +import argparse +import json +from pathlib import Path +import resource +import time +import numpy as np +from .model import StreamingLM +from .gain_factorization import consolidate,normalized,LAYERS +from .run import save_csv,write_json + + +def baseline_inplace(q,k,a,b): + q=normalized(q);q*=a + k=normalized(k);k*=b + return q,k + + +def factored_inplace(q,k,a,b,r,combined,move='q_to_k'): + q,k=normalized(q),normalized(k) + if move=='q_to_k': + q[...,:r]*=a[:r];k*=combined + else: + q*=combined;k[...,:r]*=b[:r] + return q,k + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);a=p.parse_args();root=a.root + ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text());remaining=int(7200-ledger['spent_seconds']-20) + if remaining<60:raise TimeoutError('Cumulative budget insufficient') + start=time.monotonic();result={'state':'running'} + try: + cfg=json.loads((root/'model_config.json').read_text());lm=StreamingLM(root/'weights',cfg,seconds=min(remaining,300)) + r=64;data=np.load(root/'baseline-activations.npz');checks=[] + for layer in LAYERS: + lm.guard();pfx=f'layers.{layer}.self_attn.' + ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight') + q,k=data[f'qraw_{layer}'],data[f'kraw_{layer}'] + for move in ('q_to_k','k_to_q'): + combined=np.concatenate([(bg if move=='q_to_k' else ag)[:r],ag[r:]*bg[r:]]) + expected=consolidate(q,k,ag,bg,r,move) + actual=factored_inplace(q,k,ag,bg,r,combined,move) + for x,y in zip(expected,actual):np.testing.assert_array_equal(x,y) + checks.append({'layer':layer,'direction':move,'query_elements':int(q.size),'key_elements':int(k.size),'bitwise_equal':True}) + q,k=data['qraw_3'][:1],data['kraw_3'][:1];pfx='layers.3.self_attn.' + ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight') + combined=np.concatenate([bg[:r],ag[r:]*bg[r:]]) + funcs={'baseline_legacy':lambda:(normalized(q)*ag,normalized(k)*bg), + 'baseline_inplace':lambda:baseline_inplace(q,k,ag,bg), + 'factored_inplace':lambda:factored_inplace(q,k,ag,bg,r,combined)} + for _ in range(3): + for fn in funcs.values():fn() + timing=[];names=list(funcs) + for repeat in range(30): + order=names[repeat%3:]+names[:repeat%3] + for name in order: + lm.guard();t=time.perf_counter() + for _ in range(50):funcs[name]() + timing.append({'repeat':repeat,'implementation':name,'seconds_per_call':(time.perf_counter()-t)/50,'calls_per_block':50}) + save_csv(root/'gain-factorization-optimized-timing.csv',timing) + values={name:np.array([row['seconds_per_call'] for row in timing if row['implementation']==name]) for name in names} + ratio=values['baseline_inplace']/values['factored_inplace'] + medians={name:float(np.median(v)) for name,v in values.items()} + result.update(state='completed',bitwise_checks=checks,medians_seconds=medians, + paired_baseline_over_factored_median=float(np.median(ratio)), + paired_ratio_p10=float(np.quantile(ratio,.1)),paired_ratio_p90=float(np.quantile(ratio,.9)), + fraction_blocks_factored_faster=float(np.mean(ratio>1)), + descriptive_local_gain=bool(np.median(ratio)>1.05 and np.mean(ratio>1)>=.8 and np.quantile(ratio,.1)>1), + claim_limit='Hot-array CPU normalization microbenchmark against equally allocation-optimized baseline; no full-model speedup or independent statistical significance claim.') + except Exception as exc: + result.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise + finally: + elapsed=time.monotonic()-start;result.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.benchmark_factorization','seconds':elapsed});write_json(ledger_path,ledger) + write_json(root/'gain-factorization-optimized-summary.json',result) + + +if __name__=='__main__':main() diff --git a/functional/gain_factorization.py b/functional/gain_factorization.py new file mode 100644 index 0000000..6bcd33a --- /dev/null +++ b/functional/gain_factorization.py @@ -0,0 +1,135 @@ +"""Architecture-derived Q/K gain consolidation on non-RoPE dimensions only. + +Fixed layer3 interventions; no fitting, layer selection or learned alignment. +Run only after baseline/candidate pilot finishes, within the same cumulative budget. +""" +import argparse +import csv +import json +from pathlib import Path +import resource +import time +import numpy as np +from .model import StreamingLM,norm,rope +from .run import probability_metrics,write_json,save_csv + +LAYERS=[3,7,11,15,19,23] + + +def normalized(x): + return x/np.sqrt(np.mean(x*x,axis=-1,keepdims=True)+np.float32(1e-6)) + + +def consolidate(q,k,a,b,r,move,joint=None): + q,k=normalized(q),normalized(k) + c=a[r:]*b[r:] if joint is None else joint + if move=='q_to_k': + q=np.concatenate([q[...,:r]*a[:r],q[...,r:]],axis=-1) + k=np.concatenate([k[...,:r]*b[:r],k[...,r:]*c],axis=-1) + else: + q=np.concatenate([q[...,:r]*a[:r],q[...,r:]*c],axis=-1) + k=np.concatenate([k[...,:r]*b[:r],k[...,r:]],axis=-1) + return q,k + + +class FactoredLM(StreamingLM): + def configure(self,layer,move): + self.factor_layer=layer;self.move=move + p=f'layers.{layer}.self_attn.' + self.a=1+self.small(p+'q_norm.weight');self.b=1+self.small(p+'k_norm.weight') + self.r=int(self.config['head_dim']*self.config['rope_parameters']['partial_rotary_factor']) + self.joint=self.a[self.r:]*self.b[self.r:] + + def normalization(self,x,name): + p=f'layers.{self.factor_layer}.self_attn.' + if name not in (p+'q_norm',p+'k_norm'):return super().normalization(x,name) + # This path does not multiply the unit-gain tail, unlike a full-vector mask. + unit=x/np.sqrt(np.mean(x*x,axis=-1,keepdims=True)+self.eps) + isq=name==p+'q_norm';head=self.a if isq else self.b + skip=(isq and self.move=='q_to_k') or (not isq and self.move=='k_to_q') + tail=unit[...,self.r:] if skip else unit[...,self.r:]*self.joint + return np.concatenate([unit[...,:self.r]*head[:self.r],tail],axis=-1) + + +def main(): + p=argparse.ArgumentParser();p.add_argument('--root',required=True,type=Path);a=p.parse_args();root=a.root + # Protocol is fixed before inspecting functional measurements. + protocol={'fixed_intervention_layer':3,'directions':['q_to_k','k_to_q'],'diagnostic_layers':LAYERS,'no_selection':True,'no_training':True,'precision':'FP32 product; no BF16 requantization','quality_gate':{'mean_nll_delta':.01,'sequence_p95_nll_delta':.05}} + protocol_path=root/'gain-factorization-protocol.json' + if protocol_path.exists(): + old=json.loads(protocol_path.read_text());assert all(old[k]==v for k,v in protocol.items()), 'Protocol differs from preregistration' + else:write_json(protocol_path,{**protocol,'registered_unix':time.time()}) + ledger_path=root/'compute-budget-ledger.json';ledger=json.loads(ledger_path.read_text()) + remaining=int(7200-ledger['spent_seconds']-30) + if remaining<60:raise TimeoutError('No remaining cumulative budget for deterministic identity verification') + start=time.monotonic() + report={'state':'running','records':[],'interventions':[]} + try: + cfg=json.loads((root/'model_config.json').read_text());c=cfg['text_config'];r=int(c['head_dim']*c['rope_parameters']['partial_rotary_factor']) + assert c['rms_norm_eps']==1e-6 + assert c['head_dim']==256 and r==64 and c['num_attention_heads']==8 and c['num_key_value_heads']==2 + lm=FactoredLM(root/'weights',cfg,seconds=remaining) + data=np.load(root/'baseline-activations.npz');samples=json.loads((root/'samples-tokenized.json').read_text())['samples'] + rp=c['rope_parameters'];repeats=c['num_attention_heads']//c['num_key_value_heads'] + records=[] + for layer in LAYERS: + lm.guard() + pfx=f'layers.{layer}.self_attn.' + aw=lm.small(pfx+'q_norm.weight');bw=lm.small(pfx+'k_norm.weight');ag,bg=1+aw,1+bw + q,k=data[f'qraw_{layer}'],data[f'kraw_{layer}'] + qr,kr=rope(norm(q,aw),rp['rope_theta'],rp['partial_rotary_factor']),rope(norm(k,bw),rp['rope_theta'],rp['partial_rotary_factor']) + ref=np.einsum('bthd,bshd->bhts',qr,np.repeat(kr,repeats,axis=2))/np.float32(np.sqrt(c['head_dim'])) + mask=np.triu(np.ones(ref.shape[-2:],dtype=bool),1) + def probs(s): + s=np.where(mask,-np.inf,s);e=np.exp(s-s.max(-1,keepdims=True));return e/e.sum(-1,keepdims=True) + pref=probs(ref) + for move in ('q_to_k','k_to_q'): + qq,kk=consolidate(q,k,ag,bg,r,move) + qq,kk=rope(qq,rp['rope_theta'],rp['partial_rotary_factor']),rope(kk,rp['rope_theta'],rp['partial_rotary_factor']) + changed=np.einsum('bthd,bshd->bhts',qq,np.repeat(kk,repeats,axis=2))/np.float32(np.sqrt(c['head_dim'])) + pp=probs(changed) + for bi,s in enumerate(samples): + diff=changed[bi].astype(np.float64)-ref[bi] + records.append({'layer':layer,'direction':move,'sample_id':s['id'],'split':s['split'], + 'score_max_abs_error':float(np.abs(diff).max()),'score_relative_l2':float(np.linalg.norm(diff)/max(np.linalg.norm(ref[bi].astype(np.float64)),1e-12)), + 'probability_max_abs_error':float(np.abs(pp[bi]-pref[bi]).max()),**probability_metrics(pref[bi:bi+1],pp[bi:bi+1])}) + save_csv(root/'gain-factorization-score-metrics.csv',records) + held=np.array([i for i,s in enumerate(samples) if s['split']=='heldout']);ids=np.array([s['token_ids'] for s in samples],dtype=np.int64) + base=np.load(root/'baseline-heldout-token-nll.npy').mean(1) + for move in ('q_to_k','k_to_q'): + lm.configure(3,move) + h,_=lm.forward(ids[held]) + nll=lm.nll(h,ids[held]);np.save(root/f'gain-factorization-{move}-token-nll.npy',nll) + delta=nll.mean(1)-base + report['interventions'].append({'layer':3,'direction':move,'sequence_nll_deltas':delta.tolist(),'mean_nll_delta':float(delta.mean()),'sequence_p95_delta':float(np.quantile(delta,.95)),'passed':bool(delta.mean()<=.01 and np.quantile(delta,.95)<=.05)}) + write_json(root/'gain-factorization-summary.json',report) + # Local normalization timing uses precomputed gains for both implementations. + q,k=data['qraw_3'][:1],data['kraw_3'][:1];pfx='layers.3.self_attn.' + ag,bg=1+lm.small(pfx+'q_norm.weight'),1+lm.small(pfx+'k_norm.weight') + def baseline():return normalized(q)*ag,normalized(k)*bg + joint=ag[r:]*bg[r:] + def factored():return consolidate(q,k,ag,bg,r,'q_to_k',joint=joint) + for _ in range(3):baseline();factored() + timing=[] + for repeat in range(10): + for label,fn in (('baseline',baseline),('factored',factored)) if repeat%2==0 else (('factored',factored),('baseline',baseline)): + lm.guard();t=time.perf_counter() + for _ in range(20):fn() + timing.append({'repeat':repeat,'implementation':label,'seconds_per_call':(time.perf_counter()-t)/20}) + save_csv(root/'gain-factorization-kernel-timing.csv',timing) + report.update(state='completed',maximum_score_abs_error=max(x['score_max_abs_error'] for x in records),maximum_probability_abs_error=max(x['probability_max_abs_error'] for x in records), + bidirectional_heldout_pass=all(x['passed'] for x in report['interventions']), + elementwise_gain_multiplies_saved_per_token_layer={'q_to_k':1536,'k_to_q':384}, + original_bf16_payload_bytes=1024,factored_lossless_mixed_payload_bytes=1024, + fp32_cached_gain_payload_before=2048,fp32_cached_gain_payload_after=1280, + claim_limit='Exact real-arithmetic identity only on non-RoPE coordinates; empirical FP32/NLL checks on tiny text pilot. Not evidence that two existing full layers interchange, not model-wide speedup.') + except Exception as exc: + report.update(state='stopped',error_type=type(exc).__name__,error=str(exc));raise + finally: + elapsed=time.monotonic()-start + report.update(elapsed_seconds=elapsed,peak_rss_kib=resource.getrusage(resource.RUSAGE_SELF).ru_maxrss) + ledger=json.loads(ledger_path.read_text());ledger['spent_seconds']+=elapsed;ledger['calls'].append({'module':'functional.gain_factorization','seconds':elapsed});write_json(ledger_path,ledger) + write_json(root/'gain-factorization-summary.json',report) + + +if __name__=='__main__':main() diff --git a/functional/model.py b/functional/model.py new file mode 100644 index 0000000..5942217 --- /dev/null +++ b/functional/model.py @@ -0,0 +1,247 @@ +"""Streaming FP32 text-only Qwen3.5 reference, original BF16 weights. + +Mathematical semantics checked against Transformers Qwen3.5, commit +fd3cf0140de03f7619ac95b5cfed05b711ff9e22 (Apache-2.0, Qwen Team/HuggingFace). +No cached decode, padding, images, generation, fused kernels or autograd. +Full-sequence gated delta recurrence replaces the algebraically equivalent +chunk algorithm; both are independently compared in remote tests. +""" +import ctypes +import json +import time +from pathlib import Path +import numpy as np +from pilot.tensors import load_tensors, decode + +PREFIX = 'model.language_model.' + + +def sigmoid(x): + return np.exp(-np.logaddexp(np.float32(0), -x)) + + +def silu(x): + return x * sigmoid(x) + + +def norm(x, weight, eps=1e-6, centered=True): + return (x / np.sqrt(np.mean(x*x, axis=-1, keepdims=True) + eps) + * (1 + weight if centered else weight)).astype(np.float32) + + +def rope(x, theta=10000000., fraction=.25): + # Text-only: all three MRoPE position axes coincide. + dim = int(x.shape[-1] * fraction) + inv = np.float32(1) / np.float32(theta) ** (np.arange(0, dim, 2, dtype=np.float32) / dim) + freq = np.arange(x.shape[1], dtype=np.float32)[:, None] * inv[None, :] + angle = np.concatenate([freq, freq], axis=-1)[None, :, None, :] + a = x[..., :dim] + rotated = np.concatenate([-a[..., dim//2:], a[..., :dim//2]], axis=-1) + return np.concatenate([a*np.cos(angle) + rotated*np.sin(angle), x[..., dim:]], axis=-1) + + +def delta_rule(q, k, v, g, beta): + q = q / np.sqrt(np.sum(q*q, axis=-1, keepdims=True) + 1e-6) + k = k / np.sqrt(np.sum(k*k, axis=-1, keepdims=True) + 1e-6) + q = q / np.float32(np.sqrt(q.shape[-1])) + state = np.zeros((q.shape[0], q.shape[2], q.shape[3], v.shape[3]), dtype=np.float32) + out = np.empty_like(v) + for t in range(q.shape[1]): + state *= np.exp(g[:, t, :, None, None]) + predicted = np.sum(state * k[:, t, :, :, None], axis=-2) + delta = (v[:, t] - predicted) * beta[:, t, :, None] + state += k[:, t, :, :, None] * delta[:, :, None, :] + out[:, t] = np.sum(state * q[:, t, :, :, None], axis=-2) + return out, state + + +def causal_conv(x, kernel): + out = np.zeros_like(x) + width = kernel.shape[-1] + for j in range(width): + lag = width - 1 - j + if lag < x.shape[1]: + out[:, lag:] += x[:, :x.shape[1]-lag] * kernel[:, j] + return silu(out) + + +def attention(q, k, v): + repeats = q.shape[2] // k.shape[2] + k, v = np.repeat(k, repeats, axis=2), np.repeat(v, repeats, axis=2) + scores = np.einsum('bthd,bshd->bhts', q, k) / np.float32(np.sqrt(q.shape[-1])) + mask = np.triu(np.ones(scores.shape[-2:], dtype=bool), 1) + scores = np.where(mask, -np.inf, scores) + exp = np.exp(scores - scores.max(axis=-1, keepdims=True)) + p = exp / exp.sum(axis=-1, keepdims=True) + return np.einsum('bhts,bshd->bthd', p, v), p + + +class StreamingLM: + def __init__(self, weights, config, seconds=7200, enforce_memory=True): + self.tensors = load_tensors(Path(weights)) + self.config = config.get('text_config', config) + self.deadline = time.monotonic() + seconds + self.enforce_memory = enforce_memory + self.eps = self.config['rms_norm_eps'] + self.norm_replacement = None + self.capture_qk = False + self.qk_inputs = {} + self.swap_observation = None + self.swap_pressure_windows = 0 + self.malloc_trim = getattr(ctypes.CDLL(None), 'malloc_trim', None) + + def guard(self): + if time.monotonic() > self.deadline: + raise TimeoutError('CPU computation deadline reached') + if not self.enforce_memory: + return + status = dict(line.split(':', 1) for line in Path('/proc/self/status').read_text().splitlines() if ':' in line) + rss_kib = int(status['VmRSS'].split()[0]) + available = int(next(line.split()[1] for line in Path('/proc/meminfo').read_text().splitlines() if line.startswith('MemAvailable:'))) + now = time.monotonic() + if self.swap_observation is None or now-self.swap_observation[0] >= 5: + counters = dict(line.split() for line in Path('/proc/vmstat').read_text().splitlines()) + count = int(counters['pswpin'])+int(counters['pswpout']) + if self.swap_observation is not None: + elapsed = now-self.swap_observation[0] + rate = (count-self.swap_observation[1])*4096/elapsed + self.swap_pressure_windows = self.swap_pressure_windows+1 if rate > 4*1024**2 else 0 + own_swap_kib = int(status.get('VmSwap', '0 kB').split()[0]) + if self.swap_pressure_windows >= 3 and (available < 384*1024 or own_swap_kib > 128*1024): + raise MemoryError(f'Sustained host swap pressure plus constrained memory: available={available} KiB, process_swap={own_swap_kib} KiB') + self.swap_observation = (now,count) + if rss_kib > 512*1024 or available < 256*1024: + raise MemoryError(f'Soft memory stop: RSS {rss_kib} KiB, available {available} KiB') + + def small(self, name): + t = self.tensors[PREFIX + name] + if t.numel > 1048576: + raise ValueError('Use blockwise linear for large tensors') + x = t.read().reshape(t.shape) + t.evict() + return x + + def linear(self, x, name, rows=256): + t = self.tensors[PREFIX + name + '.weight'] + outdim, indim = t.shape + flat = x.reshape(-1, indim) + result = np.empty((flat.shape[0], outdim), dtype=np.float32) + for start in range(0, outdim, rows): + self.guard() + end = min(start + rows, outdim) + w = t.read(slice(start*indim, end*indim)).reshape(end-start, indim) + result[:, start:end] = flat @ w.T + del w + t.evict() + if self.malloc_trim is not None: + self.malloc_trim(0) # Release only this process's freed temporary heap pages. + return result.reshape(*x.shape[:-1], outdim) + + def embedding(self, ids): + t = self.tensors[PREFIX + 'embed_tokens.weight'] + raw = t.raw.reshape(t.shape) + result = decode(raw[ids], t.dtype) + t.evict() + return result + + def normalization(self, x, name): + if getattr(self, 'norm_replacement', None) and name == self.norm_replacement[0]: + name = self.norm_replacement[1] + return norm(x, self.small(name + '.weight'), self.eps) + + def mlp(self, h, layer): + p = f'layers.{layer}.mlp.' + gate = silu(self.linear(h, p+'gate_proj')) + up = self.linear(h, p+'up_proj') + return self.linear(gate*up, p+'down_proj') + + def full_attention(self, h, layer, shared_p=None): + p = f'layers.{layer}.self_attn.' + c = self.config + heads, kvheads, dim = c['num_attention_heads'], c['num_key_value_heads'], c['head_dim'] + # Q and output gate are interleaved per head, not globally split. + mixed = self.linear(h, p+'q_proj').reshape(*h.shape[:-1], heads, 2*dim) + q, gate = mixed[..., :dim], mixed[..., dim:] + k = self.linear(h, p+'k_proj').reshape(*h.shape[:-1], kvheads, dim) + v = self.linear(h, p+'v_proj').reshape(*h.shape[:-1], kvheads, dim) + if getattr(self, 'capture_qk', False): + self.qk_inputs[layer] = (q.copy(), k.copy()) + q = self.normalization(q, p+'q_norm') + k = self.normalization(k, p+'k_norm') + rp = c['rope_parameters'] + q, k = (rope(x, rp['rope_theta'], rp['partial_rotary_factor']) for x in (q,k)) + out, probs = attention(q,k,v) + if shared_p is not None: + out = np.einsum('bhts,bshd->bthd', shared_p, np.repeat(v, heads//kvheads, axis=2)) + out = (out*sigmoid(gate)).reshape(*h.shape[:-1], -1) + return self.linear(out, p+'o_proj'), probs + + def linear_attention(self, h, layer): + p = f'layers.{layer}.linear_attn.' + c = self.config + kh, vh = c['linear_num_key_heads'], c['linear_num_value_heads'] + kd, vd = c['linear_key_head_dim'], c['linear_value_head_dim'] + qkv = self.linear(h, p+'in_proj_qkv') + qkv = causal_conv(qkv, self.small(p+'conv1d.weight')[:,0,:]) + q, k, v = np.split(qkv, [kh*kd, 2*kh*kd], axis=-1) + q = q.reshape(*h.shape[:-1], kh, kd) + k = k.reshape(*h.shape[:-1], kh, kd) + v = v.reshape(*h.shape[:-1], vh, vd) + if vh != kh: + q, k = np.repeat(q, vh//kh, axis=2), np.repeat(k, vh//kh, axis=2) + z = self.linear(h, p+'in_proj_z').reshape(*h.shape[:-1], vh, vd) + beta = sigmoid(self.linear(h, p+'in_proj_b')) + a = self.linear(h, p+'in_proj_a') + g = -np.exp(self.small(p+'A_log')) * np.logaddexp(np.float32(0), a+self.small(p+'dt_bias')) + out, _ = delta_rule(q,k,v,g,beta) + out = norm(out, self.small(p+'norm.weight'), self.eps, centered=False)*silu(z) + return self.linear(out.reshape(*h.shape[:-1], -1), p+'out_proj') + + def forward(self, ids, capture=None, replacement=None, progress=None): + h = self.embedding(ids) + saved = {} + capture = set(capture or []) + for layer, kind in enumerate(self.config['layer_types']): + self.guard() + raw = h + n = self.normalization(h, f'layers.{layer}.input_layernorm') + if ('attn', layer) in capture: + saved[f'attn_{layer}'] = n.copy() + donor = replacement[2] if replacement and replacement[:2] == ('attn',layer) else layer + if kind == 'full_attention': + out, _ = self.full_attention(n,donor) + else: + out = self.linear_attention(n,donor) + h = raw + out + n = self.normalization(h, f'layers.{layer}.post_attention_layernorm') + if ('mlp',layer) in capture: + saved[f'mlp_{layer}'] = n.copy() + donor = replacement[2] if replacement and replacement[:2] == ('mlp',layer) else layer + h = h + self.mlp(n,donor) + if not np.isfinite(h).all(): + raise FloatingPointError(f'Nonfinite hidden states at layer {layer}') + if progress: + progress(layer) + return self.normalization(h,'norm'), saved + + def nll(self, h, ids, rows=512): + # Tied output embedding; online logsumexp avoids full vocabulary logits. + x = h[:,:-1].reshape(-1,h.shape[-1]) + targets = ids[:,1:].reshape(-1) + t = self.tensors[PREFIX+'embed_tokens.weight'] + mx = np.full(x.shape[0],-np.inf,dtype=np.float64) + total = np.zeros(x.shape[0],dtype=np.float64) + correct = np.empty(x.shape[0],dtype=np.float64) + for start in range(0,t.shape[0],rows): + self.guard() + end=min(start+rows,t.shape[0]) + w=t.read(slice(start*t.shape[1],end*t.shape[1])).reshape(end-start,t.shape[1]) + scores=x@w.T + m=np.maximum(mx,scores.max(axis=-1)) + total=total*np.exp(mx-m)+np.exp(scores-m[:,None]).sum(axis=-1) + mx=m + which=(targets>=start)&(targets