Zayden-Jung/exp-model-parameter-similarity
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Record bounded Qwen3.5-2B pilot and stratified diagnostics

Browse Source
This commit is contained in:
Codex Bot
2026-11-09 04:07:29 +08:00
parent 6121016a26
commit 727b05eb3f
92 changed files with 776086 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pilot/plot.py
+129 -24
View File
@@ -7,6 +7,7 @@ import pathlib
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.patches import Patch, Rectangle
import numpy as np
@@ -15,6 +16,14 @@ def read_csv(path):
return list(csv.DictReader(f))
def write_csv(path, rows):
with path.open('w', newline='') as f:
if rows:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
def main():
p = argparse.ArgumentParser()
p.add_argument('--root', type=pathlib.Path, required=True)
@@ -33,6 +42,19 @@ def main():
curves = read_csv(out / 'compression_curve.csv')
summary = json.loads((out / 'summary.json').read_text())
dimensional = {}
for row in read_csv(out / 'inventory.csv'):
ndim = int(row['ndim'])
item = dimensional.setdefault(ndim, {'ndim': ndim, 'tensor_count': 0, 'weight_bytes': 0})
item['tensor_count'] += 1
item['weight_bytes'] += int(row['bytes'])
dimensional_rows = [dimensional[key] for key in sorted(dimensional)]
for item in dimensional_rows:
item['model_byte_fraction'] = item['weight_bytes'] / summary['model_weight_bytes']
with (out / 'dimensional_bytes.csv').open('w', newline='') as f:
writer = csv.DictWriter(f, fieldnames=list(dimensional_rows[0]))
writer.writeheader()
writer.writerows(dimensional_rows)
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
for scale in sorted({r['scale'] for r in curves}):
rows = [r for r in curves if r['scale'] == scale]
@@ -41,11 +63,35 @@ def main():
axes[1].plot(x, [int(r['shared_groups']) for r in rows], 'o-', label=scale)
axes[0].set(xlabel='Maximum relative weight error (%)', ylabel='Verified net savings / model bytes (%)')
axes[1].set(xlabel='Maximum relative weight error (%)', ylabel='Non-singleton sharing groups')
axes[0].legend(fontsize=8)
axes[0].legend(fontsize=8, loc='upper left')
fig.suptitle('Conservative verified sharing; block scales are separate alternatives')
save(fig, 'compression_frontier')
pair_sets = {mode: read_csv(out / f'{mode}_pairs.csv') for mode in ('tensor', 'block')}
role_rows = []
for mode, pairs in pair_sets.items():
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
buckets = {}
for row in pairs:
obj = mode_objects[int(row['a'])]
key = (obj['role'], obj['scale'], tuple(obj['shape']), obj['dtype'])
buckets.setdefault(key, []).append(row)
for (role_name, scale_name, shape, dtype), rows in sorted(buckets.items(), key=lambda item: str(item[0])):
errors = [float(r['error_member_to_rep']) for r in rows]
cosines = [float(r['cosine']) for r in rows if r['cosine']]
role_rows.append({'mode': mode, 'role': role_name, 'scale': scale_name,
'shape': str(shape), 'ndim': len(shape), 'dtype': dtype,
'verified_pairs': len(rows), 'minimum_direct_error': min(errors),
'median_direct_error': float(np.median(errors)),
'maximum_cosine': max(cosines) if cosines else '',
'minimum_scaled_error': min(float(r['scaled_error']) for r in rows),
'pairs_under_5pct_direct_error': sum(e <= .05 for e in errors),
'pairs_under_20pct_direct_error': sum(e <= .20 for e in errors)})
with (out / 'role_summary.csv').open('w', newline='') as f:
if role_rows:
writer = csv.DictWriter(f, fieldnames=list(role_rows[0]))
writer.writeheader()
writer.writerows(role_rows)
fig, axes = plt.subplots(2, 2, figsize=(11, 7))
for row, (mode, pairs) in enumerate(pair_sets.items()):
for scale in sorted({r['scale_name'] for r in pairs}):
@@ -55,12 +101,12 @@ def main():
axes[row, 1].hist(errors, bins=np.linspace(0, max(1e-6, max(errors, default=0.) * 1.01), 46), histtype='step', label=scale)
axes[row, 0].set(title=f'{mode}: exact cosine of screened candidates', xlabel='Cosine', ylabel='Candidate pairs')
axes[row, 1].set(title=f'{mode}: exact direct replacement error', xlabel='Relative Frobenius error', ylabel='Candidate pairs')
axes[row, 0].legend(fontsize=7)
axes[row, 0].legend(fontsize=7, loc='upper right')
save(fig, 'candidate_distributions')
pairs = pair_sets['tensor']
roles = sorted({r['role'] for r in pairs})
# Use a large 2D repeated projection, not tiny norms, for the primary heatmap.
# Illustrate one repeated 2D projection; this is not a byte-weighted model summary.
objects = json.loads((out / 'tensor_objects.json').read_text())
candidates = [role for role in roles if any(o['role'] == role and len(o['shape']) == 2 for o in objects)]
role = max(candidates or roles, key=lambda k: sum(r['role'] == k for r in pairs))
@@ -74,37 +120,88 @@ def main():
if a in indices and b in indices and r['cosine']:
matrix[indices[a], indices[b]] = matrix[indices[b], indices[a]] = float(r['cosine'])
fig, ax = plt.subplots(figsize=(7, 6))
cmap = plt.get_cmap('coolwarm').copy()
cmap.set_bad('#dddddd')
cmap = plt.get_cmap('coolwarm').with_extremes(bad='white')
im = ax.imshow(matrix, cmap=cmap, vmin=-1, vmax=1)
for y, x in zip(*np.where(np.isnan(matrix))):
ax.add_patch(Rectangle((x - .5, y - .5), 1, 1, facecolor='none',
edgecolor='#999999', hatch='///', linewidth=0))
labels = [o['layer'] for _, o in chosen]
ticks = np.arange(0, len(labels), max(1, len(labels) // 10))
ax.set_xticks(ticks, [labels[i] for i in ticks])
ax.set_yticks(ticks, [labels[i] for i in ticks])
ax.set(xlabel='Layer', ylabel='Layer', title=role.replace('model.', '') + '\nGray = unverified; diagonal is identity')
ax.set(xlabel='Layer', ylabel='Layer', title=role.removeprefix('model.') + '\nHatching = unverified; diagonal is identity')
ax.legend(handles=[Patch(facecolor='white', edgecolor='#999999', hatch='///', label='Unverified pair')],
loc='upper center', bbox_to_anchor=(.5, -.12), fontsize=8, frameon=False)
fig.colorbar(im, ax=ax, label='Exact cosine')
save(fig, 'layer_heatmap')
heatmap_rows = [{'a_layer': oa['layer'], 'b_layer': ob['layer'], 'role': role,
'cosine': float(matrix[i, j]) if not np.isnan(matrix[i, j]) else '',
'verified': bool(i != j and not np.isnan(matrix[i, j])), 'identity': i == j}
for i, (_, oa) in enumerate(chosen) for j, (_, ob) in enumerate(chosen)]
write_csv(out / 'heatmap_pairs.csv', heatmap_rows)
fig, ax = plt.subplots(figsize=(8, 4))
for mode in ('tensor', 'block'):
rows = pair_sets[mode]
distances = sorted({int(r['layer_distance']) for r in rows})
median = [np.median([float(r['cosine']) for r in rows if int(r['layer_distance']) == d and r['cosine']]) for d in distances]
ax.plot(distances, median, 'o-', label=mode)
ax.set(xlabel='Absolute layer distance', ylabel='Median exact cosine', title='Screened candidates; role mixture and selection bias apply')
ax.legend()
distance_rows = []
fig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)
for ax, mode in zip(axes, ('tensor', 'block')):
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
buckets = {}
for row in pair_sets[mode]:
if not row['cosine']:
continue
obj = mode_objects[int(row['a'])]
stratum = f"{len(obj['shape'])}D" if mode == 'tensor' else obj['scale']
buckets.setdefault(stratum, {}).setdefault(int(row['layer_distance']), []).append(float(row['cosine']))
for stratum, by_distance in sorted(buckets.items()):
distances = sorted(by_distance)
medians = []
for distance in distances:
values = by_distance[distance]
median = float(np.median(values))
medians.append(median)
distance_rows.append({'mode': mode, 'stratum': stratum, 'layer_distance': distance,
'pairs': len(values), 'median_cosine': median,
'q25': float(np.quantile(values, .25)), 'q75': float(np.quantile(values, .75))})
ax.plot(distances, medians, 'o-', markersize=3, label=stratum)
ax.set(xlabel='Absolute layer distance', ylabel='Median exact cosine',
title=f'{mode}: dimension / scale strata', ylim=(-.1, 1.05))
ax.legend(fontsize=8, loc='upper center', bbox_to_anchor=(.5, -.23), ncol=2, frameon=False)
fig.suptitle('Screened candidates: module mixture and selection bias still apply')
write_csv(out / 'layer_distance_stratified.csv', distance_rows)
save(fig, 'layer_distance')
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
for ax, mode in zip(axes, ('tensor', 'block')):
null_rows = []
fig, axes = plt.subplots(3, 2, figsize=(12, 10))
bins = np.linspace(-1, 1, 26)
for col, mode in enumerate(('tensor', 'block')):
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
nulls = json.loads((out / f'{mode}_nulls.json').read_text())
for method in ('shuffled', 'gaussian_norm_matched'):
values = [r['cosine'] for r in nulls if r['method'] == method and r['cosine'] is not None]
ax.hist(values, bins=np.linspace(-1, 1, 26), histtype='step', label=method)
real_random = json.loads((out / f'{mode}_random_pairs.json').read_text())
ax.hist([r['cosine'] for r in real_random if r['cosine'] is not None], bins=np.linspace(-1, 1, 26), histtype='step', label='random actual same-role pairs')
ax.set(title=f'{mode}: sampled null controls', xlabel='Sampled cosine', ylabel='Pairs')
ax.legend(fontsize=8)
cohorts = [('shuffled', [r for r in nulls if r['method'] == 'shuffled']),
('gaussian_norm_matched', [r for r in nulls if r['method'] == 'gaussian_norm_matched']),
('random_actual_same_role', real_random)]
for row, ndim in enumerate((1, 2, 3)):
ax = axes[row, col]
any_values = False
for method, records in cohorts:
values = np.asarray([r['cosine'] for r in records if r['cosine'] is not None
and len(mode_objects[r['a']]['shape']) == ndim])
if not len(values):
continue
any_values = True
values = np.clip(values, -1, 1)
ax.hist(values, bins=bins, density=True, histtype='step', label=f'{method} (n={len(values)})')
counts, _ = np.histogram(values, bins=bins)
for i, count in enumerate(counts):
null_rows.append({'mode': mode, 'ndim': ndim, 'method': method, 'samples': len(values),
'bin_left': float(bins[i]), 'bin_right': float(bins[i + 1]),
'count': int(count), 'density': float(count / (len(values) * (bins[i + 1] - bins[i])))})
ax.set(title=f'{mode}: {ndim}D sampled controls', xlabel='Sampled cosine', ylabel='Density')
if any_values:
ax.legend(fontsize=7, loc='upper right')
else:
ax.text(.5, .5, 'No eligible samples', transform=ax.transAxes, ha='center')
fig.suptitle('Dimension-stratified controls: density normalized; role mixtures may differ')
write_csv(out / 'null_histograms.csv', null_rows)
save(fig, 'null_controls')
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
@@ -117,7 +214,7 @@ def main():
[m['left_subspace_overlap'] for m in selected], s=18, alpha=.65, label=f'mode {axis}')
ax.set(title=f'{ndim}D blocks: bounded mode-unfolding probes', xlabel='Normalized spectrum distance', ylabel='Leading left subspace overlap (k <= 4)')
if probes:
ax.legend(fontsize=8)
ax.legend(fontsize=8, loc='upper right')
else:
ax.text(.5, .5, 'No eligible probes', transform=ax.transAxes, ha='center')
save(fig, 'block_structure')
@@ -127,7 +224,8 @@ def main():
f"- Revision: `{summary['revision']}`", f"- Stored parameters: {summary['parameter_count']:,}",
f"- Weight bytes: {summary['model_weight_bytes']:,}",
f"- Tensor dimensionalities: {summary['ndim_counts']}",
f"- Analysis seconds: {summary['elapsed_seconds']:.1f} / {summary['analysis_budget_seconds']}",
f"- Analysis and grouping/export wall time: {summary['elapsed_seconds']:.1f} seconds",
f"- Pair-comparison cutoff: {summary['analysis_budget_seconds']} seconds; final grouping/export is additional and remains under the supervisor hard limit.",
f"- Peak analysis RSS: {summary['peak_rss_kib'] / 1024:.1f} MiB", '',
'## Verified sharing estimates', '',
'| Representation | Weight-error limit | Verified sharing groups | Conservative total groups* | Largest group | Net model savings |',
@@ -138,6 +236,13 @@ def main():
'Savings count only verified replacement edges. Unsampled blocks remain unchanged. Different block scales must not be summed.',
'Theoretical encoding: one stored representative per group, 16 bytes per object mapping and 64 bytes per shared-group descriptor; identity fallback if larger.',
'Exact grouping uses direct replacement only. Scaled and spectral results are diagnostic.', '',
'## Stratified diagnostics', '',
'See `analysis/role_summary.csv` for same-role, same-shape statistics, separated by tensor dimensionality and block scale.',
'See `analysis/dimensional_bytes.csv` for tensor counts versus weight-byte contributions by dimensionality.',
'Heatmap missing pairs are hatched (`heatmap_pairs.csv`); identity entries are not empirical matches.',
'Layer-distance medians are split by dimension/scale (`layer_distance_stratified.csv`); role and selection bias remain.',
'Controls are split by dimension and density-normalized, with sample counts (`null_histograms.csv`); these are not significance tests.',
'These are screened-candidate statistics, not estimates of all possible pairs. Small normalization vectors must not be conflated with large matrices.', '',
'## Coverage', '']
for mode in ('tensor', 'block'):
coverage = json.loads((out / f'{mode}_coverage.json').read_text())