157 lines
9.3 KiB
Python
157 lines
9.3 KiB
Python
"""Scientific figures and a compact result report from saved data only."""
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import pathlib
|
|
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
|
|
|
|
def read_csv(path):
|
|
with path.open() as f:
|
|
return list(csv.DictReader(f))
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--root', type=pathlib.Path, required=True)
|
|
args = p.parse_args()
|
|
root = args.root
|
|
out = root / 'analysis'
|
|
figures = root / 'figures'
|
|
figures.mkdir(exist_ok=True)
|
|
plt.rcParams.update({'font.size': 10, 'axes.spines.top': False, 'axes.spines.right': False,
|
|
'pdf.fonttype': 42, 'svg.fonttype': 'none', 'figure.dpi': 120})
|
|
def save(fig, name):
|
|
fig.tight_layout()
|
|
for ext in ('png', 'pdf', 'svg'):
|
|
fig.savefig(figures / f'{name}.{ext}', dpi=200, bbox_inches='tight')
|
|
plt.close(fig)
|
|
|
|
curves = read_csv(out / 'compression_curve.csv')
|
|
summary = json.loads((out / 'summary.json').read_text())
|
|
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
|
for scale in sorted({r['scale'] for r in curves}):
|
|
rows = [r for r in curves if r['scale'] == scale]
|
|
x = [100 * float(r['threshold']) for r in rows]
|
|
axes[0].plot(x, [100 * float(r['net_saved_model_fraction']) for r in rows], 'o-', label=scale)
|
|
axes[1].plot(x, [int(r['shared_groups']) for r in rows], 'o-', label=scale)
|
|
axes[0].set(xlabel='Maximum relative weight error (%)', ylabel='Verified net savings / model bytes (%)')
|
|
axes[1].set(xlabel='Maximum relative weight error (%)', ylabel='Non-singleton sharing groups')
|
|
axes[0].legend(fontsize=8)
|
|
fig.suptitle('Conservative verified sharing; block scales are separate alternatives')
|
|
save(fig, 'compression_frontier')
|
|
|
|
pair_sets = {mode: read_csv(out / f'{mode}_pairs.csv') for mode in ('tensor', 'block')}
|
|
fig, axes = plt.subplots(2, 2, figsize=(11, 7))
|
|
for row, (mode, pairs) in enumerate(pair_sets.items()):
|
|
for scale in sorted({r['scale_name'] for r in pairs}):
|
|
group = [r for r in pairs if r['scale_name'] == scale]
|
|
axes[row, 0].hist([float(r['cosine']) for r in group if r['cosine']], bins=np.linspace(-1, 1, 46), histtype='step', label=scale)
|
|
errors = [float(r['error_member_to_rep']) for r in group]
|
|
axes[row, 1].hist(errors, bins=np.linspace(0, max(1e-6, max(errors, default=0.) * 1.01), 46), histtype='step', label=scale)
|
|
axes[row, 0].set(title=f'{mode}: exact cosine of screened candidates', xlabel='Cosine', ylabel='Candidate pairs')
|
|
axes[row, 1].set(title=f'{mode}: exact direct replacement error', xlabel='Relative Frobenius error', ylabel='Candidate pairs')
|
|
axes[row, 0].legend(fontsize=7)
|
|
save(fig, 'candidate_distributions')
|
|
|
|
pairs = pair_sets['tensor']
|
|
roles = sorted({r['role'] for r in pairs})
|
|
# Use a large 2D repeated projection, not tiny norms, for the primary heatmap.
|
|
objects = json.loads((out / 'tensor_objects.json').read_text())
|
|
candidates = [role for role in roles if any(o['role'] == role and len(o['shape']) == 2 for o in objects)]
|
|
role = max(candidates or roles, key=lambda k: sum(r['role'] == k for r in pairs))
|
|
chosen = [(i, o) for i, o in enumerate(objects) if o['role'] == role]
|
|
chosen.sort(key=lambda item: item[1]['layer'])
|
|
indices = {i: j for j, (i, _) in enumerate(chosen)}
|
|
matrix = np.full((len(chosen), len(chosen)), np.nan)
|
|
np.fill_diagonal(matrix, 1.)
|
|
for r in pairs:
|
|
a, b = int(r['a']), int(r['b'])
|
|
if a in indices and b in indices and r['cosine']:
|
|
matrix[indices[a], indices[b]] = matrix[indices[b], indices[a]] = float(r['cosine'])
|
|
fig, ax = plt.subplots(figsize=(7, 6))
|
|
cmap = plt.get_cmap('coolwarm').copy()
|
|
cmap.set_bad('#dddddd')
|
|
im = ax.imshow(matrix, cmap=cmap, vmin=-1, vmax=1)
|
|
labels = [o['layer'] for _, o in chosen]
|
|
ticks = np.arange(0, len(labels), max(1, len(labels) // 10))
|
|
ax.set_xticks(ticks, [labels[i] for i in ticks])
|
|
ax.set_yticks(ticks, [labels[i] for i in ticks])
|
|
ax.set(xlabel='Layer', ylabel='Layer', title=role.replace('model.', '') + '\nGray = unverified; diagonal is identity')
|
|
fig.colorbar(im, ax=ax, label='Exact cosine')
|
|
save(fig, 'layer_heatmap')
|
|
|
|
fig, ax = plt.subplots(figsize=(8, 4))
|
|
for mode in ('tensor', 'block'):
|
|
rows = pair_sets[mode]
|
|
distances = sorted({int(r['layer_distance']) for r in rows})
|
|
median = [np.median([float(r['cosine']) for r in rows if int(r['layer_distance']) == d and r['cosine']]) for d in distances]
|
|
ax.plot(distances, median, 'o-', label=mode)
|
|
ax.set(xlabel='Absolute layer distance', ylabel='Median exact cosine', title='Screened candidates; role mixture and selection bias apply')
|
|
ax.legend()
|
|
save(fig, 'layer_distance')
|
|
|
|
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
|
for ax, mode in zip(axes, ('tensor', 'block')):
|
|
nulls = json.loads((out / f'{mode}_nulls.json').read_text())
|
|
for method in ('shuffled', 'gaussian_norm_matched'):
|
|
values = [r['cosine'] for r in nulls if r['method'] == method and r['cosine'] is not None]
|
|
ax.hist(values, bins=np.linspace(-1, 1, 26), histtype='step', label=method)
|
|
real_random = json.loads((out / f'{mode}_random_pairs.json').read_text())
|
|
ax.hist([r['cosine'] for r in real_random if r['cosine'] is not None], bins=np.linspace(-1, 1, 26), histtype='step', label='random actual same-role pairs')
|
|
ax.set(title=f'{mode}: sampled null controls', xlabel='Sampled cosine', ylabel='Pairs')
|
|
ax.legend(fontsize=8)
|
|
save(fig, 'null_controls')
|
|
|
|
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
|
structure = json.loads((out / 'block_structure.json').read_text())
|
|
for ax, ndim in zip(axes, (2, 3)):
|
|
probes = [m for pair in structure for m in pair['modes'] if len(m['probe_shape']) == ndim and not m.get('trivial_mode_skipped')]
|
|
for axis in sorted({m['axis'] for m in probes}):
|
|
selected = [m for m in probes if m['axis'] == axis]
|
|
ax.scatter([m['normalized_spectrum_distance'] for m in selected],
|
|
[m['left_subspace_overlap'] for m in selected], s=18, alpha=.65, label=f'mode {axis}')
|
|
ax.set(title=f'{ndim}D blocks: bounded mode-unfolding probes', xlabel='Normalized spectrum distance', ylabel='Leading left subspace overlap (k <= 4)')
|
|
if probes:
|
|
ax.legend(fontsize=8)
|
|
else:
|
|
ax.text(.5, .5, 'No eligible probes', transform=ax.transAxes, ha='center')
|
|
save(fig, 'block_structure')
|
|
|
|
parts = json.loads((out / 'block_partitions.json').read_text())
|
|
lines = ['# Qwen3.5-2B parameter similarity pilot', '',
|
|
f"- Revision: `{summary['revision']}`", f"- Stored parameters: {summary['parameter_count']:,}",
|
|
f"- Weight bytes: {summary['model_weight_bytes']:,}",
|
|
f"- Tensor dimensionalities: {summary['ndim_counts']}",
|
|
f"- Analysis seconds: {summary['elapsed_seconds']:.1f} / {summary['analysis_budget_seconds']}",
|
|
f"- Peak analysis RSS: {summary['peak_rss_kib'] / 1024:.1f} MiB", '',
|
|
'## Verified sharing estimates', '',
|
|
'| Representation | Weight-error limit | Verified sharing groups | Conservative total groups* | Largest group | Net model savings |',
|
|
'|---|---:|---:|---:|---:|---:|---:|']
|
|
for r in curves:
|
|
lines.append(f"| {r['scale']} | {float(r['threshold']):.0%} | {r['shared_groups']} | {r['conservative_partition_groups']} | {r['max_group_size']} | {float(r['net_saved_model_fraction']):.6%} |")
|
|
lines += ['', '*Conservative total groups counts unexamined full blocks as singletons; boundary regions and ineligible tensors stay unchanged and are outside that block count.',
|
|
'Savings count only verified replacement edges. Unsampled blocks remain unchanged. Different block scales must not be summed.',
|
|
'Theoretical encoding: one stored representative per group, 16 bytes per object mapping and 64 bytes per shared-group descriptor; identity fallback if larger.',
|
|
'Exact grouping uses direct replacement only. Scaled and spectral results are diagnostic.', '',
|
|
'## Coverage', '']
|
|
for mode in ('tensor', 'block'):
|
|
coverage = json.loads((out / f'{mode}_coverage.json').read_text())
|
|
lines.append(f'- {mode}: `{json.dumps(coverage)}`')
|
|
lines += [f'- Complete-block partition totals (per alternative): `{json.dumps(parts)}`', '',
|
|
'## Interpretation limits', '', *['- ' + s for s in summary['limitations']],
|
|
'- Candidate histograms are selection-biased and do not represent the distribution of all tensor pairs.',
|
|
'- 1D normalization weights may cluster strongly but carry few bytes; do not infer material compression from group counts.',
|
|
'- No task accuracy, perplexity, inference speed or runtime-memory improvement has been measured.',
|
|
'- Stored parameter counts may omit tied aliases; no duplicate-weight equality search is performed.', '',
|
|
'## Figures', '', *[f'' for name in ['compression_frontier', 'candidate_distributions', 'layer_heatmap', 'layer_distance', 'null_controls', 'block_structure']]]
|
|
(root / 'REPORT.md').write_text('\n'.join(lines) + '\n')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|