262 lines
16 KiB
Python
262 lines
16 KiB
Python
"""Scientific figures and a compact result report from saved data only."""
|
|
import argparse
|
|
import csv
|
|
import json
|
|
import pathlib
|
|
|
|
import matplotlib
|
|
matplotlib.use('Agg')
|
|
import matplotlib.pyplot as plt
|
|
from matplotlib.patches import Patch, Rectangle
|
|
import numpy as np
|
|
|
|
|
|
def read_csv(path):
|
|
with path.open() as f:
|
|
return list(csv.DictReader(f))
|
|
|
|
|
|
def write_csv(path, rows):
|
|
with path.open('w', newline='') as f:
|
|
if rows:
|
|
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
|
|
|
|
def main():
|
|
p = argparse.ArgumentParser()
|
|
p.add_argument('--root', type=pathlib.Path, required=True)
|
|
args = p.parse_args()
|
|
root = args.root
|
|
out = root / 'analysis'
|
|
figures = root / 'figures'
|
|
figures.mkdir(exist_ok=True)
|
|
plt.rcParams.update({'font.size': 10, 'axes.spines.top': False, 'axes.spines.right': False,
|
|
'pdf.fonttype': 42, 'svg.fonttype': 'none', 'figure.dpi': 120})
|
|
def save(fig, name):
|
|
fig.tight_layout()
|
|
for ext in ('png', 'pdf', 'svg'):
|
|
fig.savefig(figures / f'{name}.{ext}', dpi=200, bbox_inches='tight')
|
|
plt.close(fig)
|
|
|
|
curves = read_csv(out / 'compression_curve.csv')
|
|
summary = json.loads((out / 'summary.json').read_text())
|
|
dimensional = {}
|
|
for row in read_csv(out / 'inventory.csv'):
|
|
ndim = int(row['ndim'])
|
|
item = dimensional.setdefault(ndim, {'ndim': ndim, 'tensor_count': 0, 'weight_bytes': 0})
|
|
item['tensor_count'] += 1
|
|
item['weight_bytes'] += int(row['bytes'])
|
|
dimensional_rows = [dimensional[key] for key in sorted(dimensional)]
|
|
for item in dimensional_rows:
|
|
item['model_byte_fraction'] = item['weight_bytes'] / summary['model_weight_bytes']
|
|
with (out / 'dimensional_bytes.csv').open('w', newline='') as f:
|
|
writer = csv.DictWriter(f, fieldnames=list(dimensional_rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(dimensional_rows)
|
|
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
|
for scale in sorted({r['scale'] for r in curves}):
|
|
rows = [r for r in curves if r['scale'] == scale]
|
|
x = [100 * float(r['threshold']) for r in rows]
|
|
axes[0].plot(x, [100 * float(r['net_saved_model_fraction']) for r in rows], 'o-', label=scale)
|
|
axes[1].plot(x, [int(r['shared_groups']) for r in rows], 'o-', label=scale)
|
|
axes[0].set(xlabel='Maximum relative weight error (%)', ylabel='Verified net savings / model bytes (%)')
|
|
axes[1].set(xlabel='Maximum relative weight error (%)', ylabel='Non-singleton sharing groups')
|
|
axes[0].legend(fontsize=8, loc='upper left')
|
|
fig.suptitle('Conservative verified sharing; block scales are separate alternatives')
|
|
save(fig, 'compression_frontier')
|
|
|
|
pair_sets = {mode: read_csv(out / f'{mode}_pairs.csv') for mode in ('tensor', 'block')}
|
|
role_rows = []
|
|
for mode, pairs in pair_sets.items():
|
|
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
|
|
buckets = {}
|
|
for row in pairs:
|
|
obj = mode_objects[int(row['a'])]
|
|
key = (obj['role'], obj['scale'], tuple(obj['shape']), obj['dtype'])
|
|
buckets.setdefault(key, []).append(row)
|
|
for (role_name, scale_name, shape, dtype), rows in sorted(buckets.items(), key=lambda item: str(item[0])):
|
|
errors = [float(r['error_member_to_rep']) for r in rows]
|
|
cosines = [float(r['cosine']) for r in rows if r['cosine']]
|
|
role_rows.append({'mode': mode, 'role': role_name, 'scale': scale_name,
|
|
'shape': str(shape), 'ndim': len(shape), 'dtype': dtype,
|
|
'verified_pairs': len(rows), 'minimum_direct_error': min(errors),
|
|
'median_direct_error': float(np.median(errors)),
|
|
'maximum_cosine': max(cosines) if cosines else '',
|
|
'minimum_scaled_error': min(float(r['scaled_error']) for r in rows),
|
|
'pairs_under_5pct_direct_error': sum(e <= .05 for e in errors),
|
|
'pairs_under_20pct_direct_error': sum(e <= .20 for e in errors)})
|
|
with (out / 'role_summary.csv').open('w', newline='') as f:
|
|
if role_rows:
|
|
writer = csv.DictWriter(f, fieldnames=list(role_rows[0]))
|
|
writer.writeheader()
|
|
writer.writerows(role_rows)
|
|
fig, axes = plt.subplots(2, 2, figsize=(11, 7))
|
|
for row, (mode, pairs) in enumerate(pair_sets.items()):
|
|
for scale in sorted({r['scale_name'] for r in pairs}):
|
|
group = [r for r in pairs if r['scale_name'] == scale]
|
|
axes[row, 0].hist([float(r['cosine']) for r in group if r['cosine']], bins=np.linspace(-1, 1, 46), histtype='step', label=scale)
|
|
errors = [float(r['error_member_to_rep']) for r in group]
|
|
axes[row, 1].hist(errors, bins=np.linspace(0, max(1e-6, max(errors, default=0.) * 1.01), 46), histtype='step', label=scale)
|
|
axes[row, 0].set(title=f'{mode}: exact cosine of screened candidates', xlabel='Cosine', ylabel='Candidate pairs')
|
|
axes[row, 1].set(title=f'{mode}: exact direct replacement error', xlabel='Relative Frobenius error', ylabel='Candidate pairs')
|
|
axes[row, 0].legend(fontsize=7, loc='upper right')
|
|
save(fig, 'candidate_distributions')
|
|
|
|
pairs = pair_sets['tensor']
|
|
roles = sorted({r['role'] for r in pairs})
|
|
# Illustrate one repeated 2D projection; this is not a byte-weighted model summary.
|
|
objects = json.loads((out / 'tensor_objects.json').read_text())
|
|
candidates = [role for role in roles if any(o['role'] == role and len(o['shape']) == 2 for o in objects)]
|
|
role = max(candidates or roles, key=lambda k: sum(r['role'] == k for r in pairs))
|
|
chosen = [(i, o) for i, o in enumerate(objects) if o['role'] == role]
|
|
chosen.sort(key=lambda item: item[1]['layer'])
|
|
indices = {i: j for j, (i, _) in enumerate(chosen)}
|
|
matrix = np.full((len(chosen), len(chosen)), np.nan)
|
|
np.fill_diagonal(matrix, 1.)
|
|
for r in pairs:
|
|
a, b = int(r['a']), int(r['b'])
|
|
if a in indices and b in indices and r['cosine']:
|
|
matrix[indices[a], indices[b]] = matrix[indices[b], indices[a]] = float(r['cosine'])
|
|
fig, ax = plt.subplots(figsize=(7, 6))
|
|
cmap = plt.get_cmap('coolwarm').with_extremes(bad='white')
|
|
im = ax.imshow(matrix, cmap=cmap, vmin=-1, vmax=1)
|
|
for y, x in zip(*np.where(np.isnan(matrix))):
|
|
ax.add_patch(Rectangle((x - .5, y - .5), 1, 1, facecolor='none',
|
|
edgecolor='#999999', hatch='///', linewidth=0))
|
|
labels = [o['layer'] for _, o in chosen]
|
|
ticks = np.arange(0, len(labels), max(1, len(labels) // 10))
|
|
ax.set_xticks(ticks, [labels[i] for i in ticks])
|
|
ax.set_yticks(ticks, [labels[i] for i in ticks])
|
|
ax.set(xlabel='Layer', ylabel='Layer', title=role.removeprefix('model.') + '\nHatching = unverified; diagonal is identity')
|
|
ax.legend(handles=[Patch(facecolor='white', edgecolor='#999999', hatch='///', label='Unverified pair')],
|
|
loc='upper center', bbox_to_anchor=(.5, -.12), fontsize=8, frameon=False)
|
|
fig.colorbar(im, ax=ax, label='Exact cosine')
|
|
save(fig, 'layer_heatmap')
|
|
heatmap_rows = [{'a_layer': oa['layer'], 'b_layer': ob['layer'], 'role': role,
|
|
'cosine': float(matrix[i, j]) if not np.isnan(matrix[i, j]) else '',
|
|
'verified': bool(i != j and not np.isnan(matrix[i, j])), 'identity': i == j}
|
|
for i, (_, oa) in enumerate(chosen) for j, (_, ob) in enumerate(chosen)]
|
|
write_csv(out / 'heatmap_pairs.csv', heatmap_rows)
|
|
|
|
distance_rows = []
|
|
fig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)
|
|
for ax, mode in zip(axes, ('tensor', 'block')):
|
|
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
|
|
buckets = {}
|
|
for row in pair_sets[mode]:
|
|
if not row['cosine']:
|
|
continue
|
|
obj = mode_objects[int(row['a'])]
|
|
stratum = f"{len(obj['shape'])}D" if mode == 'tensor' else obj['scale']
|
|
buckets.setdefault(stratum, {}).setdefault(int(row['layer_distance']), []).append(float(row['cosine']))
|
|
for stratum, by_distance in sorted(buckets.items()):
|
|
distances = sorted(by_distance)
|
|
medians = []
|
|
for distance in distances:
|
|
values = by_distance[distance]
|
|
median = float(np.median(values))
|
|
medians.append(median)
|
|
distance_rows.append({'mode': mode, 'stratum': stratum, 'layer_distance': distance,
|
|
'pairs': len(values), 'median_cosine': median,
|
|
'q25': float(np.quantile(values, .25)), 'q75': float(np.quantile(values, .75))})
|
|
ax.plot(distances, medians, 'o-', markersize=3, label=stratum)
|
|
ax.set(xlabel='Absolute layer distance', ylabel='Median exact cosine',
|
|
title=f'{mode}: dimension / scale strata', ylim=(-.1, 1.05))
|
|
ax.legend(fontsize=8, loc='upper center', bbox_to_anchor=(.5, -.23), ncol=2, frameon=False)
|
|
fig.suptitle('Screened candidates: module mixture and selection bias still apply')
|
|
write_csv(out / 'layer_distance_stratified.csv', distance_rows)
|
|
save(fig, 'layer_distance')
|
|
|
|
null_rows = []
|
|
fig, axes = plt.subplots(3, 2, figsize=(12, 10))
|
|
bins = np.linspace(-1, 1, 26)
|
|
for col, mode in enumerate(('tensor', 'block')):
|
|
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
|
|
nulls = json.loads((out / f'{mode}_nulls.json').read_text())
|
|
real_random = json.loads((out / f'{mode}_random_pairs.json').read_text())
|
|
cohorts = [('shuffled', [r for r in nulls if r['method'] == 'shuffled']),
|
|
('gaussian_norm_matched', [r for r in nulls if r['method'] == 'gaussian_norm_matched']),
|
|
('random_actual_same_role', real_random)]
|
|
for row, ndim in enumerate((1, 2, 3)):
|
|
ax = axes[row, col]
|
|
any_values = False
|
|
for method, records in cohorts:
|
|
values = np.asarray([r['cosine'] for r in records if r['cosine'] is not None
|
|
and len(mode_objects[r['a']]['shape']) == ndim])
|
|
if not len(values):
|
|
continue
|
|
any_values = True
|
|
values = np.clip(values, -1, 1)
|
|
ax.hist(values, bins=bins, density=True, histtype='step', label=f'{method} (n={len(values)})')
|
|
counts, _ = np.histogram(values, bins=bins)
|
|
for i, count in enumerate(counts):
|
|
null_rows.append({'mode': mode, 'ndim': ndim, 'method': method, 'samples': len(values),
|
|
'bin_left': float(bins[i]), 'bin_right': float(bins[i + 1]),
|
|
'count': int(count), 'density': float(count / (len(values) * (bins[i + 1] - bins[i])))})
|
|
ax.set(title=f'{mode}: {ndim}D sampled controls', xlabel='Sampled cosine', ylabel='Density')
|
|
if any_values:
|
|
ax.legend(fontsize=7, loc='upper right')
|
|
else:
|
|
ax.text(.5, .5, 'No eligible samples', transform=ax.transAxes, ha='center')
|
|
fig.suptitle('Dimension-stratified controls: density normalized; role mixtures may differ')
|
|
write_csv(out / 'null_histograms.csv', null_rows)
|
|
save(fig, 'null_controls')
|
|
|
|
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
|
structure = json.loads((out / 'block_structure.json').read_text())
|
|
for ax, ndim in zip(axes, (2, 3)):
|
|
probes = [m for pair in structure for m in pair['modes'] if len(m['probe_shape']) == ndim and not m.get('trivial_mode_skipped')]
|
|
for axis in sorted({m['axis'] for m in probes}):
|
|
selected = [m for m in probes if m['axis'] == axis]
|
|
ax.scatter([m['normalized_spectrum_distance'] for m in selected],
|
|
[m['left_subspace_overlap'] for m in selected], s=18, alpha=.65, label=f'mode {axis}')
|
|
ax.set(title=f'{ndim}D blocks: bounded mode-unfolding probes', xlabel='Normalized spectrum distance', ylabel='Leading left subspace overlap (k <= 4)')
|
|
if probes:
|
|
ax.legend(fontsize=8, loc='upper right')
|
|
else:
|
|
ax.text(.5, .5, 'No eligible probes', transform=ax.transAxes, ha='center')
|
|
save(fig, 'block_structure')
|
|
|
|
parts = json.loads((out / 'block_partitions.json').read_text())
|
|
lines = ['# Qwen3.5-2B parameter similarity pilot', '',
|
|
f"- Revision: `{summary['revision']}`", f"- Stored parameters: {summary['parameter_count']:,}",
|
|
f"- Weight bytes: {summary['model_weight_bytes']:,}",
|
|
f"- Tensor dimensionalities: {summary['ndim_counts']}",
|
|
f"- Analysis and grouping/export wall time: {summary['elapsed_seconds']:.1f} seconds",
|
|
f"- Pair-comparison cutoff: {summary['analysis_budget_seconds']} seconds; final grouping/export is additional and remains under the supervisor hard limit.",
|
|
f"- Peak analysis RSS: {summary['peak_rss_kib'] / 1024:.1f} MiB", '',
|
|
'## Verified sharing estimates', '',
|
|
'| Representation | Weight-error limit | Verified sharing groups | Conservative total groups* | Largest group | Net model savings |',
|
|
'|---|---:|---:|---:|---:|---:|---:|']
|
|
for r in curves:
|
|
lines.append(f"| {r['scale']} | {float(r['threshold']):.0%} | {r['shared_groups']} | {r['conservative_partition_groups']} | {r['max_group_size']} | {float(r['net_saved_model_fraction']):.6%} |")
|
|
lines += ['', '*Conservative total groups counts unexamined full blocks as singletons; boundary regions and ineligible tensors stay unchanged and are outside that block count.',
|
|
'Savings count only verified replacement edges. Unsampled blocks remain unchanged. Different block scales must not be summed.',
|
|
'Theoretical encoding: one stored representative per group, 16 bytes per object mapping and 64 bytes per shared-group descriptor; identity fallback if larger.',
|
|
'Exact grouping uses direct replacement only. Scaled and spectral results are diagnostic.', '',
|
|
'## Stratified diagnostics', '',
|
|
'See `analysis/role_summary.csv` for same-role, same-shape statistics, separated by tensor dimensionality and block scale.',
|
|
'See `analysis/dimensional_bytes.csv` for tensor counts versus weight-byte contributions by dimensionality.',
|
|
'Heatmap missing pairs are hatched (`heatmap_pairs.csv`); identity entries are not empirical matches.',
|
|
'Layer-distance medians are split by dimension/scale (`layer_distance_stratified.csv`); role and selection bias remain.',
|
|
'Controls are split by dimension and density-normalized, with sample counts (`null_histograms.csv`); these are not significance tests.',
|
|
'These are screened-candidate statistics, not estimates of all possible pairs. Small normalization vectors must not be conflated with large matrices.', '',
|
|
'## Coverage', '']
|
|
for mode in ('tensor', 'block'):
|
|
coverage = json.loads((out / f'{mode}_coverage.json').read_text())
|
|
lines.append(f'- {mode}: `{json.dumps(coverage)}`')
|
|
lines += [f'- Complete-block partition totals (per alternative): `{json.dumps(parts)}`', '',
|
|
'## Interpretation limits', '', *['- ' + s for s in summary['limitations']],
|
|
'- Candidate histograms are selection-biased and do not represent the distribution of all tensor pairs.',
|
|
'- 1D normalization weights may cluster strongly but carry few bytes; do not infer material compression from group counts.',
|
|
'- No task accuracy, perplexity, inference speed or runtime-memory improvement has been measured.',
|
|
'- Stored parameter counts may omit tied aliases; no duplicate-weight equality search is performed.', '',
|
|
'## Figures', '', *[f'' for name in ['compression_frontier', 'candidate_distributions', 'layer_heatmap', 'layer_distance', 'null_controls', 'block_structure']]]
|
|
(root / 'REPORT.md').write_text('\n'.join(lines) + '\n')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|