Record bounded Qwen3.5-2B pilot and stratified diagnostics
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
"""Audit saved numerical data and freeze provenance before archive/cleanup."""
|
||||
import argparse
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import pathlib
|
||||
import platform
|
||||
import re
|
||||
import shutil
|
||||
import time
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--root', required=True, type=pathlib.Path)
|
||||
p.add_argument('--analysis-code-commit', help='Commit actually used for the weight analysis; omit if unknown')
|
||||
args = p.parse_args()
|
||||
root = args.root.resolve()
|
||||
if not __debug__:
|
||||
raise RuntimeError('Validation requires Python assertions enabled')
|
||||
checked = {}
|
||||
for mode in ('tensor', 'block'):
|
||||
with (root / 'analysis' / f'{mode}_pairs.csv').open() as f:
|
||||
rows = list(csv.DictReader(f))
|
||||
coverage = json.loads((root / 'analysis' / f'{mode}_coverage.json').read_text())
|
||||
if len(rows) != coverage['pairs_verified']:
|
||||
raise ValueError('Coverage count mismatch')
|
||||
for row in rows:
|
||||
for key in ('cosine', 'pearson'):
|
||||
if row[key] and (not math.isfinite(float(row[key])) or abs(float(row[key])) > 1 + 1e-7):
|
||||
raise ValueError(f'Invalid correlation: {mode}/{key}')
|
||||
for key in ('error_member_to_rep', 'error_rep_to_member', 'symmetric_error', 'scaled_error',
|
||||
'sampled_error_q99_over_member_rms', 'screen_vs_exact_min_error_absolute_difference'):
|
||||
value = float(row[key])
|
||||
if not math.isfinite(value) or value < 0:
|
||||
raise ValueError(f'Invalid error: {mode}/{key}')
|
||||
if not math.isfinite(float(row['scale'])):
|
||||
raise ValueError('Invalid scaling diagnostic')
|
||||
checked[mode] = {'rows': len(rows), 'finite_metrics': True, 'coverage_matches': True}
|
||||
test_log = (root / 'tests.log').read_text()
|
||||
match = re.search(r'Ran (\d+) tests in ([\d.]+)s', test_log)
|
||||
if not match or '\nOK\n' not in test_log:
|
||||
raise ValueError('Final tests did not pass')
|
||||
qc = {'passed': True, 'checked_unix': time.time(), 'numerical_tables': checked,
|
||||
'final_tests': int(match[1]), 'final_test_seconds': float(match[2]),
|
||||
'earlier_parallel_test_timeout_seconds': 300 if (root / 'tests-during-io-timeout.txt').exists() else None}
|
||||
(root / 'numerical-qc.json').write_text(json.dumps(qc, indent=2))
|
||||
for src, dest in [('tests.log', 'test-results.txt'), ('analysis.log', 'analysis-run.txt'),
|
||||
('plot.log', 'plot-run.txt'), ('download-ranged.log', 'download-progress.txt'),
|
||||
('supervisor.log', 'supervisor-run.txt')]:
|
||||
shutil.copy2(root / src, root / dest)
|
||||
mem = {}
|
||||
for line in pathlib.Path('/proc/meminfo').read_text().splitlines():
|
||||
key, value = line.split(':', 1)
|
||||
if key in ('MemTotal', 'MemAvailable', 'SwapTotal', 'SwapFree'):
|
||||
mem[key + '_KiB'] = int(value.split()[0])
|
||||
environment = {'recorded_unix': time.time(), 'python': platform.python_version(),
|
||||
'platform': platform.platform(), 'machine': platform.machine(),
|
||||
'logical_cpus': os.cpu_count(), 'memory_snapshot': mem,
|
||||
'blas_threads': 1, 'assertions_enabled': __debug__,
|
||||
'analysis_io_priority': 'best-effort 7, applied during block verification',
|
||||
'analysis_code_commit': args.analysis_code_commit,
|
||||
'note': 'Shared host; see resource-observation.txt. Memory snapshot is post-analysis.'}
|
||||
(root / 'execution-environment.json').write_text(json.dumps(environment, indent=2))
|
||||
project = pathlib.Path(__file__).resolve().parents[1]
|
||||
files = sorted((project / 'pilot').glob('*.py')) + [project / 'requirements.txt', project / 'tests/test_pilot.py']
|
||||
hashes = [f'{hashlib.sha256(file.read_bytes()).hexdigest()} {file.relative_to(project)}' for file in files]
|
||||
(root / 'code_checksums.txt').write_text('\n'.join(hashes) + '\n')
|
||||
print(json.dumps(qc))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+129
-24
@@ -7,6 +7,7 @@ import pathlib
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Patch, Rectangle
|
||||
import numpy as np
|
||||
|
||||
|
||||
@@ -15,6 +16,14 @@ def read_csv(path):
|
||||
return list(csv.DictReader(f))
|
||||
|
||||
|
||||
def write_csv(path, rows):
|
||||
with path.open('w', newline='') as f:
|
||||
if rows:
|
||||
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--root', type=pathlib.Path, required=True)
|
||||
@@ -33,6 +42,19 @@ def main():
|
||||
|
||||
curves = read_csv(out / 'compression_curve.csv')
|
||||
summary = json.loads((out / 'summary.json').read_text())
|
||||
dimensional = {}
|
||||
for row in read_csv(out / 'inventory.csv'):
|
||||
ndim = int(row['ndim'])
|
||||
item = dimensional.setdefault(ndim, {'ndim': ndim, 'tensor_count': 0, 'weight_bytes': 0})
|
||||
item['tensor_count'] += 1
|
||||
item['weight_bytes'] += int(row['bytes'])
|
||||
dimensional_rows = [dimensional[key] for key in sorted(dimensional)]
|
||||
for item in dimensional_rows:
|
||||
item['model_byte_fraction'] = item['weight_bytes'] / summary['model_weight_bytes']
|
||||
with (out / 'dimensional_bytes.csv').open('w', newline='') as f:
|
||||
writer = csv.DictWriter(f, fieldnames=list(dimensional_rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(dimensional_rows)
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
||||
for scale in sorted({r['scale'] for r in curves}):
|
||||
rows = [r for r in curves if r['scale'] == scale]
|
||||
@@ -41,11 +63,35 @@ def main():
|
||||
axes[1].plot(x, [int(r['shared_groups']) for r in rows], 'o-', label=scale)
|
||||
axes[0].set(xlabel='Maximum relative weight error (%)', ylabel='Verified net savings / model bytes (%)')
|
||||
axes[1].set(xlabel='Maximum relative weight error (%)', ylabel='Non-singleton sharing groups')
|
||||
axes[0].legend(fontsize=8)
|
||||
axes[0].legend(fontsize=8, loc='upper left')
|
||||
fig.suptitle('Conservative verified sharing; block scales are separate alternatives')
|
||||
save(fig, 'compression_frontier')
|
||||
|
||||
pair_sets = {mode: read_csv(out / f'{mode}_pairs.csv') for mode in ('tensor', 'block')}
|
||||
role_rows = []
|
||||
for mode, pairs in pair_sets.items():
|
||||
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
|
||||
buckets = {}
|
||||
for row in pairs:
|
||||
obj = mode_objects[int(row['a'])]
|
||||
key = (obj['role'], obj['scale'], tuple(obj['shape']), obj['dtype'])
|
||||
buckets.setdefault(key, []).append(row)
|
||||
for (role_name, scale_name, shape, dtype), rows in sorted(buckets.items(), key=lambda item: str(item[0])):
|
||||
errors = [float(r['error_member_to_rep']) for r in rows]
|
||||
cosines = [float(r['cosine']) for r in rows if r['cosine']]
|
||||
role_rows.append({'mode': mode, 'role': role_name, 'scale': scale_name,
|
||||
'shape': str(shape), 'ndim': len(shape), 'dtype': dtype,
|
||||
'verified_pairs': len(rows), 'minimum_direct_error': min(errors),
|
||||
'median_direct_error': float(np.median(errors)),
|
||||
'maximum_cosine': max(cosines) if cosines else '',
|
||||
'minimum_scaled_error': min(float(r['scaled_error']) for r in rows),
|
||||
'pairs_under_5pct_direct_error': sum(e <= .05 for e in errors),
|
||||
'pairs_under_20pct_direct_error': sum(e <= .20 for e in errors)})
|
||||
with (out / 'role_summary.csv').open('w', newline='') as f:
|
||||
if role_rows:
|
||||
writer = csv.DictWriter(f, fieldnames=list(role_rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(role_rows)
|
||||
fig, axes = plt.subplots(2, 2, figsize=(11, 7))
|
||||
for row, (mode, pairs) in enumerate(pair_sets.items()):
|
||||
for scale in sorted({r['scale_name'] for r in pairs}):
|
||||
@@ -55,12 +101,12 @@ def main():
|
||||
axes[row, 1].hist(errors, bins=np.linspace(0, max(1e-6, max(errors, default=0.) * 1.01), 46), histtype='step', label=scale)
|
||||
axes[row, 0].set(title=f'{mode}: exact cosine of screened candidates', xlabel='Cosine', ylabel='Candidate pairs')
|
||||
axes[row, 1].set(title=f'{mode}: exact direct replacement error', xlabel='Relative Frobenius error', ylabel='Candidate pairs')
|
||||
axes[row, 0].legend(fontsize=7)
|
||||
axes[row, 0].legend(fontsize=7, loc='upper right')
|
||||
save(fig, 'candidate_distributions')
|
||||
|
||||
pairs = pair_sets['tensor']
|
||||
roles = sorted({r['role'] for r in pairs})
|
||||
# Use a large 2D repeated projection, not tiny norms, for the primary heatmap.
|
||||
# Illustrate one repeated 2D projection; this is not a byte-weighted model summary.
|
||||
objects = json.loads((out / 'tensor_objects.json').read_text())
|
||||
candidates = [role for role in roles if any(o['role'] == role and len(o['shape']) == 2 for o in objects)]
|
||||
role = max(candidates or roles, key=lambda k: sum(r['role'] == k for r in pairs))
|
||||
@@ -74,37 +120,88 @@ def main():
|
||||
if a in indices and b in indices and r['cosine']:
|
||||
matrix[indices[a], indices[b]] = matrix[indices[b], indices[a]] = float(r['cosine'])
|
||||
fig, ax = plt.subplots(figsize=(7, 6))
|
||||
cmap = plt.get_cmap('coolwarm').copy()
|
||||
cmap.set_bad('#dddddd')
|
||||
cmap = plt.get_cmap('coolwarm').with_extremes(bad='white')
|
||||
im = ax.imshow(matrix, cmap=cmap, vmin=-1, vmax=1)
|
||||
for y, x in zip(*np.where(np.isnan(matrix))):
|
||||
ax.add_patch(Rectangle((x - .5, y - .5), 1, 1, facecolor='none',
|
||||
edgecolor='#999999', hatch='///', linewidth=0))
|
||||
labels = [o['layer'] for _, o in chosen]
|
||||
ticks = np.arange(0, len(labels), max(1, len(labels) // 10))
|
||||
ax.set_xticks(ticks, [labels[i] for i in ticks])
|
||||
ax.set_yticks(ticks, [labels[i] for i in ticks])
|
||||
ax.set(xlabel='Layer', ylabel='Layer', title=role.replace('model.', '') + '\nGray = unverified; diagonal is identity')
|
||||
ax.set(xlabel='Layer', ylabel='Layer', title=role.removeprefix('model.') + '\nHatching = unverified; diagonal is identity')
|
||||
ax.legend(handles=[Patch(facecolor='white', edgecolor='#999999', hatch='///', label='Unverified pair')],
|
||||
loc='upper center', bbox_to_anchor=(.5, -.12), fontsize=8, frameon=False)
|
||||
fig.colorbar(im, ax=ax, label='Exact cosine')
|
||||
save(fig, 'layer_heatmap')
|
||||
heatmap_rows = [{'a_layer': oa['layer'], 'b_layer': ob['layer'], 'role': role,
|
||||
'cosine': float(matrix[i, j]) if not np.isnan(matrix[i, j]) else '',
|
||||
'verified': bool(i != j and not np.isnan(matrix[i, j])), 'identity': i == j}
|
||||
for i, (_, oa) in enumerate(chosen) for j, (_, ob) in enumerate(chosen)]
|
||||
write_csv(out / 'heatmap_pairs.csv', heatmap_rows)
|
||||
|
||||
fig, ax = plt.subplots(figsize=(8, 4))
|
||||
for mode in ('tensor', 'block'):
|
||||
rows = pair_sets[mode]
|
||||
distances = sorted({int(r['layer_distance']) for r in rows})
|
||||
median = [np.median([float(r['cosine']) for r in rows if int(r['layer_distance']) == d and r['cosine']]) for d in distances]
|
||||
ax.plot(distances, median, 'o-', label=mode)
|
||||
ax.set(xlabel='Absolute layer distance', ylabel='Median exact cosine', title='Screened candidates; role mixture and selection bias apply')
|
||||
ax.legend()
|
||||
distance_rows = []
|
||||
fig, axes = plt.subplots(1, 2, figsize=(12, 4), sharey=True)
|
||||
for ax, mode in zip(axes, ('tensor', 'block')):
|
||||
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
|
||||
buckets = {}
|
||||
for row in pair_sets[mode]:
|
||||
if not row['cosine']:
|
||||
continue
|
||||
obj = mode_objects[int(row['a'])]
|
||||
stratum = f"{len(obj['shape'])}D" if mode == 'tensor' else obj['scale']
|
||||
buckets.setdefault(stratum, {}).setdefault(int(row['layer_distance']), []).append(float(row['cosine']))
|
||||
for stratum, by_distance in sorted(buckets.items()):
|
||||
distances = sorted(by_distance)
|
||||
medians = []
|
||||
for distance in distances:
|
||||
values = by_distance[distance]
|
||||
median = float(np.median(values))
|
||||
medians.append(median)
|
||||
distance_rows.append({'mode': mode, 'stratum': stratum, 'layer_distance': distance,
|
||||
'pairs': len(values), 'median_cosine': median,
|
||||
'q25': float(np.quantile(values, .25)), 'q75': float(np.quantile(values, .75))})
|
||||
ax.plot(distances, medians, 'o-', markersize=3, label=stratum)
|
||||
ax.set(xlabel='Absolute layer distance', ylabel='Median exact cosine',
|
||||
title=f'{mode}: dimension / scale strata', ylim=(-.1, 1.05))
|
||||
ax.legend(fontsize=8, loc='upper center', bbox_to_anchor=(.5, -.23), ncol=2, frameon=False)
|
||||
fig.suptitle('Screened candidates: module mixture and selection bias still apply')
|
||||
write_csv(out / 'layer_distance_stratified.csv', distance_rows)
|
||||
save(fig, 'layer_distance')
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
||||
for ax, mode in zip(axes, ('tensor', 'block')):
|
||||
null_rows = []
|
||||
fig, axes = plt.subplots(3, 2, figsize=(12, 10))
|
||||
bins = np.linspace(-1, 1, 26)
|
||||
for col, mode in enumerate(('tensor', 'block')):
|
||||
mode_objects = json.loads((out / f'{mode}_objects.json').read_text())
|
||||
nulls = json.loads((out / f'{mode}_nulls.json').read_text())
|
||||
for method in ('shuffled', 'gaussian_norm_matched'):
|
||||
values = [r['cosine'] for r in nulls if r['method'] == method and r['cosine'] is not None]
|
||||
ax.hist(values, bins=np.linspace(-1, 1, 26), histtype='step', label=method)
|
||||
real_random = json.loads((out / f'{mode}_random_pairs.json').read_text())
|
||||
ax.hist([r['cosine'] for r in real_random if r['cosine'] is not None], bins=np.linspace(-1, 1, 26), histtype='step', label='random actual same-role pairs')
|
||||
ax.set(title=f'{mode}: sampled null controls', xlabel='Sampled cosine', ylabel='Pairs')
|
||||
ax.legend(fontsize=8)
|
||||
cohorts = [('shuffled', [r for r in nulls if r['method'] == 'shuffled']),
|
||||
('gaussian_norm_matched', [r for r in nulls if r['method'] == 'gaussian_norm_matched']),
|
||||
('random_actual_same_role', real_random)]
|
||||
for row, ndim in enumerate((1, 2, 3)):
|
||||
ax = axes[row, col]
|
||||
any_values = False
|
||||
for method, records in cohorts:
|
||||
values = np.asarray([r['cosine'] for r in records if r['cosine'] is not None
|
||||
and len(mode_objects[r['a']]['shape']) == ndim])
|
||||
if not len(values):
|
||||
continue
|
||||
any_values = True
|
||||
values = np.clip(values, -1, 1)
|
||||
ax.hist(values, bins=bins, density=True, histtype='step', label=f'{method} (n={len(values)})')
|
||||
counts, _ = np.histogram(values, bins=bins)
|
||||
for i, count in enumerate(counts):
|
||||
null_rows.append({'mode': mode, 'ndim': ndim, 'method': method, 'samples': len(values),
|
||||
'bin_left': float(bins[i]), 'bin_right': float(bins[i + 1]),
|
||||
'count': int(count), 'density': float(count / (len(values) * (bins[i + 1] - bins[i])))})
|
||||
ax.set(title=f'{mode}: {ndim}D sampled controls', xlabel='Sampled cosine', ylabel='Density')
|
||||
if any_values:
|
||||
ax.legend(fontsize=7, loc='upper right')
|
||||
else:
|
||||
ax.text(.5, .5, 'No eligible samples', transform=ax.transAxes, ha='center')
|
||||
fig.suptitle('Dimension-stratified controls: density normalized; role mixtures may differ')
|
||||
write_csv(out / 'null_histograms.csv', null_rows)
|
||||
save(fig, 'null_controls')
|
||||
|
||||
fig, axes = plt.subplots(1, 2, figsize=(11, 4))
|
||||
@@ -117,7 +214,7 @@ def main():
|
||||
[m['left_subspace_overlap'] for m in selected], s=18, alpha=.65, label=f'mode {axis}')
|
||||
ax.set(title=f'{ndim}D blocks: bounded mode-unfolding probes', xlabel='Normalized spectrum distance', ylabel='Leading left subspace overlap (k <= 4)')
|
||||
if probes:
|
||||
ax.legend(fontsize=8)
|
||||
ax.legend(fontsize=8, loc='upper right')
|
||||
else:
|
||||
ax.text(.5, .5, 'No eligible probes', transform=ax.transAxes, ha='center')
|
||||
save(fig, 'block_structure')
|
||||
@@ -127,7 +224,8 @@ def main():
|
||||
f"- Revision: `{summary['revision']}`", f"- Stored parameters: {summary['parameter_count']:,}",
|
||||
f"- Weight bytes: {summary['model_weight_bytes']:,}",
|
||||
f"- Tensor dimensionalities: {summary['ndim_counts']}",
|
||||
f"- Analysis seconds: {summary['elapsed_seconds']:.1f} / {summary['analysis_budget_seconds']}",
|
||||
f"- Analysis and grouping/export wall time: {summary['elapsed_seconds']:.1f} seconds",
|
||||
f"- Pair-comparison cutoff: {summary['analysis_budget_seconds']} seconds; final grouping/export is additional and remains under the supervisor hard limit.",
|
||||
f"- Peak analysis RSS: {summary['peak_rss_kib'] / 1024:.1f} MiB", '',
|
||||
'## Verified sharing estimates', '',
|
||||
'| Representation | Weight-error limit | Verified sharing groups | Conservative total groups* | Largest group | Net model savings |',
|
||||
@@ -138,6 +236,13 @@ def main():
|
||||
'Savings count only verified replacement edges. Unsampled blocks remain unchanged. Different block scales must not be summed.',
|
||||
'Theoretical encoding: one stored representative per group, 16 bytes per object mapping and 64 bytes per shared-group descriptor; identity fallback if larger.',
|
||||
'Exact grouping uses direct replacement only. Scaled and spectral results are diagnostic.', '',
|
||||
'## Stratified diagnostics', '',
|
||||
'See `analysis/role_summary.csv` for same-role, same-shape statistics, separated by tensor dimensionality and block scale.',
|
||||
'See `analysis/dimensional_bytes.csv` for tensor counts versus weight-byte contributions by dimensionality.',
|
||||
'Heatmap missing pairs are hatched (`heatmap_pairs.csv`); identity entries are not empirical matches.',
|
||||
'Layer-distance medians are split by dimension/scale (`layer_distance_stratified.csv`); role and selection bias remain.',
|
||||
'Controls are split by dimension and density-normalized, with sample counts (`null_histograms.csv`); these are not significance tests.',
|
||||
'These are screened-candidate statistics, not estimates of all possible pairs. Small normalization vectors must not be conflated with large matrices.', '',
|
||||
'## Coverage', '']
|
||||
for mode in ('tensor', 'block'):
|
||||
coverage = json.loads((out / f'{mode}_coverage.json').read_text())
|
||||
|
||||
Reference in New Issue
Block a user