258 lines
15 KiB
TypeScript
258 lines
15 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import seed from "../../../content/research/ai-companies.seed.json";
|
|
import coverageAudit from "../../../content/research/ai-companies.coverage.json";
|
|
import incomeReference from "../../../content/research/ai-income-analysis.json";
|
|
import { readResearchSnapshot } from "../src/lib/ai-research-store";
|
|
import { openSiteDatabase } from "../src/lib/site-database";
|
|
import { categorySummary, evidenceUrl, reviewCompanyIds, validateResearch, type ResearchSnapshot } from "../src/lib/ai-research";
|
|
import { consumeCode, initializeStore } from "../src/lib/auth-store";
|
|
import { newAuthConfig } from "../src/lib/auth";
|
|
import { describeIncome, groupIncome, platformIncomeSample } from "../src/lib/research-statistics";
|
|
|
|
test("research revisions share the authentication database and preserve replay limits", () => {
|
|
const dir = mkdtempSync(join(tmpdir(), "site-research-"));
|
|
const previous = process.env.AUTH_CONFIG_FILE;
|
|
process.env.AUTH_CONFIG_FILE = join(dir, "config.json");
|
|
try {
|
|
initializeStore();
|
|
const config = newAuthConfig();
|
|
for (let i = 0; i < 20; i++) consumeCode(config, "invalid");
|
|
assert.equal(consumeCode(config, "invalid"), "limited");
|
|
const initial = readResearchSnapshot();
|
|
assert.equal(initial.companies.length, 1015);
|
|
assert.equal(initial.evidence.length, 1091);
|
|
assert.equal(initial.observations.length, 183);
|
|
assert.deepEqual(readResearchSnapshot(), initial, "reopening does not duplicate records");
|
|
const db = openSiteDatabase();
|
|
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as { name: string }[];
|
|
assert.ok(tables.some((r) => r.name === "login_state"));
|
|
assert.ok(tables.some((r) => r.name === "research_companies"));
|
|
assert.equal(db.prepare("PRAGMA foreign_keys").get()?.foreign_keys, 1);
|
|
db.close();
|
|
const next = structuredClone(seed) as ResearchSnapshot;
|
|
next.version += "-revision-test";
|
|
next.companies[0].name = "Revised company";
|
|
assert.equal(readResearchSnapshot(next).companies[0].name, "Revised company");
|
|
assert.deepEqual(readResearchSnapshot(), initial, "old release still reads its original revision");
|
|
assert.equal(consumeCode(config, "invalid"), "limited", "research import must not reset auth limits");
|
|
const collision = structuredClone(seed) as ResearchSnapshot;
|
|
collision.companies[0].name = "Changed without revision";
|
|
assert.throws(() => readResearchSnapshot(collision), /new revision/);
|
|
assert.deepEqual(readResearchSnapshot(), initial, "failed import leaves valid revision intact");
|
|
} finally {
|
|
if (previous === undefined) delete process.env.AUTH_CONFIG_FILE;
|
|
else process.env.AUTH_CONFIG_FILE = previous;
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("research validation rejects broken evidence and unsafe source links", () => {
|
|
validateResearch(seed as ResearchSnapshot);
|
|
const invalid = structuredClone(seed) as ResearchSnapshot;
|
|
invalid.observations[0].evidence = "missing";
|
|
assert.throws(() => validateResearch(invalid), /observation/);
|
|
assert.equal(evidenceUrl("javascript:alert(1)"), undefined);
|
|
assert.equal(evidenceUrl("/local-link"), undefined);
|
|
assert.equal(evidenceUrl("https://example.com/evidence"), "https://example.com/evidence");
|
|
});
|
|
|
|
test("reviews link explicit companies without turning global or excluded cases into companies", () => {
|
|
const snapshot = structuredClone(seed) as ResearchSnapshot;
|
|
assert.equal(snapshot.reviews.filter((r) => r.scope === "companies").length, 43);
|
|
assert.equal(snapshot.reviews.filter((r) => r.scope === "dataset").length, 3);
|
|
assert.equal(snapshot.reviews.filter((r) => r.scope === "excluded-candidate").length, 8);
|
|
const review = snapshot.reviews.find((r) => r.id === "R4")!;
|
|
assert.deepEqual(reviewCompanyIds(snapshot, review), ["AI007", "AI013", "AI021", "AI034"]);
|
|
assert.deepEqual(reviewCompanyIds(snapshot, snapshot.reviews.find((r) => r.id === "CN-R-PSI")!), []);
|
|
const oldReview = structuredClone(snapshot.reviews.find((r) => r.id === "TECH-GRAD-R-SCOPE")!);
|
|
delete oldReview.scope;
|
|
delete oldReview.companyIds;
|
|
assert.deepEqual(reviewCompanyIds(snapshot, oldReview), ["AI051"], "old revisions retain evidence-owner fallback");
|
|
review.companyIds = ["unknown-company"];
|
|
assert.throws(() => validateResearch(snapshot), /research review/);
|
|
review.companyIds = ["AI007", "AI007"];
|
|
assert.throws(() => validateResearch(snapshot), /research review/);
|
|
review.companyIds = [];
|
|
assert.throws(() => validateResearch(snapshot), /research review/);
|
|
});
|
|
|
|
test("disclosure counts exclude client income, biographies, targets and duplicate milestones", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const summary = categorySummary(snapshot);
|
|
assert.equal(summary.reduce((n, c) => n + c.count, 0), 1015);
|
|
assert.equal(summary.reduce((n, c) => n + c.self, 0), 26);
|
|
assert.equal(summary.reduce((n, c) => n + c.estimate, 0), 1);
|
|
assert.equal(summary.reduce((n, c) => n + c.platform, 0), 16);
|
|
assert.equal(summary.reduce((n, c) => n + c.unknown, 0), 972);
|
|
assert.equal(snapshot.companies.find((c) => c.name === "Rork")?.revenueStatus, "unknown");
|
|
assert.equal(snapshot.companies.find((c) => c.name === "Result")?.revenueStatus, "unknown");
|
|
assert.ok(snapshot.observations.filter((o) => o.selection === "signal").every((o) => o.scope === "公司整体"));
|
|
});
|
|
|
|
test("technical reviews keep accessible materials separate from tested capabilities", () => {
|
|
const snapshot = structuredClone(seed) as ResearchSnapshot;
|
|
assert.equal(snapshot.companies.filter((c) => c.technicalReview).length, 7);
|
|
assert.equal(snapshot.companies.filter((c) => c.technicalReview?.level === "public-artifact").length, 3);
|
|
assert.equal(snapshot.companies.filter((c) => c.technicalReview?.level === "independent-test").length, 1);
|
|
assert.equal(snapshot.companies.find((c) => c.id === "AI051")?.revenueStatus, "unknown");
|
|
assert.match(snapshot.companies.find((c) => c.id === "AI051")!.technicalReview!.testScope!, /Sep 2026/);
|
|
assert.ok(snapshot.observations.filter((o) => o.id.startsWith("TECH-GRAD-O-")).every((o) => o.selection === "technical" && o.comparisonGroup === null));
|
|
const company = snapshot.companies.find((c) => c.id === "AI061")!;
|
|
const review = company.technicalReview!;
|
|
review.evidenceIds = ["missing-technical-evidence"];
|
|
assert.throws(() => validateResearch(snapshot), /technical review/);
|
|
review.evidenceIds = ["TECH-NAIVE-HF"];
|
|
review.reviewedAt = "2026-02-30";
|
|
assert.throws(() => validateResearch(snapshot), /technical review/);
|
|
review.reviewedAt = "2099-01-01";
|
|
assert.throws(() => validateResearch(snapshot), /technical review/);
|
|
review.reviewedAt = snapshot.asOf;
|
|
review.level = "independent-test";
|
|
assert.throws(() => validateResearch(snapshot), /technical review/,
|
|
"an accessible artifact cannot become a test record without a stated test scope");
|
|
});
|
|
|
|
test("ambiguous ARR claims retain currency, precision and attribution limits", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const variant = snapshot.observations.filter((o) => o.company === "AI066");
|
|
assert.deepEqual(variant.filter((o) => o.selection === "crosscheck").map((o) => o.value).sort(), [500000, 600000]);
|
|
assert.equal(variant.filter((o) => o.selection === "signal").length, 0);
|
|
assert.ok(variant.filter((o) => [2500000, 40000000].includes(o.value)).every((o) => o.selection === "excluded"));
|
|
const codos = snapshot.observations.find((o) => o.id === "SR-O-AI065-arr")!;
|
|
assert.equal(codos.bound, "≥");
|
|
assert.equal(codos.unit, "币种未注明");
|
|
assert.equal(codos.comparisonGroup, null);
|
|
assert.ok(snapshot.observations.filter((o) => o.id.startsWith("SR-O-")).every((o) => o.comparisonGroup === null && o.unit !== "USD"));
|
|
assert.equal(snapshot.companies.find((c) => c.id === "AI071")?.year, 2021);
|
|
const rocket = snapshot.observations.find((o) => o.id === "ROCKET-O-arr")!;
|
|
assert.equal(rocket.selection, "historical");
|
|
assert.equal(rocket.comparisonGroup, null);
|
|
assert.notEqual(rocket.scope, "公司整体");
|
|
assert.equal(snapshot.companies.find((c) => c.id === "AI070")?.revenueStatus, "unknown");
|
|
assert.equal(snapshot.observations.filter((o) => o.company === "AI070").length, 0,
|
|
"unreadable announcement summaries do not become numeric ARR");
|
|
});
|
|
|
|
// Retractions and forecasts must stay visible for review without becoming revenue signals.
|
|
test("retracted ARR and projected margins remain excluded", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const cluely = snapshot.companies.find((c) => c.name === "Cluely")!;
|
|
assert.equal(cluely.revenueStatus, "unknown");
|
|
assert.ok(snapshot.reviews.some((r) => r.company === "Cluely" && r.evidenceIds.includes("E100")));
|
|
const retracted = snapshot.observations.filter((o) => o.company === cluely.id);
|
|
assert.ok(retracted.length > 0);
|
|
assert.ok(retracted.every((o) => o.selection === "excluded"));
|
|
const forecast = snapshot.observations.find((o) => o.kind === "非GAAP毛利率预期")!;
|
|
assert.equal(forecast.selection, "excluded");
|
|
const peec = snapshot.companies.find((c) => c.name === "Peec AI")!;
|
|
assert.equal(peec.teamBound, ">");
|
|
assert.equal(snapshot.observations.filter((o) => o.company === peec.id && o.selection === "signal").length, 1);
|
|
assert.ok(snapshot.observations.some((o) => o.company === peec.id && o.selection === "historical-estimate"));
|
|
});
|
|
|
|
test("directory coverage reconciles every candidate and keeps profile-only income unknown", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const coverage = snapshot.coverage!;
|
|
assert.equal(coverageAudit.records.length, coverage.candidateRecords);
|
|
assert.equal(new Set(coverageAudit.records.map((r) => r.ycId)).size, coverage.candidateRecords);
|
|
const count = (status: string) => coverageAudit.records.filter((r) => r.status === status).length;
|
|
assert.equal(count("imported"), coverage.newProfiles);
|
|
assert.equal(count("existing"), coverage.existingProfiles);
|
|
assert.equal(count("date-conflict"), coverage.dateConflicts);
|
|
assert.equal(count("year-outside-or-missing"), coverage.otherOrMissingYear);
|
|
assert.equal(count("ai-unclear"), coverage.unclearAI);
|
|
for (const row of coverageAudit.records.filter((r) => r.status === "imported")) {
|
|
const company = snapshot.companies.find((c) => c.id === `YC${row.ycId}`)!;
|
|
assert.ok(company.profileOnly || company.directoryOrigin === "yc-2025-2026");
|
|
assert.equal(company.url, row.source);
|
|
if (company.profileOnly) assert.equal(company.revenueStatus, "unknown");
|
|
assert.equal(company.year, row.foundedYear);
|
|
assert.ok([2025, 2026].includes(company.year!));
|
|
}
|
|
const invalid = structuredClone(snapshot);
|
|
invalid.coverage!.newProfiles += 1;
|
|
assert.throws(() => validateResearch(invalid), /coverage accounting/);
|
|
const promoted = snapshot.companies.find((c) => c.id === "YC30215")!;
|
|
assert.equal(promoted.profileOnly, false);
|
|
assert.equal(promoted.directoryOrigin, "yc-2025-2026");
|
|
assert.equal(promoted.revenueStatus, "self");
|
|
const lostOrigin = structuredClone(snapshot);
|
|
delete lostOrigin.companies.find((c) => c.id === promoted.id)!.directoryOrigin;
|
|
assert.throws(() => validateResearch(lostOrigin), /coverage accounting/);
|
|
});
|
|
|
|
test("income descriptions report sample dispersion and maximum-value sensitivity", () => {
|
|
const stats = describeIncome([100, 200, 300])!;
|
|
assert.equal(stats.mean, 200);
|
|
assert.equal(stats.median, 200);
|
|
assert.equal(stats.sampleDeviation, 100);
|
|
assert.equal(stats.cv, .5);
|
|
assert.equal(stats.largestShare, .5);
|
|
assert.equal(stats.meanWithoutLargest, 150);
|
|
assert.equal(stats.largestRemovalDrop, .25);
|
|
assert.equal(describeIncome([5])!.cv, null);
|
|
assert.equal(describeIncome([0, 0])!.mean, 0, "observed zero is distinct from missing");
|
|
assert.equal(describeIncome([0, 0])!.cv, null);
|
|
assert.equal(describeIncome([]), null);
|
|
assert.equal(describeIncome([100, NaN]), null);
|
|
assert.equal(describeIncome([-1]), null);
|
|
assert.ok(describeIncome([1, 2, 100])!.largestRemovalDrop! > .95);
|
|
});
|
|
|
|
test("platform samples exclude stale data and keep ARR, estimates and collections separate", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const recent = platformIncomeSample(snapshot, { windowDays: 7 });
|
|
assert.equal(recent.points.length, 14);
|
|
assert.equal(recent.excluded.length, 3);
|
|
assert.ok(recent.points.every((p) => p.observation.metric === "mrr" && p.observation.selection === "platform"));
|
|
assert.deepEqual(new Set(recent.excluded.map((p) => p.observation.company)), new Set(["AI016", "MR013", "MR014"]));
|
|
assert.ok(recent.points.some((p) => p.company.name === "TrackAI" && p.observation.connectionExpired), "dated recent value is not a claim of active connection");
|
|
const older = platformIncomeSample(snapshot, { windowDays: 90 });
|
|
assert.equal(older.points.length, 15);
|
|
assert.ok(older.points.some((p) => p.company.id === "AI016"));
|
|
const scaled = platformIncomeSample(snapshot, { windowDays: 7, annualize: true });
|
|
const monthlyGroups = groupIncome(recent.points);
|
|
const annualGroups = groupIncome(scaled.points);
|
|
for (const group of monthlyGroups) {
|
|
const annual = annualGroups.find((g) => g.key === group.key)!;
|
|
assert.ok(Math.abs(annual.stats.mean - group.stats.mean * 12) < 1e-6);
|
|
if (group.stats.cv !== null) assert.ok(Math.abs(annual.stats.cv! - group.stats.cv) < 1e-10);
|
|
}
|
|
const invalid = structuredClone(snapshot);
|
|
invalid.observations = invalid.observations.filter((o) => o.company === "MR001" && o.metric === "mrr");
|
|
const first = invalid.observations[0];
|
|
invalid.observations.push({ ...first, id: "future", observedAt: "2026-11-01", value: 999999 });
|
|
invalid.observations.push({ ...first, id: "bad-date", observedAt: "2026-02-30", value: 999999 });
|
|
invalid.observations.push({ ...first, id: "currency", unit: "EUR", value: 999999 });
|
|
invalid.observations.push({ ...first, id: "older", observedAt: "2026-10-28", value: 1 });
|
|
const selected = platformIncomeSample(invalid, { windowDays: 7 });
|
|
assert.equal(selected.points.length, 1);
|
|
assert.equal(selected.points[0].observation.id, first.id, "bad newer data and older duplicate must not replace latest valid point");
|
|
assert.equal(selected.excluded.length, 4);
|
|
assert.throws(() => platformIncomeSample(snapshot, { windowDays: -1 }), /window/);
|
|
});
|
|
|
|
test("distribution statistics match independently calculated snapshot references", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
assert.equal(incomeReference.sourceVersion, snapshot.version);
|
|
for (const reference of incomeReference.windows) {
|
|
const sample = platformIncomeSample(snapshot, { windowDays: reference.days });
|
|
assert.deepEqual(new Set(sample.points.map((p) => p.company.id)), new Set(reference.products));
|
|
const groups = groupIncome(sample.points);
|
|
assert.equal(groups.length, reference.groups.length);
|
|
for (const expected of reference.groups) {
|
|
const actual = groups.find((g) => g.category === expected.category && g.provider === expected.provider && g.model === expected.model)!;
|
|
assert.equal(actual.stats.n, expected.n);
|
|
for (const metric of ["mean", "median", "sampleDeviation", "cv", "meanWithoutLargest", "largestRemovalDrop"] as const) {
|
|
const value = expected[metric];
|
|
if (value === null) assert.equal(actual.stats[metric], null);
|
|
else assert.ok(Math.abs(actual.stats[metric]! - value) < 1e-6, metric);
|
|
}
|
|
}
|
|
}
|
|
});
|