345 lines
21 KiB
TypeScript
345 lines
21 KiB
TypeScript
import test from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { mkdtempSync, rmSync } from "node:fs";
|
|
import { join } from "node:path";
|
|
import { tmpdir } from "node:os";
|
|
import seed from "../../../content/research/ai-companies.seed.json";
|
|
import coverageAudit from "../../../content/research/ai-companies.coverage.json";
|
|
import incomeReference from "../../../content/research/ai-income-analysis.json";
|
|
import { readResearchSnapshot } from "../src/lib/ai-research-store";
|
|
import { openSiteDatabase } from "../src/lib/site-database";
|
|
import { categorySummary, evidenceUrl, reviewCompanyIds, validateResearch, type ResearchSnapshot } from "../src/lib/ai-research";
|
|
import { consumeCode, initializeStore } from "../src/lib/auth-store";
|
|
import { newAuthConfig } from "../src/lib/auth";
|
|
import { describeIncome, groupIncome, platformIncomeSample } from "../src/lib/research-statistics";
|
|
|
|
test("research revisions share the authentication database and preserve replay limits", () => {
|
|
const dir = mkdtempSync(join(tmpdir(), "site-research-"));
|
|
const previous = process.env.AUTH_CONFIG_FILE;
|
|
process.env.AUTH_CONFIG_FILE = join(dir, "config.json");
|
|
try {
|
|
initializeStore();
|
|
const config = newAuthConfig();
|
|
for (let i = 0; i < 20; i++) consumeCode(config, "invalid");
|
|
assert.equal(consumeCode(config, "invalid"), "limited");
|
|
const initial = readResearchSnapshot();
|
|
assert.equal(initial.companies.length, 1021);
|
|
assert.equal(initial.evidence.length, 1201);
|
|
assert.equal(initial.observations.length, 187);
|
|
assert.deepEqual(initial.candidateBatches, seed.candidateBatches, "candidate gaps persist in the same database metadata");
|
|
assert.deepEqual(initial.directoryRechecks, seed.directoryRechecks, "follow-up classifications persist without rewriting the discovery audit");
|
|
assert.deepEqual(readResearchSnapshot(), initial, "reopening does not duplicate records");
|
|
const db = openSiteDatabase();
|
|
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as { name: string }[];
|
|
assert.ok(tables.some((r) => r.name === "login_state"));
|
|
assert.ok(tables.some((r) => r.name === "research_companies"));
|
|
assert.equal(db.prepare("PRAGMA foreign_keys").get()?.foreign_keys, 1);
|
|
db.close();
|
|
const next = structuredClone(seed) as ResearchSnapshot;
|
|
next.version += "-revision-test";
|
|
next.companies[0].name = "Revised company";
|
|
assert.equal(readResearchSnapshot(next).companies[0].name, "Revised company");
|
|
assert.deepEqual(readResearchSnapshot(), initial, "old release still reads its original revision");
|
|
assert.equal(consumeCode(config, "invalid"), "limited", "research import must not reset auth limits");
|
|
const collision = structuredClone(seed) as ResearchSnapshot;
|
|
collision.companies[0].name = "Changed without revision";
|
|
assert.throws(() => readResearchSnapshot(collision), /new revision/);
|
|
assert.deepEqual(readResearchSnapshot(), initial, "failed import leaves valid revision intact");
|
|
} finally {
|
|
if (previous === undefined) delete process.env.AUTH_CONFIG_FILE;
|
|
else process.env.AUTH_CONFIG_FILE = previous;
|
|
rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
test("research validation rejects broken evidence and unsafe source links", () => {
|
|
validateResearch(seed as ResearchSnapshot);
|
|
const invalid = structuredClone(seed) as ResearchSnapshot;
|
|
invalid.observations[0].evidence = "missing";
|
|
assert.throws(() => validateResearch(invalid), /observation/);
|
|
assert.equal(evidenceUrl("javascript:alert(1)"), undefined);
|
|
assert.equal(evidenceUrl("/local-link"), undefined);
|
|
assert.equal(evidenceUrl("https://example.com/evidence"), "https://example.com/evidence");
|
|
});
|
|
|
|
test("reviews link explicit companies without turning global or excluded cases into companies", () => {
|
|
const snapshot = structuredClone(seed) as ResearchSnapshot;
|
|
assert.equal(snapshot.reviews.filter((r) => r.scope === "companies").length, 52);
|
|
assert.equal(snapshot.reviews.filter((r) => r.scope === "dataset").length, 4);
|
|
assert.equal(snapshot.reviews.filter((r) => r.scope === "excluded-candidate").length, 12);
|
|
const review = snapshot.reviews.find((r) => r.id === "R4")!;
|
|
assert.deepEqual(reviewCompanyIds(snapshot, review), ["AI007", "AI013", "AI021", "AI034"]);
|
|
assert.deepEqual(reviewCompanyIds(snapshot, snapshot.reviews.find((r) => r.id === "CN-R-PSI")!), []);
|
|
const oldReview = structuredClone(snapshot.reviews.find((r) => r.id === "TECH-GRAD-R-SCOPE")!);
|
|
delete oldReview.scope;
|
|
delete oldReview.companyIds;
|
|
assert.deepEqual(reviewCompanyIds(snapshot, oldReview), ["AI051"], "old revisions retain evidence-owner fallback");
|
|
review.companyIds = ["unknown-company"];
|
|
assert.throws(() => validateResearch(snapshot), /research review/);
|
|
review.companyIds = ["AI007", "AI007"];
|
|
assert.throws(() => validateResearch(snapshot), /research review/);
|
|
review.companyIds = [];
|
|
assert.throws(() => validateResearch(snapshot), /research review/);
|
|
});
|
|
|
|
test("disclosure counts exclude client income, biographies, targets and duplicate milestones", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const summary = categorySummary(snapshot);
|
|
assert.equal(summary.reduce((n, c) => n + c.count, 0), 1021);
|
|
assert.equal(summary.reduce((n, c) => n + c.self, 0), 26);
|
|
assert.equal(summary.reduce((n, c) => n + c.estimate, 0), 1);
|
|
assert.equal(summary.reduce((n, c) => n + c.platform, 0), 16);
|
|
assert.equal(summary.reduce((n, c) => n + c.unknown, 0), 978);
|
|
assert.equal(snapshot.companies.find((c) => c.name === "Rork")?.revenueStatus, "unknown");
|
|
assert.equal(snapshot.companies.find((c) => c.name === "Result")?.revenueStatus, "unknown");
|
|
assert.ok(snapshot.observations.filter((o) => o.selection === "signal").every((o) => o.scope === "公司整体"));
|
|
});
|
|
|
|
test("technical reviews keep accessible materials separate from tested capabilities", () => {
|
|
const snapshot = structuredClone(seed) as ResearchSnapshot;
|
|
assert.equal(snapshot.companies.filter((c) => c.technicalReview).length, 13);
|
|
assert.equal(snapshot.companies.filter((c) => c.technicalReview?.level === "public-artifact").length, 6);
|
|
assert.equal(snapshot.companies.filter((c) => c.technicalReview?.level === "independent-test").length, 1);
|
|
assert.equal(snapshot.companies.find((c) => c.id === "AI051")?.revenueStatus, "unknown");
|
|
assert.match(snapshot.companies.find((c) => c.id === "AI051")!.technicalReview!.testScope!, /Sep 2026/);
|
|
assert.ok(snapshot.observations.filter((o) => o.id.startsWith("TECH-GRAD-O-")).every((o) => o.selection === "technical" && o.comparisonGroup === null));
|
|
const company = snapshot.companies.find((c) => c.id === "AI061")!;
|
|
const review = company.technicalReview!;
|
|
review.evidenceIds = ["missing-technical-evidence"];
|
|
assert.throws(() => validateResearch(snapshot), /technical review/);
|
|
review.evidenceIds = ["TECH-NAIVE-HF"];
|
|
review.reviewedAt = "2026-02-30";
|
|
assert.throws(() => validateResearch(snapshot), /technical review/);
|
|
review.reviewedAt = "2099-01-01";
|
|
assert.throws(() => validateResearch(snapshot), /technical review/);
|
|
review.reviewedAt = snapshot.asOf;
|
|
review.level = "independent-test";
|
|
assert.throws(() => validateResearch(snapshot), /technical review/,
|
|
"an accessible artifact cannot become a test record without a stated test scope");
|
|
});
|
|
|
|
test("ambiguous ARR claims retain currency, precision and attribution limits", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const variant = snapshot.observations.filter((o) => o.company === "AI066");
|
|
assert.deepEqual(variant.filter((o) => o.selection === "crosscheck").map((o) => o.value).sort(), [500000, 600000]);
|
|
assert.equal(variant.filter((o) => o.selection === "signal").length, 0);
|
|
assert.ok(variant.filter((o) => [2500000, 40000000].includes(o.value)).every((o) => o.selection === "excluded"));
|
|
const codos = snapshot.observations.find((o) => o.id === "SR-O-AI065-arr")!;
|
|
assert.equal(codos.bound, "≥");
|
|
assert.equal(codos.unit, "币种未注明");
|
|
assert.equal(codos.comparisonGroup, null);
|
|
assert.ok(snapshot.observations.filter((o) => o.id.startsWith("SR-O-")).every((o) => o.comparisonGroup === null && o.unit !== "USD"));
|
|
assert.equal(snapshot.companies.find((c) => c.id === "AI071")?.year, 2021);
|
|
const rocket = snapshot.observations.find((o) => o.id === "ROCKET-O-arr")!;
|
|
assert.equal(rocket.selection, "historical");
|
|
assert.equal(rocket.comparisonGroup, null);
|
|
assert.notEqual(rocket.scope, "公司整体");
|
|
assert.equal(snapshot.companies.find((c) => c.id === "AI070")?.revenueStatus, "unknown");
|
|
assert.equal(snapshot.observations.filter((o) => o.company === "AI070").length, 0,
|
|
"unreadable announcement summaries do not become numeric ARR");
|
|
});
|
|
|
|
// Retractions and forecasts must stay visible for review without becoming revenue signals.
|
|
test("retracted ARR and projected margins remain excluded", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const cluely = snapshot.companies.find((c) => c.name === "Cluely")!;
|
|
assert.equal(cluely.revenueStatus, "unknown");
|
|
assert.ok(snapshot.reviews.some((r) => r.company === "Cluely" && r.evidenceIds.includes("E100")));
|
|
const retracted = snapshot.observations.filter((o) => o.company === cluely.id);
|
|
assert.ok(retracted.length > 0);
|
|
assert.ok(retracted.every((o) => o.selection === "excluded"));
|
|
const forecast = snapshot.observations.find((o) => o.kind === "非GAAP毛利率预期")!;
|
|
assert.equal(forecast.selection, "excluded");
|
|
const peec = snapshot.companies.find((c) => c.name === "Peec AI")!;
|
|
assert.equal(peec.teamBound, ">");
|
|
assert.equal(snapshot.observations.filter((o) => o.company === peec.id && o.selection === "signal").length, 1);
|
|
assert.ok(snapshot.observations.some((o) => o.company === peec.id && o.selection === "historical-estimate"));
|
|
});
|
|
|
|
test("directory coverage reconciles every candidate and keeps profile-only income unknown", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const coverage = snapshot.coverage!;
|
|
assert.equal(coverageAudit.records.length, coverage.candidateRecords);
|
|
assert.equal(new Set(coverageAudit.records.map((r) => r.ycId)).size, coverage.candidateRecords);
|
|
const count = (status: string) => coverageAudit.records.filter((r) => r.status === status).length;
|
|
assert.equal(count("imported"), coverage.newProfiles);
|
|
assert.equal(count("existing"), coverage.existingProfiles);
|
|
assert.equal(count("date-conflict"), coverage.dateConflicts);
|
|
assert.equal(count("year-outside-or-missing"), coverage.otherOrMissingYear);
|
|
assert.equal(count("ai-unclear"), coverage.unclearAI);
|
|
for (const row of coverageAudit.records.filter((r) => r.status === "imported")) {
|
|
const company = snapshot.companies.find((c) => c.id === `YC${row.ycId}`)!;
|
|
assert.ok(company.profileOnly || company.directoryOrigin === "yc-2025-2026");
|
|
assert.equal(company.url, row.source);
|
|
if (company.profileOnly) assert.equal(company.revenueStatus, "unknown");
|
|
assert.equal(company.year, row.foundedYear);
|
|
assert.ok([2025, 2026].includes(company.year!));
|
|
}
|
|
const invalid = structuredClone(snapshot);
|
|
invalid.coverage!.newProfiles += 1;
|
|
assert.throws(() => validateResearch(invalid), /coverage accounting/);
|
|
const promoted = snapshot.companies.find((c) => c.id === "YC30215")!;
|
|
assert.equal(promoted.profileOnly, false);
|
|
assert.equal(promoted.directoryOrigin, "yc-2025-2026");
|
|
assert.equal(promoted.revenueStatus, "self");
|
|
const lostOrigin = structuredClone(snapshot);
|
|
delete lostOrigin.companies.find((c) => c.id === promoted.id)!.directoryOrigin;
|
|
assert.throws(() => validateResearch(lostOrigin), /coverage accounting/);
|
|
});
|
|
|
|
test("income descriptions report sample dispersion and maximum-value sensitivity", () => {
|
|
const stats = describeIncome([100, 200, 300])!;
|
|
assert.equal(stats.mean, 200);
|
|
assert.equal(stats.median, 200);
|
|
assert.equal(stats.sampleDeviation, 100);
|
|
assert.equal(stats.cv, .5);
|
|
assert.equal(stats.largestShare, .5);
|
|
assert.equal(stats.meanWithoutLargest, 150);
|
|
assert.equal(stats.largestRemovalDrop, .25);
|
|
assert.equal(describeIncome([5])!.cv, null);
|
|
assert.equal(describeIncome([0, 0])!.mean, 0, "observed zero is distinct from missing");
|
|
assert.equal(describeIncome([0, 0])!.cv, null);
|
|
assert.equal(describeIncome([]), null);
|
|
assert.equal(describeIncome([100, NaN]), null);
|
|
assert.equal(describeIncome([-1]), null);
|
|
assert.ok(describeIncome([1, 2, 100])!.largestRemovalDrop! > .95);
|
|
});
|
|
|
|
test("platform samples exclude stale data and keep ARR, estimates and collections separate", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const recent = platformIncomeSample(snapshot, { windowDays: 7 });
|
|
assert.equal(recent.points.length, 14);
|
|
assert.equal(recent.excluded.length, 3);
|
|
assert.ok(recent.points.every((p) => p.observation.metric === "mrr" && p.observation.selection === "platform"));
|
|
assert.deepEqual(new Set(recent.excluded.map((p) => p.observation.company)), new Set(["AI016", "MR013", "MR014"]));
|
|
assert.ok(recent.points.some((p) => p.company.name === "TrackAI" && p.observation.connectionExpired), "dated recent value is not a claim of active connection");
|
|
const older = platformIncomeSample(snapshot, { windowDays: 90 });
|
|
assert.equal(older.points.length, 15);
|
|
assert.ok(older.points.some((p) => p.company.id === "AI016"));
|
|
const scaled = platformIncomeSample(snapshot, { windowDays: 7, annualize: true });
|
|
const monthlyGroups = groupIncome(recent.points);
|
|
const annualGroups = groupIncome(scaled.points);
|
|
for (const group of monthlyGroups) {
|
|
const annual = annualGroups.find((g) => g.key === group.key)!;
|
|
assert.ok(Math.abs(annual.stats.mean - group.stats.mean * 12) < 1e-6);
|
|
if (group.stats.cv !== null) assert.ok(Math.abs(annual.stats.cv! - group.stats.cv) < 1e-10);
|
|
}
|
|
const invalid = structuredClone(snapshot);
|
|
invalid.observations = invalid.observations.filter((o) => o.company === "MR001" && o.metric === "mrr");
|
|
const first = invalid.observations[0];
|
|
invalid.observations.push({ ...first, id: "future", observedAt: "2026-11-01", value: 999999 });
|
|
invalid.observations.push({ ...first, id: "bad-date", observedAt: "2026-02-30", value: 999999 });
|
|
invalid.observations.push({ ...first, id: "currency", unit: "EUR", value: 999999 });
|
|
invalid.observations.push({ ...first, id: "older", observedAt: "2026-10-28", value: 1 });
|
|
const selected = platformIncomeSample(invalid, { windowDays: 7 });
|
|
assert.equal(selected.points.length, 1);
|
|
assert.equal(selected.points[0].observation.id, first.id, "bad newer data and older duplicate must not replace latest valid point");
|
|
assert.equal(selected.excluded.length, 4);
|
|
assert.throws(() => platformIncomeSample(snapshot, { windowDays: -1 }), /window/);
|
|
});
|
|
|
|
test("distribution statistics match independently calculated snapshot references", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
assert.equal(incomeReference.sourceVersion, snapshot.version);
|
|
for (const reference of incomeReference.windows) {
|
|
const sample = platformIncomeSample(snapshot, { windowDays: reference.days });
|
|
assert.deepEqual(new Set(sample.points.map((p) => p.company.id)), new Set(reference.products));
|
|
const groups = groupIncome(sample.points);
|
|
assert.equal(groups.length, reference.groups.length);
|
|
for (const expected of reference.groups) {
|
|
const actual = groups.find((g) => g.category === expected.category && g.provider === expected.provider && g.model === expected.model)!;
|
|
assert.equal(actual.stats.n, expected.n);
|
|
for (const metric of ["mean", "median", "sampleDeviation", "cv", "meanWithoutLargest", "largestRemovalDrop"] as const) {
|
|
const value = expected[metric];
|
|
if (value === null) assert.equal(actual.stats[metric], null);
|
|
else assert.ok(Math.abs(actual.stats[metric]! - value) < 1e-6, metric);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
|
|
test("product launch candidates preserve unresolved years and many launches per company", () => {
|
|
const snapshot = structuredClone(seed) as ResearchSnapshot;
|
|
const batch = snapshot.candidateBatches![0];
|
|
assert.equal(batch.complete, false);
|
|
assert.equal(batch.records.length, 19);
|
|
const kilo = batch.records.filter((r) => r.companyId === "AI072");
|
|
assert.equal(kilo.length, 2, "IDE and hosted products resolve to one company");
|
|
assert.equal(snapshot.companies.filter((c) => c.id === "AI072").length, 1);
|
|
assert.ok(batch.records.some((r) => r.status === "pending-year" && !r.companyId));
|
|
assert.ok(batch.records.some((r) => r.status === "existing-outside-cohort" && r.companyId === "AI054"));
|
|
kilo[0].companyId = "missing-company";
|
|
assert.throws(() => validateResearch(snapshot), /research candidate/);
|
|
});
|
|
|
|
test("annual launch archives exclude older companies only with evidence and reuse existing records", () => {
|
|
const snapshot = structuredClone(seed) as ResearchSnapshot;
|
|
const batch = snapshot.candidateBatches!.find((b) => b.id === "PH-2025-visible-1")!;
|
|
assert.equal(batch.complete, false);
|
|
assert.equal(batch.records.length, 19);
|
|
const older = batch.records.filter((r) => r.status === "outside-cohort");
|
|
assert.equal(older.length, 7);
|
|
assert.ok(older.every((r) => !r.companyId && r.evidenceIds?.length));
|
|
assert.equal(batch.records.find((r) => r.name === "Trace")?.companyId, "YC30687");
|
|
const trace = snapshot.companies.find((c) => c.id === "YC30687")!;
|
|
assert.equal(trace.profileOnly, false);
|
|
assert.equal(trace.directoryOrigin, "yc-2025-2026");
|
|
assert.equal(trace.revenueStatus, "unknown");
|
|
assert.ok(!snapshot.observations.some((o) => o.evidence === "TRACE-HOME"), "demo metrics must not become reported results");
|
|
older[0].evidenceIds = [];
|
|
assert.throws(() => validateResearch(snapshot), /research candidate/);
|
|
older[0].evidenceIds = ["missing-proof"];
|
|
assert.throws(() => validateResearch(snapshot), /research candidate/);
|
|
});
|
|
|
|
test("unreconciled directory financials remain source reviews instead of income observations", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const review = snapshot.reviews.find((r) => r.id === "PH-R-SOURCE-QUALITY")!;
|
|
assert.equal(review.scope, "dataset");
|
|
assert.ok(review.evidenceIds.includes("PH-TANA-DR"));
|
|
assert.equal(reviewCompanyIds(snapshot, review).length, 0);
|
|
assert.ok(!snapshot.observations.some((o) => o.evidence === "PH-TANA-DR"));
|
|
assert.equal(snapshot.candidateBatches!.flatMap((b) => b.records).filter((r) => r.status === "pending-year").length, 21);
|
|
});
|
|
|
|
test("product pivots and founder histories do not become new company years or current revenue", () => {
|
|
const snapshot = seed as ResearchSnapshot;
|
|
const pivot = snapshot.companies.find((c) => c.id === "AI073")!;
|
|
assert.equal(pivot.year, null, "CRM launch is not a verified company founding year");
|
|
assert.equal(pivot.revenueStatus, "unknown");
|
|
const candidate = snapshot.candidateBatches!.flatMap((b) => b.records).find((r) => r.id === "PH26-11")!;
|
|
assert.equal(candidate.status, "existing-outside-cohort");
|
|
assert.equal(candidate.companyId, pivot.id);
|
|
assert.ok(!platformIncomeSample(snapshot, { windowDays: null }).points.some((point) => point.company.id === pivot.id));
|
|
assert.ok(!snapshot.observations.some((o) => ["PH-FUNDRAISLY-ABOUT", "LF-PIVOT", "LF-HISTORY"].includes(o.evidence)), "other ventures' ARR, users and funding must not transfer");
|
|
});
|
|
|
|
test("follow-up evidence can admit an uncertain directory candidate without rewriting its original status", () => {
|
|
const snapshot = structuredClone(seed) as ResearchSnapshot;
|
|
const followup = snapshot.directoryRechecks!.find((row) => row.ycId === 36088)!;
|
|
const original = coverageAudit.records.find((row) => row.ycId === followup.ycId)!;
|
|
assert.equal(original.status, "ai-unclear");
|
|
assert.equal(followup.originalStatus, original.status);
|
|
assert.equal(snapshot.companies.find((c) => c.id === followup.companyId)?.year, 2026);
|
|
assert.equal(snapshot.candidateBatches!.flatMap((b) => b.records).find((r) => r.name === "OpenSEO")?.companyId, followup.companyId);
|
|
assert.ok(!snapshot.observations.some((o) => ["SEO-PRICE", "UP-YC", "PAZI-PRICE"].includes(o.evidence)), "pricing, other ventures' ARR and prepaid balances stay outside income observations");
|
|
followup.companyId = "missing-company";
|
|
assert.throws(() => validateResearch(snapshot), /directory recheck/);
|
|
});
|
|
|
|
test("Chinese discovery entries preserve unread articles and use separate primary evidence", () => {
|
|
const snapshot = structuredClone(seed) as ResearchSnapshot;
|
|
const batch = snapshot.candidateBatches!.find((b) => b.id === "JQ-2026-targeted-1")!;
|
|
assert.equal(batch.complete, false);
|
|
assert.equal(batch.records.length, 4);
|
|
assert.ok(batch.records.every((row) => row.kind === "company" && row.profileStatus === "unavailable"));
|
|
assert.equal(batch.records.filter((row) => row.status === "linked").length, 2);
|
|
assert.equal(batch.records.find((row) => row.name === "Utopai")?.status, "pending-year");
|
|
assert.ok(!snapshot.observations.some((o) => ["PS-CODE", "SG-START", "SG-APR", "SSI-HISTORY"].includes(o.evidence)), "task costs, investment, capacity and financing are not revenue");
|
|
const prism = snapshot.companies.find((c) => c.id === "AI075")!;
|
|
assert.equal(prism.team, null, "a team range or linked employees must not become exact headcount");
|
|
batch.records[0].url = null;
|
|
assert.throws(() => validateResearch(snapshot), /research candidate/);
|
|
});
|