13924f8985
- Next.js/React/TypeScript app: overview, paper explorer with FTS5 search, paper detail, global taxonomy tree with per-conference tag breakdown - Python ingestion (ICLR/ICML 2026 via paperlists, ACL Anthology 2026) into data/papers.db, plus keyword-scored taxonomy (scripts/taxonomy.py) Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
133 lines
6.8 KiB
TypeScript
133 lines
6.8 KiB
TypeScript
import { db } from './db';
|
|
import { Conference, PAGE_SIZE, Paper, PaperQuery, TaxNode } from './types';
|
|
|
|
export function getConferences(): Conference[] {
|
|
return db()
|
|
.prepare(
|
|
`SELECT c.*, (SELECT COUNT(*) FROM papers p WHERE p.conf = c.slug) AS paper_count
|
|
FROM conferences c ORDER BY COALESCE(c.start_date, '9999'), c.short`
|
|
)
|
|
.all() as Conference[];
|
|
}
|
|
|
|
export function getConference(slug: string): Conference | null {
|
|
return (getConferences().find((c) => c.slug === slug) ?? null) as Conference | null;
|
|
}
|
|
|
|
/** Turn free text into a safe FTS5 prefix query: each token quoted, last one prefix-matched. */
|
|
function ftsQuery(q: string): string | null {
|
|
const toks = q.match(/[\p{L}\p{N}][\p{L}\p{N}\-]*/gu);
|
|
if (!toks) return null;
|
|
return toks.map((t, i) => `"${t.replace(/"/g, '')}"${i === toks.length - 1 ? '*' : ''}`).join(' ');
|
|
}
|
|
|
|
interface Where { sql: string; params: unknown[]; fts: string | null }
|
|
|
|
function buildWhere(q: Partial<PaperQuery>): Where {
|
|
const parts: string[] = [];
|
|
const params: unknown[] = [];
|
|
const fts = q.q ? ftsQuery(q.q) : null;
|
|
if (fts) { parts.push('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)'); params.push(fts); }
|
|
if (q.conf) { parts.push('p.conf = ?'); params.push(q.conf); }
|
|
if (q.track) { parts.push('p.track = ?'); params.push(q.track); }
|
|
if (q.llm) parts.push('p.is_llm = 1');
|
|
if (q.node) {
|
|
parts.push('p.id IN (SELECT paper_id FROM paper_taxonomy WHERE node_id = ? OR node_id LIKE ?)');
|
|
params.push(q.node, q.node + '/%');
|
|
}
|
|
return { sql: parts.length ? 'WHERE ' + parts.join(' AND ') : '', params, fts };
|
|
}
|
|
|
|
export function searchPapers(q: PaperQuery): { papers: (Paper & { topics: string[] })[]; total: number } {
|
|
const w = buildWhere(q);
|
|
const total = (db().prepare(`SELECT COUNT(*) AS n FROM papers p ${w.sql}`).get(...w.params) as { n: number }).n;
|
|
const ranked = q.sort === 'rank' && !!w.fts;
|
|
const order = q.sort === 'title' ? 'p.title COLLATE NOCASE' : ranked ? 'f.r' : 'p.conf, p.id';
|
|
const join = ranked
|
|
? 'JOIN (SELECT rowid, bm25(papers_fts) AS r FROM papers_fts WHERE papers_fts MATCH ?) f ON f.rowid = p.id'
|
|
: '';
|
|
const sql = `SELECT p.*, c.short AS conf_short FROM papers p JOIN conferences c ON c.slug = p.conf ${join} ${w.sql} ORDER BY ${order} LIMIT ? OFFSET ?`;
|
|
const params = ranked ? [w.fts, ...w.params] : w.params;
|
|
const papers = db().prepare(sql).all(...params, PAGE_SIZE, (q.page - 1) * PAGE_SIZE) as Paper[];
|
|
const topicStmt = db().prepare(
|
|
`SELECT t.label FROM paper_taxonomy pt JOIN taxonomy t ON t.id = pt.node_id WHERE pt.paper_id = ? ORDER BY pt.is_primary DESC, pt.score DESC`
|
|
);
|
|
return {
|
|
total,
|
|
papers: papers.map((p) => ({
|
|
...p,
|
|
abstract: p.abstract ? p.abstract.slice(0, 420) : null,
|
|
topics: (topicStmt.all(p.id) as { label: string }[]).map((r) => r.label),
|
|
})),
|
|
};
|
|
}
|
|
|
|
export function getPaper(id: number): (Paper & { topics: { id: string; label: string }[] }) | null {
|
|
const p = db().prepare(`SELECT p.*, c.short AS conf_short FROM papers p JOIN conferences c ON c.slug = p.conf WHERE p.id = ?`).get(id) as Paper | undefined;
|
|
if (!p) return null;
|
|
const topics = db()
|
|
.prepare(`SELECT t.id, t.label FROM paper_taxonomy pt JOIN taxonomy t ON t.id = pt.node_id WHERE pt.paper_id = ? ORDER BY pt.is_primary DESC`)
|
|
.all(id) as { id: string; label: string }[];
|
|
return { ...p, topics };
|
|
}
|
|
|
|
export function getTracks(conf: string): string[] {
|
|
const sql = conf
|
|
? `SELECT DISTINCT track FROM papers WHERE conf = ? ORDER BY track`
|
|
: `SELECT DISTINCT track FROM papers ORDER BY track`;
|
|
const rows = (conf ? db().prepare(sql).all(conf) : db().prepare(sql).all()) as { track: string }[];
|
|
return rows.map((r) => r.track).filter(Boolean);
|
|
}
|
|
|
|
/** Global taxonomy tree (spans all conferences) with distinct-paper counts per node. */
|
|
export function getTaxonomyTree(opts: { llm?: boolean } = {}): TaxNode[] {
|
|
const nodes = db().prepare('SELECT id, parent, label, depth FROM taxonomy ORDER BY sort').all() as Omit<TaxNode, 'count' | 'children'>[];
|
|
const where = opts.llm ? 'WHERE p.is_llm = 1' : '';
|
|
const leafRows = db()
|
|
.prepare(`SELECT pt.node_id AS id, COUNT(DISTINCT pt.paper_id) AS n FROM paper_taxonomy pt JOIN papers p ON p.id = pt.paper_id ${where} GROUP BY pt.node_id`)
|
|
.all() as { id: string; n: number }[];
|
|
const topRows = db()
|
|
.prepare(`SELECT t.parent AS id, COUNT(DISTINCT pt.paper_id) AS n FROM paper_taxonomy pt JOIN taxonomy t ON t.id = pt.node_id JOIN papers p ON p.id = pt.paper_id ${where ? where + ' AND' : 'WHERE'} t.parent IS NOT NULL GROUP BY t.parent`)
|
|
.all() as { id: string; n: number }[];
|
|
const counts = new Map<string, number>([...leafRows, ...topRows].map((r) => [r.id, r.n]));
|
|
const byId = new Map<string, TaxNode>();
|
|
nodes.forEach((n) => byId.set(n.id, { ...n, count: counts.get(n.id) ?? 0, children: [] }));
|
|
const roots: TaxNode[] = [];
|
|
byId.forEach((n) => (n.parent ? byId.get(n.parent)!.children.push(n) : roots.push(n)));
|
|
return roots;
|
|
}
|
|
|
|
/** node id -> { conference slug -> paper count }, for leaves and their parents. */
|
|
export function getNodeConfBreakdown(opts: { llm?: boolean } = {}): Record<string, Record<string, number>> {
|
|
const where = opts.llm ? 'WHERE p.is_llm = 1' : '';
|
|
const leaf = db()
|
|
.prepare(`SELECT pt.node_id AS id, p.conf AS conf, COUNT(DISTINCT pt.paper_id) AS n FROM paper_taxonomy pt JOIN papers p ON p.id = pt.paper_id ${where} GROUP BY pt.node_id, p.conf`)
|
|
.all() as { id: string; conf: string; n: number }[];
|
|
const top = db()
|
|
.prepare(`SELECT t.parent AS id, p.conf AS conf, COUNT(DISTINCT pt.paper_id) AS n FROM paper_taxonomy pt JOIN taxonomy t ON t.id = pt.node_id JOIN papers p ON p.id = pt.paper_id ${where ? where + ' AND' : 'WHERE'} t.parent IS NOT NULL GROUP BY t.parent, p.conf`)
|
|
.all() as { id: string; conf: string; n: number }[];
|
|
const out: Record<string, Record<string, number>> = {};
|
|
for (const r of [...leaf, ...top]) (out[r.id] ??= {})[r.conf] = r.n;
|
|
return out;
|
|
}
|
|
|
|
/** Conference-tag counts for the current query, ignoring the conference filter itself. */
|
|
export function getConfFacets(q: PaperQuery): Record<string, number> {
|
|
const w = buildWhere({ ...q, conf: '' });
|
|
const rows = db().prepare(`SELECT p.conf AS conf, COUNT(*) AS n FROM papers p ${w.sql} GROUP BY p.conf`).all(...w.params) as { conf: string; n: number }[];
|
|
return Object.fromEntries(rows.map((r) => [r.conf, r.n]));
|
|
}
|
|
|
|
export function getTaxonomyLabel(id: string): string | null {
|
|
const r = db().prepare('SELECT label FROM taxonomy WHERE id = ?').get(id) as { label: string } | undefined;
|
|
return r?.label ?? null;
|
|
}
|
|
|
|
export function getStats() {
|
|
const d = db();
|
|
return {
|
|
papers: (d.prepare('SELECT COUNT(*) AS n FROM papers').get() as { n: number }).n,
|
|
llm: (d.prepare('SELECT COUNT(*) AS n FROM papers WHERE is_llm = 1').get() as { n: number }).n,
|
|
};
|
|
}
|