Files
ai-taxonomy-literature-anal…/scripts/load-pg.js
T
2026-10-22 15:38:57 +00:00

83 lines
5.3 KiB
JavaScript

#!/usr/bin/env node
// Load the SQLite staging database (built by ingest.py + build_worktypes.py) into PostgreSQL.
// Builds schema "next", verifies row counts, then swaps it in as schema "live" in one transaction.
// usage: DATABASE_URL=postgres://... node scripts/load-pg.js --sqlite build/papers.db
const { Client } = require('pg');
const Database = require('better-sqlite3');
const arg = (n, d) => { const i = process.argv.indexOf('--' + n); return i > 0 ? process.argv[i + 1] : d; };
const sqliteFile = arg('sqlite', 'build/papers.db');
if (!process.env.DATABASE_URL) { console.error('DATABASE_URL is not set'); process.exit(1); }
const DDL = `
CREATE TABLE conferences(slug text PRIMARY KEY, name text, short text, year integer, kind text, start_date text, end_date text, location text, url text, papers_source text);
CREATE TABLE papers(id integer PRIMARY KEY, conf text NOT NULL REFERENCES conferences(slug), track text, award text, ext_id text, title text NOT NULL, abstract text, authors text, keywords text, tldr text, url text, pdf_url text, primary_area text, is_llm integer DEFAULT 0, search tsvector, UNIQUE(conf, ext_id));
CREATE TABLE taxonomy(id text PRIMARY KEY, parent text, label text, depth integer, sort integer);
CREATE TABLE paper_taxonomy(paper_id integer NOT NULL REFERENCES papers(id), node_id text NOT NULL REFERENCES taxonomy(id), score double precision, is_primary integer, PRIMARY KEY(paper_id, node_id));
CREATE TABLE areas(id text PRIMARY KEY, label text, kind text, sort integer);
CREATE TABLE roles(id text PRIMARY KEY, label text, descr text, fixes text, varies text, pattern text, sort integer);
CREATE TABLE work_types(id text PRIMARY KEY, area text REFERENCES areas(id), role text REFERENCES roles(id), n_papers integer, n_llm_labeled integer);
CREATE TABLE benchmarks(id integer PRIMARY KEY, key text UNIQUE, name text, n_papers integer, n_new integer);
CREATE TABLE paper_benchmarks(paper_id integer, bench_id integer, purpose text, is_new integer, source text, PRIMARY KEY(paper_id, bench_id));
CREATE TABLE paper_meta(paper_id integer PRIMARY KEY, summary text, fixes text, varies text, workflow text, in_agent_scope integer, label_source text);
CREATE TABLE paper_work_types(paper_id integer NOT NULL, type_id text NOT NULL, why text, source text, PRIMARY KEY(paper_id, type_id));
CREATE TABLE results(id integer PRIMARY KEY, paper_id integer, bench_id integer, system text, base_model text, setting text, metric text, score double precision, unit text);
CREATE TABLE type_notes(type_id text PRIMARY KEY, why_same text, workflow text, bench_summary text, narrative text);
CREATE TABLE type_reviews(type_id text PRIMARY KEY, json text);
`;
const INDEXES = `
CREATE INDEX idx_papers_conf ON papers(conf);
CREATE INDEX idx_papers_search ON papers USING gin(search);
CREATE INDEX idx_pt_node ON paper_taxonomy(node_id);
CREATE INDEX idx_pb_bench ON paper_benchmarks(bench_id);
CREATE INDEX idx_pwt_type ON paper_work_types(type_id);
CREATE INDEX idx_res_bench ON results(bench_id);
`;
// parents first (foreign keys)
const TABLES = ['conferences', 'taxonomy', 'areas', 'roles', 'benchmarks', 'papers', 'paper_taxonomy', 'work_types', 'paper_benchmarks',
'paper_meta', 'paper_work_types', 'results', 'type_notes', 'type_reviews'];
(async () => {
const lite = new Database(sqliteFile, { readonly: true, fileMustExist: true });
const pg = new Client({ connectionString: process.env.DATABASE_URL });
await pg.connect();
try {
await pg.query('DROP SCHEMA IF EXISTS next CASCADE; CREATE SCHEMA next; SET search_path = next');
await pg.query(DDL);
for (const t of TABLES) {
const cols = lite.prepare(`PRAGMA table_info(${t})`).all().map((c) => c.name);
const rows = lite.prepare(`SELECT ${cols.map((c) => `"${c}"`).join(',')} FROM ${t}`).raw().all();
const per = Math.max(1, Math.floor(30000 / cols.length));
for (let i = 0; i < rows.length; i += per) {
const chunk = rows.slice(i, i + per);
const ph = chunk.map((_, r) => '(' + cols.map((_, c) => `$${r * cols.length + c + 1}`).join(',') + ')').join(',');
await pg.query(`INSERT INTO ${t} (${cols.map((c) => `"${c}"`).join(',')}) VALUES ${ph}`, chunk.flat());
}
const n = (await pg.query(`SELECT COUNT(*)::int AS n FROM ${t}`)).rows[0].n;
if (n !== rows.length) throw new Error(`${t}: loaded ${n} of ${rows.length} rows`);
console.log(`${t.padEnd(18)} ${n}`);
}
await pg.query(`UPDATE papers SET search =
setweight(to_tsvector('english', coalesce(title,'')), 'A') ||
setweight(to_tsvector('english', coalesce(keywords,'')), 'B') ||
setweight(to_tsvector('english', coalesce(abstract,'')), 'C') ||
setweight(to_tsvector('simple', coalesce(authors,'')), 'D')`);
await pg.query(INDEXES);
await pg.query('ANALYZE');
const papers = (await pg.query('SELECT COUNT(*)::int AS n FROM papers')).rows[0].n;
if (papers < 1000) throw new Error(`refusing to publish: only ${papers} papers`);
await pg.query('BEGIN');
await pg.query('DROP SCHEMA IF EXISTS live CASCADE');
await pg.query('ALTER SCHEMA next RENAME TO live');
await pg.query('COMMIT');
console.log(`published schema live (${papers} papers)`);
} catch (e) {
await pg.query('ROLLBACK').catch(() => {});
console.error('load failed:', e.message);
process.exitCode = 1;
} finally {
await pg.end();
lite.close();
}
})();