34bca145e4
Deploy / deploy (push) Failing after 9m27s
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01FbMbJPadPzVwvt9rU9GBNo
83 lines
5.3 KiB
JavaScript
83 lines
5.3 KiB
JavaScript
#!/usr/bin/env node
|
|
// Load the SQLite staging database (built by ingest.py + build_worktypes.py) into PostgreSQL.
|
|
// Builds schema "next", verifies row counts, then swaps it in as schema "live" in one transaction.
|
|
// usage: DATABASE_URL=postgres://... node scripts/load-pg.js --sqlite build/papers.db
|
|
const { Client } = require('pg');
|
|
const Database = require('better-sqlite3');
|
|
|
|
const arg = (n, d) => { const i = process.argv.indexOf('--' + n); return i > 0 ? process.argv[i + 1] : d; };
|
|
const sqliteFile = arg('sqlite', 'build/papers.db');
|
|
if (!process.env.DATABASE_URL) { console.error('DATABASE_URL is not set'); process.exit(1); }
|
|
|
|
const DDL = `
|
|
CREATE TABLE conferences(slug text PRIMARY KEY, name text, short text, year integer, kind text, start_date text, end_date text, location text, url text, papers_source text);
|
|
CREATE TABLE papers(id integer PRIMARY KEY, conf text NOT NULL REFERENCES conferences(slug), track text, award text, ext_id text, title text NOT NULL, abstract text, authors text, keywords text, tldr text, url text, pdf_url text, primary_area text, is_llm integer DEFAULT 0, search tsvector, UNIQUE(conf, ext_id));
|
|
CREATE TABLE taxonomy(id text PRIMARY KEY, parent text, label text, depth integer, sort integer);
|
|
CREATE TABLE paper_taxonomy(paper_id integer NOT NULL REFERENCES papers(id), node_id text NOT NULL REFERENCES taxonomy(id), score double precision, is_primary integer, PRIMARY KEY(paper_id, node_id));
|
|
CREATE TABLE areas(id text PRIMARY KEY, label text, kind text, sort integer);
|
|
CREATE TABLE roles(id text PRIMARY KEY, label text, descr text, fixes text, varies text, pattern text, sort integer);
|
|
CREATE TABLE work_types(id text PRIMARY KEY, area text REFERENCES areas(id), role text REFERENCES roles(id), n_papers integer, n_llm_labeled integer);
|
|
CREATE TABLE benchmarks(id integer PRIMARY KEY, key text UNIQUE, name text, n_papers integer, n_new integer);
|
|
CREATE TABLE paper_benchmarks(paper_id integer, bench_id integer, purpose text, is_new integer, source text, PRIMARY KEY(paper_id, bench_id));
|
|
CREATE TABLE paper_meta(paper_id integer PRIMARY KEY, summary text, fixes text, varies text, workflow text, in_agent_scope integer, label_source text);
|
|
CREATE TABLE paper_work_types(paper_id integer NOT NULL, type_id text NOT NULL, why text, source text, PRIMARY KEY(paper_id, type_id));
|
|
CREATE TABLE results(id integer PRIMARY KEY, paper_id integer, bench_id integer, system text, base_model text, setting text, metric text, score double precision, unit text);
|
|
CREATE TABLE type_notes(type_id text PRIMARY KEY, why_same text, workflow text, bench_summary text, narrative text);
|
|
CREATE TABLE type_reviews(type_id text PRIMARY KEY, json text);
|
|
`;
|
|
const INDEXES = `
|
|
CREATE INDEX idx_papers_conf ON papers(conf);
|
|
CREATE INDEX idx_papers_search ON papers USING gin(search);
|
|
CREATE INDEX idx_pt_node ON paper_taxonomy(node_id);
|
|
CREATE INDEX idx_pb_bench ON paper_benchmarks(bench_id);
|
|
CREATE INDEX idx_pwt_type ON paper_work_types(type_id);
|
|
CREATE INDEX idx_res_bench ON results(bench_id);
|
|
`;
|
|
// parents first (foreign keys)
|
|
const TABLES = ['conferences', 'taxonomy', 'areas', 'roles', 'benchmarks', 'papers', 'paper_taxonomy', 'work_types', 'paper_benchmarks',
|
|
'paper_meta', 'paper_work_types', 'results', 'type_notes', 'type_reviews'];
|
|
|
|
(async () => {
|
|
const lite = new Database(sqliteFile, { readonly: true, fileMustExist: true });
|
|
const pg = new Client({ connectionString: process.env.DATABASE_URL });
|
|
await pg.connect();
|
|
try {
|
|
await pg.query('DROP SCHEMA IF EXISTS next CASCADE; CREATE SCHEMA next; SET search_path = next');
|
|
await pg.query(DDL);
|
|
for (const t of TABLES) {
|
|
const cols = lite.prepare(`PRAGMA table_info(${t})`).all().map((c) => c.name);
|
|
const rows = lite.prepare(`SELECT ${cols.map((c) => `"${c}"`).join(',')} FROM ${t}`).raw().all();
|
|
const per = Math.max(1, Math.floor(30000 / cols.length));
|
|
for (let i = 0; i < rows.length; i += per) {
|
|
const chunk = rows.slice(i, i + per);
|
|
const ph = chunk.map((_, r) => '(' + cols.map((_, c) => `$${r * cols.length + c + 1}`).join(',') + ')').join(',');
|
|
await pg.query(`INSERT INTO ${t} (${cols.map((c) => `"${c}"`).join(',')}) VALUES ${ph}`, chunk.flat());
|
|
}
|
|
const n = (await pg.query(`SELECT COUNT(*)::int AS n FROM ${t}`)).rows[0].n;
|
|
if (n !== rows.length) throw new Error(`${t}: loaded ${n} of ${rows.length} rows`);
|
|
console.log(`${t.padEnd(18)} ${n}`);
|
|
}
|
|
await pg.query(`UPDATE papers SET search =
|
|
setweight(to_tsvector('english', coalesce(title,'')), 'A') ||
|
|
setweight(to_tsvector('english', coalesce(keywords,'')), 'B') ||
|
|
setweight(to_tsvector('english', coalesce(abstract,'')), 'C') ||
|
|
setweight(to_tsvector('simple', coalesce(authors,'')), 'D')`);
|
|
await pg.query(INDEXES);
|
|
await pg.query('ANALYZE');
|
|
const papers = (await pg.query('SELECT COUNT(*)::int AS n FROM papers')).rows[0].n;
|
|
if (papers < 1000) throw new Error(`refusing to publish: only ${papers} papers`);
|
|
await pg.query('BEGIN');
|
|
await pg.query('DROP SCHEMA IF EXISTS live CASCADE');
|
|
await pg.query('ALTER SCHEMA next RENAME TO live');
|
|
await pg.query('COMMIT');
|
|
console.log(`published schema live (${papers} papers)`);
|
|
} catch (e) {
|
|
await pg.query('ROLLBACK').catch(() => {});
|
|
console.error('load failed:', e.message);
|
|
process.exitCode = 1;
|
|
} finally {
|
|
await pg.end();
|
|
lite.close();
|
|
}
|
|
})();
|