#!/usr/bin/env node // Upsert analysis articles (content/articles/*.json) into the persistent "analysis" schema in PostgreSQL. // Idempotent: an article is replaced as a whole (sections, blocks, links) inside one transaction. // usage: DATABASE_URL=postgres://... node scripts/load-articles.js [content/articles] // // Block shapes (data column): // markdown {text} paragraphs, ###/#### headings, - and 1. lists, **bold**, *italic*, `code`, [text](url) // list {items[], ordered?} items are inline markdown // callout {text, title?, tone} tone: info | note | warn // table {columns[], rows[][], caption?, numeric?, highlightCol?} cells are inline markdown // chart {kind:'bar', title, labels[], series:[{name, values[]}], unit?, note?, highlight?[]} // mermaid {code, caption?} Mermaid.js diagram source // figure {src, alt, caption?} image (https:// or /path) // code {text, lang?} const fs = require('fs'); const path = require('path'); const { Client } = require('pg'); const TYPES = ['markdown', 'list', 'callout', 'table', 'chart', 'mermaid', 'figure', 'code']; const KINDS = ['paper', 'code', 'project', 'demo', 'dataset', 'related', 'other']; const dir = process.argv[2] || 'content/articles'; function fail(file, msg) { throw new Error(`${file}: ${msg}`); } function validate(file, a) { for (const k of ['slug', 'title']) if (!a[k] || typeof a[k] !== 'string') fail(file, `missing ${k}`); if (!a.subject || !a.subject.conf || !a.subject.ext_id) fail(file, 'missing subject.conf / subject.ext_id'); if (!Array.isArray(a.sections) || !a.sections.length) fail(file, 'no sections'); const keys = new Set(); const walk = (secs) => { for (const s of secs) { if (!s.key || !s.title) fail(file, 'section needs key and title'); if (keys.has(s.key)) fail(file, `duplicate section key ${s.key}`); keys.add(s.key); (s.blocks || []).forEach((b, i) => { if (!TYPES.includes(b.type)) fail(file, `${s.key}[${i}]: unknown block type ${b.type}`); const d = b.data || {}; if (b.type === 'markdown' && typeof d.text !== 'string') fail(file, `${s.key}[${i}]: markdown needs text`); if (b.type === 'list' && !Array.isArray(d.items)) fail(file, `${s.key}[${i}]: list needs items`); if (b.type === 'mermaid' && typeof d.code !== 'string') fail(file, `${s.key}[${i}]: mermaid needs code`); if (b.type === 'table' && !(Array.isArray(d.columns) && Array.isArray(d.rows) && d.rows.every((r) => r.length === d.columns.length))) fail(file, `${s.key}[${i}]: table rows must match columns`); if (b.type === 'chart' && !(Array.isArray(d.labels) && Array.isArray(d.series) && d.series.every((x) => x.values.length === d.labels.length))) fail(file, `${s.key}[${i}]: chart series must match labels`); }); walk(s.children || []); } }; walk(a.sections); for (const l of a.links || []) if (!KINDS.includes(l.kind) || !l.url || !l.title) fail(file, `bad link ${JSON.stringify(l)}`); } async function load(pg, a) { await pg.query('BEGIN'); try { const ex = await pg.query('SELECT id, revision FROM analysis.article WHERE slug = $1', [a.slug]); let id; if (ex.rows.length) { id = ex.rows[0].id; await pg.query('DELETE FROM analysis.section WHERE article_id = $1', [id]); // cascades to blocks; links go below await pg.query('DELETE FROM analysis.link WHERE article_id = $1', [id]); await pg.query( 'UPDATE analysis.article SET subject_conf=$2, subject_ext_id=$3, title=$4, summary=$5, revision=revision+1, updated_at=now() WHERE id=$1', [id, a.subject.conf, a.subject.ext_id, a.title, a.summary ?? null]); } else { id = (await pg.query( 'INSERT INTO analysis.article(slug, subject_conf, subject_ext_id, title, summary) VALUES ($1,$2,$3,$4,$5) RETURNING id', [a.slug, a.subject.conf, a.subject.ext_id, a.title, a.summary ?? null])).rows[0].id; } const secIds = {}; const insertSecs = async (secs, parent) => { for (let i = 0; i < secs.length; i++) { const s = secs[i]; const sid = (await pg.query( 'INSERT INTO analysis.section(article_id, parent_id, key, title, level, position) VALUES ($1,$2,$3,$4,$5,$6) RETURNING id', [id, parent, s.key, s.title, s.level ?? (parent ? 3 : 2), i])).rows[0].id; secIds[s.key] = sid; for (let j = 0; j < (s.blocks || []).length; j++) await pg.query('INSERT INTO analysis.block(section_id, position, type, data) VALUES ($1,$2,$3,$4)', [sid, j, s.blocks[j].type, JSON.stringify(s.blocks[j].data)]); await insertSecs(s.children || [], sid); } }; await insertSecs(a.sections, null); const links = a.links || []; for (let i = 0; i < links.length; i++) { const l = links[i]; await pg.query('INSERT INTO analysis.link(article_id, section_id, position, kind, title, url, note) VALUES ($1,$2,$3,$4,$5,$6,$7)', [id, l.section ? secIds[l.section] ?? null : null, i, l.kind, l.title, l.url, l.note ?? null]); } await pg.query('COMMIT'); return { id, sections: Object.keys(secIds).length, blocks: a.sections.reduce(function c(n, s) { return n + (s.blocks || []).length + (s.children || []).reduce(c, 0); }, 0), links: links.length }; } catch (e) { await pg.query('ROLLBACK'); throw e; } } (async () => { if (!process.env.DATABASE_URL) { console.error('DATABASE_URL is not set'); process.exit(1); } const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.json')).sort() : []; const articles = files.map((f) => { const a = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')); validate(f, a); return [f, a]; }); const pg = new Client({ connectionString: process.env.DATABASE_URL }); await pg.connect(); try { await pg.query(fs.readFileSync(path.join(__dirname, '..', 'db', 'analysis-schema.sql'), 'utf8')); for (const [f, a] of articles) { const r = await load(pg, a); console.log(`article ${a.slug} <- ${f}: ${r.sections} sections, ${r.blocks} blocks, ${r.links} links`); } const n = (await pg.query('SELECT COUNT(*)::int AS n FROM analysis.article')).rows[0].n; console.log(`analysis.article rows: ${n}`); } catch (e) { console.error('load-articles failed:', e.message); process.exitCode = 1; } finally { await pg.end(); } })();