Files
ai-taxonomy-literature-anal…/scripts/load-articles.js
T

112 lines
6.3 KiB
JavaScript

#!/usr/bin/env node
// Upsert analysis articles (content/articles/*.json) into the persistent "analysis" schema in PostgreSQL.
// Idempotent: an article is replaced as a whole (sections, blocks, links) inside one transaction.
// usage: DATABASE_URL=postgres://... node scripts/load-articles.js [content/articles]
//
// Block shapes (data column):
// markdown {text} paragraphs, ###/#### headings, - and 1. lists, **bold**, *italic*, `code`, [text](url)
// list {items[], ordered?} items are inline markdown
// callout {text, title?, tone} tone: info | note | warn
// table {columns[], rows[][], caption?, numeric?, highlightCol?} cells are inline markdown
// chart {kind:'bar', title, labels[], series:[{name, values[]}], unit?, note?, highlight?[]}
// mermaid {code, caption?} Mermaid.js diagram source
// figure {src, alt, caption?} image (https:// or /path)
// code {text, lang?}
const fs = require('fs');
const path = require('path');
const { Client } = require('pg');
const TYPES = ['markdown', 'list', 'callout', 'table', 'chart', 'mermaid', 'figure', 'code'];
const KINDS = ['paper', 'code', 'project', 'demo', 'dataset', 'related', 'other'];
const dir = process.argv[2] || 'content/articles';
function fail(file, msg) { throw new Error(`${file}: ${msg}`); }
function validate(file, a) {
for (const k of ['slug', 'title']) if (!a[k] || typeof a[k] !== 'string') fail(file, `missing ${k}`);
if (!a.subject || !a.subject.conf || !a.subject.ext_id) fail(file, 'missing subject.conf / subject.ext_id');
if (!Array.isArray(a.sections) || !a.sections.length) fail(file, 'no sections');
const keys = new Set();
const walk = (secs) => {
for (const s of secs) {
if (!s.key || !s.title) fail(file, 'section needs key and title');
if (keys.has(s.key)) fail(file, `duplicate section key ${s.key}`);
keys.add(s.key);
(s.blocks || []).forEach((b, i) => {
if (!TYPES.includes(b.type)) fail(file, `${s.key}[${i}]: unknown block type ${b.type}`);
const d = b.data || {};
if (b.type === 'markdown' && typeof d.text !== 'string') fail(file, `${s.key}[${i}]: markdown needs text`);
if (b.type === 'list' && !Array.isArray(d.items)) fail(file, `${s.key}[${i}]: list needs items`);
if (b.type === 'mermaid' && typeof d.code !== 'string') fail(file, `${s.key}[${i}]: mermaid needs code`);
if (b.type === 'table' && !(Array.isArray(d.columns) && Array.isArray(d.rows) && d.rows.every((r) => r.length === d.columns.length)))
fail(file, `${s.key}[${i}]: table rows must match columns`);
if (b.type === 'chart' && !(Array.isArray(d.labels) && Array.isArray(d.series) && d.series.every((x) => x.values.length === d.labels.length)))
fail(file, `${s.key}[${i}]: chart series must match labels`);
});
walk(s.children || []);
}
};
walk(a.sections);
for (const l of a.links || []) if (!KINDS.includes(l.kind) || !l.url || !l.title) fail(file, `bad link ${JSON.stringify(l)}`);
}
async function load(pg, a) {
await pg.query('BEGIN');
try {
const ex = await pg.query('SELECT id, revision FROM analysis.article WHERE slug = $1', [a.slug]);
let id;
if (ex.rows.length) {
id = ex.rows[0].id;
await pg.query('DELETE FROM analysis.section WHERE article_id = $1', [id]); // cascades to blocks; links go below
await pg.query('DELETE FROM analysis.link WHERE article_id = $1', [id]);
await pg.query(
'UPDATE analysis.article SET subject_conf=$2, subject_ext_id=$3, title=$4, summary=$5, revision=revision+1, updated_at=now() WHERE id=$1',
[id, a.subject.conf, a.subject.ext_id, a.title, a.summary ?? null]);
} else {
id = (await pg.query(
'INSERT INTO analysis.article(slug, subject_conf, subject_ext_id, title, summary) VALUES ($1,$2,$3,$4,$5) RETURNING id',
[a.slug, a.subject.conf, a.subject.ext_id, a.title, a.summary ?? null])).rows[0].id;
}
const secIds = {};
const insertSecs = async (secs, parent) => {
for (let i = 0; i < secs.length; i++) {
const s = secs[i];
const sid = (await pg.query(
'INSERT INTO analysis.section(article_id, parent_id, key, title, level, position) VALUES ($1,$2,$3,$4,$5,$6) RETURNING id',
[id, parent, s.key, s.title, s.level ?? (parent ? 3 : 2), i])).rows[0].id;
secIds[s.key] = sid;
for (let j = 0; j < (s.blocks || []).length; j++)
await pg.query('INSERT INTO analysis.block(section_id, position, type, data) VALUES ($1,$2,$3,$4)', [sid, j, s.blocks[j].type, JSON.stringify(s.blocks[j].data)]);
await insertSecs(s.children || [], sid);
}
};
await insertSecs(a.sections, null);
const links = a.links || [];
for (let i = 0; i < links.length; i++) {
const l = links[i];
await pg.query('INSERT INTO analysis.link(article_id, section_id, position, kind, title, url, note) VALUES ($1,$2,$3,$4,$5,$6,$7)',
[id, l.section ? secIds[l.section] ?? null : null, i, l.kind, l.title, l.url, l.note ?? null]);
}
await pg.query('COMMIT');
return { id, sections: Object.keys(secIds).length, blocks: a.sections.reduce(function c(n, s) { return n + (s.blocks || []).length + (s.children || []).reduce(c, 0); }, 0), links: links.length };
} catch (e) { await pg.query('ROLLBACK'); throw e; }
}
(async () => {
if (!process.env.DATABASE_URL) { console.error('DATABASE_URL is not set'); process.exit(1); }
const files = fs.existsSync(dir) ? fs.readdirSync(dir).filter((f) => f.endsWith('.json')).sort() : [];
const articles = files.map((f) => { const a = JSON.parse(fs.readFileSync(path.join(dir, f), 'utf8')); validate(f, a); return [f, a]; });
const pg = new Client({ connectionString: process.env.DATABASE_URL });
await pg.connect();
try {
await pg.query(fs.readFileSync(path.join(__dirname, '..', 'db', 'analysis-schema.sql'), 'utf8'));
for (const [f, a] of articles) {
const r = await load(pg, a);
console.log(`article ${a.slug} <- ${f}: ${r.sections} sections, ${r.blocks} blocks, ${r.links} links`);
}
const n = (await pg.query('SELECT COUNT(*)::int AS n FROM analysis.article')).rows[0].n;
console.log(`analysis.article rows: ${n}`);
} catch (e) { console.error('load-articles failed:', e.message); process.exitCode = 1; }
finally { await pg.end(); }
})();