Add provenance-aware paper metadata storage and PMLR collection

schema.sql

@@ -0,0 +1,92 @@
+PRAGMA foreign_keys=ON;
+PRAGMA journal_mode=WAL;
+CREATE TABLE IF NOT EXISTS sources (
+ id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, homepage TEXT NOT NULL
+);
+CREATE TABLE IF NOT EXISTS papers (
+ id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, abstract TEXT,
+ title_key TEXT NOT NULL, authors_json TEXT NOT NULL DEFAULT '[]',
+ first_published TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE INDEX IF NOT EXISTS papers_title_key ON papers(title_key);
+CREATE TABLE IF NOT EXISTS identifiers (
+ scheme TEXT NOT NULL, value TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id),
+ PRIMARY KEY(scheme,value)
+);
+CREATE INDEX IF NOT EXISTS identifiers_paper ON identifiers(paper_id);
+CREATE TABLE IF NOT EXISTS source_records (
+ id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id),
+ external_id TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id),
+ url TEXT NOT NULL, title TEXT NOT NULL, abstract TEXT, authors_json TEXT NOT NULL,
+ metadata_json TEXT NOT NULL, retrieved_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ UNIQUE(source_id,external_id)
+);
+CREATE INDEX IF NOT EXISTS source_records_paper ON source_records(paper_id);
+CREATE TABLE IF NOT EXISTS venues (
+ id INTEGER PRIMARY KEY, acronym TEXT NOT NULL UNIQUE, name TEXT NOT NULL,
+ field TEXT NOT NULL, venue_type TEXT NOT NULL CHECK(venue_type IN ('conference','journal','review_platform'))
+);
+CREATE TABLE IF NOT EXISTS editions (
+ id INTEGER PRIMARY KEY, venue_id INTEGER NOT NULL REFERENCES venues(id),
+ year INTEGER NOT NULL, track TEXT NOT NULL DEFAULT 'main',
+ UNIQUE(venue_id,year,track)
+);
+CREATE TABLE IF NOT EXISTS edition_sources (
+ edition_id INTEGER NOT NULL REFERENCES editions(id), source_id INTEGER NOT NULL REFERENCES sources(id),
+ list_url TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending'
+ CHECK(status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')),
+ expected_count INTEGER, imported_count INTEGER NOT NULL DEFAULT 0,
+ cursor TEXT, last_success_at TEXT, last_error TEXT,
+ PRIMARY KEY(edition_id,source_id,list_url)
+);
+CREATE TABLE IF NOT EXISTS submissions (
+ id INTEGER PRIMARY KEY, paper_id INTEGER NOT NULL REFERENCES papers(id),
+ edition_id INTEGER NOT NULL REFERENCES editions(id),
+ source_record_id INTEGER NOT NULL REFERENCES source_records(id),
+ source_submission_id TEXT NOT NULL, submitted_at TEXT,
+ UNIQUE(source_record_id,edition_id,source_submission_id)
+);
+CREATE INDEX IF NOT EXISTS submissions_paper ON submissions(paper_id);
+CREATE TABLE IF NOT EXISTS decisions (
+ id INTEGER PRIMARY KEY, submission_id INTEGER NOT NULL REFERENCES submissions(id),
+ status TEXT NOT NULL CHECK(status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')),
+ raw_value TEXT NOT NULL, evidence_url TEXT NOT NULL, observed_at TEXT NOT NULL,
+ decision_date TEXT, UNIQUE(submission_id,status,raw_value,evidence_url,observed_at)
+);
+CREATE TABLE IF NOT EXISTS tags (
+ id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, value TEXT NOT NULL,
+ description TEXT, UNIQUE(namespace,value)
+);
+CREATE TABLE IF NOT EXISTS paper_tags (
+ paper_id INTEGER NOT NULL REFERENCES papers(id), tag_id INTEGER NOT NULL REFERENCES tags(id),
+ source_record_id INTEGER REFERENCES source_records(id),
+ evidence_key TEXT NOT NULL, PRIMARY KEY(paper_id,tag_id,evidence_key)
+);
+CREATE TABLE IF NOT EXISTS match_candidates (
+ left_paper_id INTEGER NOT NULL REFERENCES papers(id), right_paper_id INTEGER NOT NULL REFERENCES papers(id),
+ method TEXT NOT NULL, score REAL, evidence_json TEXT NOT NULL,
+ status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','merged','distinct')),
+ PRIMARY KEY(left_paper_id,right_paper_id,method)
+);
+CREATE TABLE IF NOT EXISTS paper_merges (
+ id INTEGER PRIMARY KEY, former_paper_id INTEGER NOT NULL,
+ surviving_paper_id INTEGER NOT NULL REFERENCES papers(id),
+ reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+CREATE TABLE IF NOT EXISTS ingest_runs (
+ id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id),
+ started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT,
+ status TEXT NOT NULL DEFAULT 'running', records_seen INTEGER NOT NULL DEFAULT 0,
+ records_written INTEGER NOT NULL DEFAULT 0, error TEXT, checkpoint_json TEXT
+);
+CREATE VIRTUAL TABLE IF NOT EXISTS papers_fts USING fts5(title,abstract,content='papers',content_rowid='id');
+CREATE TRIGGER IF NOT EXISTS papers_insert AFTER INSERT ON papers BEGIN
+ INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract);
+END;
+CREATE TRIGGER IF NOT EXISTS papers_delete AFTER DELETE ON papers BEGIN
+ INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract);
+END;
+CREATE TRIGGER IF NOT EXISTS papers_update AFTER UPDATE OF title,abstract ON papers BEGIN
+ INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract);
+ INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract);
+END;