#!/usr/bin/env python3 """Build data/papers.db (SQLite + FTS5) from raw accepted-paper lists. usage: python3 scripts/ingest.py [--raw data/raw] [--out data/papers.db] """ import argparse, json, os, re, sqlite3, sys, xml.etree.ElementTree as ET sys.path.insert(0, os.path.dirname(__file__)) from taxonomy import TREE CONFERENCES = [ # slug, name, short, year, kind, start, end, location, url, papers_source ("aaai2026", "AAAI Conference on Artificial Intelligence", "AAAI", 2026, "ML", "2026-01-20", "2026-01-27", "Singapore", "https://aaai.org/conference/aaai/aaai-26/", None), ("eacl2026", "EACL 2026 (19th Conf. of the European Chapter of the ACL)", "EACL", 2026, "NLP", "2026-03-24", "2026-03-29", "Rabat, Morocco", "https://2026.eacl.org/", "ACL Anthology"), ("iclr2026", "ICLR 2026 (14th Intl. Conf. on Learning Representations)", "ICLR", 2026, "ML", "2026-04-23", "2026-04-27", "Rio de Janeiro, Brazil", "https://iclr.cc/Conferences/2026", "OpenReview via papercopilot/paperlists"), ("lrec2026", "LREC 2026 (15th Language Resources and Evaluation Conf.)", "LREC", 2026, "NLP", "2026-05-11", "2026-05-16", "Palma, Mallorca, Spain", "https://lrec2026.info/", "ACL Anthology"), ("acl2026", "ACL 2026 (64th Annual Meeting of the ACL)", "ACL", 2026, "NLP", "2026-07-02", "2026-07-07", "San Diego, USA", "https://2026.aclweb.org/", "ACL Anthology"), ("icml2026", "ICML 2026 (43rd Intl. Conf. on Machine Learning)", "ICML", 2026, "ML", "2026-07-06", "2026-07-12", "Seoul, South Korea", "https://icml.cc/", "OpenReview via papercopilot/paperlists"), ("conll2026", "CoNLL 2026 (30th Conf. on Computational Natural Language Learning)", "CoNLL", 2026, "NLP", None, None, None, "https://www.conll.org/", "ACL Anthology"), ("sigdial2026", "SIGDIAL 2026 (27th Meeting of the SIG on Discourse and Dialogue)", "SIGDIAL", 2026, "NLP", None, None, None, "https://www.sigdial.org/", "ACL Anthology"), ("tacl2026", "TACL 2026 (Transactions of the ACL)", "TACL", 2026, "NLP", None, None, "Journal", "https://transacl.org/", "ACL Anthology"), ("cl2026", "Computational Linguistics (Vol. 52)", "CL", 2026, "NLP", None, None, "Journal", "https://direct.mit.edu/coli", "ACL Anthology"), ("colm2026", "COLM 2026 (Conference on Language Modeling)", "COLM", 2026, "ML", "2026-10-06", "2026-10-09", "San Francisco, USA", "https://colm.cc/", None), ("inlg2026", "INLG 2026 (19th Intl. Natural Language Generation Conf.)", "INLG", 2026, "NLP", "2026-10-17", "2026-10-21", "Utrecht, Netherlands", None, None), ("emnlp2026", "EMNLP 2026", "EMNLP", 2026, "NLP", "2026-10-24", "2026-10-29", "Budapest, Hungary", "https://2026.emnlp.org/", None), ("aacl2026", "AACL-IJCNLP 2026", "AACL", 2026, "NLP", "2026-11-06", None, "Hengqin, Zhuhai, China", None, None), ("neurips2026", "NeurIPS 2026", "NeurIPS", 2026, "ML", "2026-12-06", "2026-12-12", "Sydney, Australia", "https://neurips.cc/", None), ("paclic2026", "PACLIC 40", "PACLIC", 2026, "NLP", "2026-12-10", "2026-12-12", "Taipei, Taiwan", None, None), ] SCHEMA = """ PRAGMA journal_mode=OFF; CREATE TABLE conferences(slug TEXT PRIMARY KEY, name TEXT, short TEXT, year INT, kind TEXT, start_date TEXT, end_date TEXT, location TEXT, url TEXT, papers_source TEXT); CREATE TABLE papers(id INTEGER PRIMARY KEY, conf TEXT NOT NULL REFERENCES conferences(slug), track TEXT, award TEXT, ext_id TEXT, title TEXT NOT NULL, abstract TEXT, authors TEXT, keywords TEXT, tldr TEXT, url TEXT, pdf_url TEXT, primary_area TEXT, is_llm INT DEFAULT 0, UNIQUE(conf, ext_id)); CREATE TABLE taxonomy(id TEXT PRIMARY KEY, parent TEXT, label TEXT, depth INT, sort INT); CREATE TABLE paper_taxonomy(paper_id INT NOT NULL REFERENCES papers(id), node_id TEXT NOT NULL REFERENCES taxonomy(id), score REAL, is_primary INT, PRIMARY KEY(paper_id, node_id)); CREATE INDEX idx_papers_conf ON papers(conf); CREATE INDEX idx_pt_node ON paper_taxonomy(node_id); """ LLM_RE = re.compile(r"\b(llms?|large language models?|language models?|lms|gpt-?\d*|chatgpt|foundation models?|instruction[- ]tun\w*|in-context|mllms?|vlms?|rlhf|dpo|grpo|llama|qwen|deepseek|gemini|claude|pre-?trained (language )?models?|transformers?|chain[- ]of[- ]thought|prompting|agentic|llm-based|bert|t5)\b", re.I) # compile leaf patterns LEAVES = [] for top_id, top_label, kids in TREE: for lid, label, pats in kids: LEAVES.append((lid, [re.compile(p, re.I) for p in pats])) def classify(title, keywords, abstract): scores = {} for lid, pats in LEAVES: s = 0.0 for p in pats: if p.search(title): s += 3 if keywords and p.search(keywords): s += 2 if abstract: n = len(p.findall(abstract)) if n: s += min(n, 2) if s: scores[lid] = s return sorted(scores.items(), key=lambda kv: -kv[1]) def txt(el): return re.sub(r"\s+", " ", "".join(el.itertext())).strip() if el is not None else "" def load_openreview(path, conf, accepted): with open(path) as f: data = json.load(f) for p in data: st = p.get("status", "") if not any(st.startswith(a) or a in st for a in accepted): continue award = "Oral" if "Oral" in st else ("Spotlight" if "Spotlight" in st else "Poster") fid = p.get("id") yield dict(conf=conf, track=p.get("track") or "main", award=award, ext_id=fid, title=p["title"].strip(), abstract=(p.get("abstract") or "").strip(), authors=(p.get("author") or "").replace(";", ", "), keywords=(p.get("keywords") or "").replace(";", ", "), tldr=p.get("tldr") or "", url=f"https://openreview.net/forum?id={fid}", pdf_url=f"https://openreview.net/pdf?id={fid}", primary_area=(p.get("primary_area") or "").replace("->", " > ")) ANTH = [ # (file, conf, {volume: track} or None for all volumes) ("2026.acl.xml", "acl2026", {"long": "Long", "short": "Short", "demo": "Demo", "srw": "Student Research", "industry": "Industry"}), ("2026.findings.xml", "acl2026", {"acl": "Findings"}), ("2026.findings.xml", "eacl2026", {"eacl": "Findings"}), ("2026.eacl.xml", "eacl2026", {"long": "Long", "short": "Short", "demo": "Demo", "srw": "Student Research", "industry": "Industry"}), ("2026.lrec.xml", "lrec2026", {"1": "Main"}), ("2026.conll.xml", "conll2026", {"main": "Main"}), ("2026.sigdial.xml", "sigdial2026", {"1": "Main"}), ("2026.tacl.xml", "tacl2026", {"1": "Journal"}), ("2026.cl.xml", "cl2026", {"1": "Journal", "2": "Journal"}), ] def load_anthology(xmldir): seen = set() for fname, conf, vols in ANTH: root = ET.parse(os.path.join(xmldir, fname)).getroot() cid = root.get("id") for v in root.findall("volume"): vid = v.get("id") if vid not in vols: continue for p in v.findall("paper"): aid = f"{cid}-{vid}.{p.get('id')}" if (conf, aid) in seen: continue seen.add((conf, aid)) authors = ", ".join(f"{txt(a.find('first'))} {txt(a.find('last'))}".strip() for a in p.findall("author")) yield dict(conf=conf, track=vols[vid], award=None, ext_id=aid, title=txt(p.find("title")), abstract=txt(p.find("abstract")), authors=authors, keywords="", tldr="", url=f"https://aclanthology.org/{aid}/", pdf_url=f"https://aclanthology.org/{aid}.pdf", primary_area="") def main(): ap = argparse.ArgumentParser() ap.add_argument("--raw", default="data/raw"); ap.add_argument("--out", default="data/papers.db") a = ap.parse_args() if os.path.exists(a.out): os.remove(a.out) db = sqlite3.connect(a.out); db.executescript(SCHEMA) db.executemany("INSERT INTO conferences VALUES(?,?,?,?,?,?,?,?,?,?)", CONFERENCES) # taxonomy rows, sort = [], 0 for tid, tlabel, kids in TREE: rows.append((tid, None, tlabel, 0, sort)); sort += 1 for lid, label, _ in kids: rows.append((lid, tid, label, 1, sort)); sort += 1 db.executemany("INSERT INTO taxonomy VALUES(?,?,?,?,?)", rows) papers = [] papers += load_openreview(os.path.join(a.raw, "iclr2026.json"), "iclr2026", ["Poster", "Oral", "Spotlight"]) papers += load_openreview(os.path.join(a.raw, "icml2026.json"), "icml2026", ["Poster", "Oral", "Spotlight"]) papers += load_anthology(os.path.join(a.raw, "acl-anthology", "data", "xml")) kind = {c[0]: c[4] for c in CONFERENCES} n_cls = 0 for p in papers: if not p["title"]: continue text = f"{p['title']} {p['keywords']} {p['abstract']}" p["is_llm"] = 1 if (kind[p["conf"]] == "NLP" or LLM_RE.search(text)) and (LLM_RE.search(text) or kind[p["conf"]] == "NLP") else 0 cur = db.execute("INSERT INTO papers(conf,track,award,ext_id,title,abstract,authors,keywords,tldr,url,pdf_url,primary_area,is_llm) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)", (p["conf"], p["track"], p["award"], p["ext_id"], p["title"], p["abstract"], p["authors"], p["keywords"], p["tldr"], p["url"], p["pdf_url"], p["primary_area"], p["is_llm"])) pid = cur.lastrowid if kind[p["conf"]] == "ML" and not LLM_RE.search(text): nodes = [("other/ml", 1.0, 1)] else: ranked = classify(p["title"], p["keywords"], p["abstract"]) if not ranked or ranked[0][1] < 2: nodes = [("other/unclassified", 0.0, 1)] else: top = ranked[0][1] nodes = [(ranked[0][0], top, 1)] for lid, s in ranked[1:3]: if s >= max(3, 0.6 * top): nodes.append((lid, s, 0)) n_cls += 1 db.executemany("INSERT INTO paper_taxonomy VALUES(?,?,?,?)", [(pid, n, s, pr) for n, s, pr in nodes]) db.execute("CREATE VIRTUAL TABLE papers_fts USING fts5(title, abstract, keywords, authors, content='papers', content_rowid='id', tokenize='porter unicode61')") db.execute("INSERT INTO papers_fts(papers_fts) VALUES('rebuild')") db.commit() print(f"papers: {len(papers)} classified into a topic: {n_cls}") for r in db.execute("SELECT conf, COUNT(*) FROM papers GROUP BY conf ORDER BY 2 DESC"): print(" ", *r) db.execute("VACUUM"); db.close() if __name__ == "__main__": main()