Files
ai-taxonomy-literature-anal…/scripts/ingest.py
T
zzyang 13924f8985 Add LLM research survey site (Next.js pages router + SQLite)
- Next.js/React/TypeScript app: overview, paper explorer with FTS5 search,
  paper detail, global taxonomy tree with per-conference tag breakdown
- Python ingestion (ICLR/ICML 2026 via paperlists, ACL Anthology 2026) into
  data/papers.db, plus keyword-scored taxonomy (scripts/taxonomy.py)

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bhs9LU8eMixk6g1kb1d3j7
2026-10-21 11:30:14 +00:00

157 lines
10 KiB
Python

#!/usr/bin/env python3
"""Build data/papers.db (SQLite + FTS5) from raw accepted-paper lists.
usage: python3 scripts/ingest.py [--raw data/raw] [--out data/papers.db]
"""
import argparse, json, os, re, sqlite3, sys, xml.etree.ElementTree as ET
sys.path.insert(0, os.path.dirname(__file__))
from taxonomy import TREE
CONFERENCES = [
# slug, name, short, year, kind, start, end, location, url, papers_source
("aaai2026", "AAAI Conference on Artificial Intelligence", "AAAI", 2026, "ML", "2026-01-20", "2026-01-27", "Singapore", "https://aaai.org/conference/aaai/aaai-26/", None),
("eacl2026", "EACL 2026 (19th Conf. of the European Chapter of the ACL)", "EACL", 2026, "NLP", "2026-03-24", "2026-03-29", "Rabat, Morocco", "https://2026.eacl.org/", "ACL Anthology"),
("iclr2026", "ICLR 2026 (14th Intl. Conf. on Learning Representations)", "ICLR", 2026, "ML", "2026-04-23", "2026-04-27", "Rio de Janeiro, Brazil", "https://iclr.cc/Conferences/2026", "OpenReview via papercopilot/paperlists"),
("lrec2026", "LREC 2026 (15th Language Resources and Evaluation Conf.)", "LREC", 2026, "NLP", "2026-05-11", "2026-05-16", "Palma, Mallorca, Spain", "https://lrec2026.info/", "ACL Anthology"),
("acl2026", "ACL 2026 (64th Annual Meeting of the ACL)", "ACL", 2026, "NLP", "2026-07-02", "2026-07-07", "San Diego, USA", "https://2026.aclweb.org/", "ACL Anthology"),
("icml2026", "ICML 2026 (43rd Intl. Conf. on Machine Learning)", "ICML", 2026, "ML", "2026-07-06", "2026-07-12", "Seoul, South Korea", "https://icml.cc/", "OpenReview via papercopilot/paperlists"),
("conll2026", "CoNLL 2026 (30th Conf. on Computational Natural Language Learning)", "CoNLL", 2026, "NLP", None, None, None, "https://www.conll.org/", "ACL Anthology"),
("sigdial2026", "SIGDIAL 2026 (27th Meeting of the SIG on Discourse and Dialogue)", "SIGDIAL", 2026, "NLP", None, None, None, "https://www.sigdial.org/", "ACL Anthology"),
("tacl2026", "TACL 2026 (Transactions of the ACL)", "TACL", 2026, "NLP", None, None, "Journal", "https://transacl.org/", "ACL Anthology"),
("cl2026", "Computational Linguistics (Vol. 52)", "CL", 2026, "NLP", None, None, "Journal", "https://direct.mit.edu/coli", "ACL Anthology"),
("colm2026", "COLM 2026 (Conference on Language Modeling)", "COLM", 2026, "ML", "2026-10-06", "2026-10-09", "San Francisco, USA", "https://colm.cc/", None),
("inlg2026", "INLG 2026 (19th Intl. Natural Language Generation Conf.)", "INLG", 2026, "NLP", "2026-10-17", "2026-10-21", "Utrecht, Netherlands", None, None),
("emnlp2026", "EMNLP 2026", "EMNLP", 2026, "NLP", "2026-10-24", "2026-10-29", "Budapest, Hungary", "https://2026.emnlp.org/", None),
("aacl2026", "AACL-IJCNLP 2026", "AACL", 2026, "NLP", "2026-11-06", None, "Hengqin, Zhuhai, China", None, None),
("neurips2026", "NeurIPS 2026", "NeurIPS", 2026, "ML", "2026-12-06", "2026-12-12", "Sydney, Australia", "https://neurips.cc/", None),
("paclic2026", "PACLIC 40", "PACLIC", 2026, "NLP", "2026-12-10", "2026-12-12", "Taipei, Taiwan", None, None),
]
SCHEMA = """
PRAGMA journal_mode=OFF;
CREATE TABLE conferences(slug TEXT PRIMARY KEY, name TEXT, short TEXT, year INT, kind TEXT, start_date TEXT, end_date TEXT, location TEXT, url TEXT, papers_source TEXT);
CREATE TABLE papers(id INTEGER PRIMARY KEY, conf TEXT NOT NULL REFERENCES conferences(slug), track TEXT, award TEXT, ext_id TEXT, title TEXT NOT NULL, abstract TEXT, authors TEXT, keywords TEXT, tldr TEXT, url TEXT, pdf_url TEXT, primary_area TEXT, is_llm INT DEFAULT 0, UNIQUE(conf, ext_id));
CREATE TABLE taxonomy(id TEXT PRIMARY KEY, parent TEXT, label TEXT, depth INT, sort INT);
CREATE TABLE paper_taxonomy(paper_id INT NOT NULL REFERENCES papers(id), node_id TEXT NOT NULL REFERENCES taxonomy(id), score REAL, is_primary INT, PRIMARY KEY(paper_id, node_id));
CREATE INDEX idx_papers_conf ON papers(conf);
CREATE INDEX idx_pt_node ON paper_taxonomy(node_id);
"""
LLM_RE = re.compile(r"\b(llms?|large language models?|language models?|lms|gpt-?\d*|chatgpt|foundation models?|instruction[- ]tun\w*|in-context|mllms?|vlms?|rlhf|dpo|grpo|llama|qwen|deepseek|gemini|claude|pre-?trained (language )?models?|transformers?|chain[- ]of[- ]thought|prompting|agentic|llm-based|bert|t5)\b", re.I)
# compile leaf patterns
LEAVES = []
for top_id, top_label, kids in TREE:
for lid, label, pats in kids:
LEAVES.append((lid, [re.compile(p, re.I) for p in pats]))
def classify(title, keywords, abstract):
scores = {}
for lid, pats in LEAVES:
s = 0.0
for p in pats:
if p.search(title): s += 3
if keywords and p.search(keywords): s += 2
if abstract:
n = len(p.findall(abstract))
if n: s += min(n, 2)
if s: scores[lid] = s
return sorted(scores.items(), key=lambda kv: -kv[1])
def txt(el):
return re.sub(r"\s+", " ", "".join(el.itertext())).strip() if el is not None else ""
def load_openreview(path, conf, accepted):
with open(path) as f:
data = json.load(f)
for p in data:
st = p.get("status", "")
if not any(st.startswith(a) or a in st for a in accepted): continue
award = "Oral" if "Oral" in st else ("Spotlight" if "Spotlight" in st else "Poster")
fid = p.get("id")
yield dict(conf=conf, track=p.get("track") or "main", award=award, ext_id=fid, title=p["title"].strip(),
abstract=(p.get("abstract") or "").strip(), authors=(p.get("author") or "").replace(";", ", "),
keywords=(p.get("keywords") or "").replace(";", ", "), tldr=p.get("tldr") or "",
url=f"https://openreview.net/forum?id={fid}", pdf_url=f"https://openreview.net/pdf?id={fid}",
primary_area=(p.get("primary_area") or "").replace("->", " > "))
ANTH = [ # (file, conf, {volume: track} or None for all volumes)
("2026.acl.xml", "acl2026", {"long": "Long", "short": "Short", "demo": "Demo", "srw": "Student Research", "industry": "Industry"}),
("2026.findings.xml", "acl2026", {"acl": "Findings"}),
("2026.findings.xml", "eacl2026", {"eacl": "Findings"}),
("2026.eacl.xml", "eacl2026", {"long": "Long", "short": "Short", "demo": "Demo", "srw": "Student Research", "industry": "Industry"}),
("2026.lrec.xml", "lrec2026", {"1": "Main"}),
("2026.conll.xml", "conll2026", {"main": "Main"}),
("2026.sigdial.xml", "sigdial2026", {"1": "Main"}),
("2026.tacl.xml", "tacl2026", {"1": "Journal"}),
("2026.cl.xml", "cl2026", {"1": "Journal", "2": "Journal"}),
]
def load_anthology(xmldir):
seen = set()
for fname, conf, vols in ANTH:
root = ET.parse(os.path.join(xmldir, fname)).getroot()
cid = root.get("id")
for v in root.findall("volume"):
vid = v.get("id")
if vid not in vols: continue
for p in v.findall("paper"):
aid = f"{cid}-{vid}.{p.get('id')}"
if (conf, aid) in seen: continue
seen.add((conf, aid))
authors = ", ".join(f"{txt(a.find('first'))} {txt(a.find('last'))}".strip() for a in p.findall("author"))
yield dict(conf=conf, track=vols[vid], award=None, ext_id=aid, title=txt(p.find("title")),
abstract=txt(p.find("abstract")), authors=authors, keywords="", tldr="",
url=f"https://aclanthology.org/{aid}/", pdf_url=f"https://aclanthology.org/{aid}.pdf", primary_area="")
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--raw", default="data/raw"); ap.add_argument("--out", default="data/papers.db")
a = ap.parse_args()
if os.path.exists(a.out): os.remove(a.out)
db = sqlite3.connect(a.out); db.executescript(SCHEMA)
db.executemany("INSERT INTO conferences VALUES(?,?,?,?,?,?,?,?,?,?)", CONFERENCES)
# taxonomy
rows, sort = [], 0
for tid, tlabel, kids in TREE:
rows.append((tid, None, tlabel, 0, sort)); sort += 1
for lid, label, _ in kids:
rows.append((lid, tid, label, 1, sort)); sort += 1
db.executemany("INSERT INTO taxonomy VALUES(?,?,?,?,?)", rows)
papers = []
papers += load_openreview(os.path.join(a.raw, "iclr2026.json"), "iclr2026", ["Poster", "Oral", "Spotlight"])
papers += load_openreview(os.path.join(a.raw, "icml2026.json"), "icml2026", ["Poster", "Oral", "Spotlight"])
papers += load_anthology(os.path.join(a.raw, "acl-anthology", "data", "xml"))
kind = {c[0]: c[4] for c in CONFERENCES}
n_cls = 0
for p in papers:
if not p["title"]: continue
text = f"{p['title']} {p['keywords']} {p['abstract']}"
p["is_llm"] = 1 if (kind[p["conf"]] == "NLP" or LLM_RE.search(text)) and (LLM_RE.search(text) or kind[p["conf"]] == "NLP") else 0
cur = db.execute("INSERT INTO papers(conf,track,award,ext_id,title,abstract,authors,keywords,tldr,url,pdf_url,primary_area,is_llm) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?)",
(p["conf"], p["track"], p["award"], p["ext_id"], p["title"], p["abstract"], p["authors"], p["keywords"], p["tldr"], p["url"], p["pdf_url"], p["primary_area"], p["is_llm"]))
pid = cur.lastrowid
if kind[p["conf"]] == "ML" and not LLM_RE.search(text):
nodes = [("other/ml", 1.0, 1)]
else:
ranked = classify(p["title"], p["keywords"], p["abstract"])
if not ranked or ranked[0][1] < 2:
nodes = [("other/unclassified", 0.0, 1)]
else:
top = ranked[0][1]
nodes = [(ranked[0][0], top, 1)]
for lid, s in ranked[1:3]:
if s >= max(3, 0.6 * top): nodes.append((lid, s, 0))
n_cls += 1
db.executemany("INSERT INTO paper_taxonomy VALUES(?,?,?,?)", [(pid, n, s, pr) for n, s, pr in nodes])
db.execute("CREATE VIRTUAL TABLE papers_fts USING fts5(title, abstract, keywords, authors, content='papers', content_rowid='id', tokenize='porter unicode61')")
db.execute("INSERT INTO papers_fts(papers_fts) VALUES('rebuild')")
db.commit()
print(f"papers: {len(papers)} classified into a topic: {n_cls}")
for r in db.execute("SELECT conf, COUNT(*) FROM papers GROUP BY conf ORDER BY 2 DESC"): print(" ", *r)
db.execute("VACUUM"); db.close()
if __name__ == "__main__":
main()