Files
paper-atlas/merge_conferences.py
T
codex-bot ad9967c1b2
Metadata integrity checks / metadata (push) Successful in 4s
Collect official COLM and NeurIPS abstracts with track-aware coverage
2026-11-06 12:04:18 +08:00

73 lines
3.5 KiB
Python

"""One-shot server operation: merge collected files at an arXiv checkpoint.
Run as a transient systemd job so an SSH disconnect cannot strand the importer.
Only the Paper Atlas arXiv unit is controlled; business services are untouched.
"""
import argparse
from contextlib import closing
import json
import sqlite3
import subprocess
from pathlib import Path
from import_records import ingest_file
from index_core import Store
UNIT='paper-atlas-arxiv-import.service'
def property_value(name):
return subprocess.check_output(['systemctl','show',UNIT,'-p',name,'--value'],text=True).strip()
def merge(db,archive,files):
# Validate inputs before stopping the existing writer.
for filename in files:
path=Path(filename)
if not path.is_file() or not path.with_suffix('.manifest.json').is_file():
raise ValueError('Missing collected file or manifest: '+filename)
if not Path(archive).is_file():
raise ValueError('Missing arXiv archive')
if not Path(db).is_file():
raise ValueError('Missing existing database')
if property_value('ActiveState')!='active':
raise RuntimeError('Expected the existing arXiv unit to be active; inspect it before proceeding')
subprocess.run(['systemctl','stop',UNIT],check=True)
try:
with closing(sqlite3.connect(db)) as c, c:
checkpoint=c.execute("SELECT records_written FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone()
print('Saved arxiv checkpoint:',checkpoint,flush=True)
schema_store=Store(db)
schema_store.conn.close()
for filename in files:
ingest_file(db,filename)
with closing(sqlite3.connect(db)) as c, c:
if files:
errors=c.execute('PRAGMA foreign_key_check').fetchall()
if errors:
raise RuntimeError('Foreign key violations after merge')
else:
print('Tag refresh plan:',c.execute('EXPLAIN QUERY PLAN DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(1,'arxiv:categories')).fetchall(),flush=True)
print('Decision totals:',c.execute('SELECT status,count(*) FROM decisions GROUP BY status').fetchall(),flush=True)
finally:
if property_value('LoadState')=='loaded':
subprocess.run(['systemctl','start',UNIT],check=True)
else:
subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).parent),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1','python3','-B',str(Path(__file__).with_name('import_arxiv.py')),'--db',db,'--archive',archive],check=True)
if property_value('ActiveState')!='active':
raise RuntimeError('arXiv unit did not resume; inspect the logs')
print('ARXIV_RESUMED',flush=True)
if __name__=='__main__':
parser=argparse.ArgumentParser()
parser.add_argument('--db',required=True)
parser.add_argument('--archive',required=True)
parser.add_argument('--schema-only',action='store_true',help='Apply idempotent schema/index updates at a saved checkpoint')
parser.add_argument('files',nargs='*')
args=parser.parse_args()
if not args.files and not args.schema_only:
parser.error('Provide collected files or --schema-only')
merge(args.db,args.archive,args.files)