Files
paper-atlas/paper_atlas/ingestion/arxiv.py
codex-bot 6ea150690d
Metadata integrity checks / metadata (push) Successful in 1m7s
Support native PostgreSQL metadata ingestion and provenance writes
2026-11-07 15:38:42 +08:00

90 lines
5.0 KiB
Python

"""Stream the official metadata ZIP into the configured metadata database."""
import argparse
import json
import time
import shutil
import zipfile
from pathlib import Path
from paper_atlas.db.backend import database_arguments, open_store, inserted_id
from paper_atlas.db.submissions import timestamp
def refresh_categories(store,paper,record,categories):
"""Keep category evidence for each arXiv record, including merged papers."""
if not isinstance(categories,str):raise ValueError('arXiv categories must be text')
store.record_tags(paper,record,'arxiv',[{'namespace':'arxiv:category','value':value} for value in categories.split()])
store.conn.execute("DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key='arxiv:categories' AND tag_id IN (SELECT id FROM tags WHERE namespace='arxiv:category')",(record,))
def ingest(db, archive, *, pg_service=None, pg_schema=None, storage_path=None):
store = open_store(db,pg_service=pg_service,pg_schema=pg_schema)
conn = store.conn
run = None
try:
source = store.source('arxiv','https://arxiv.org')
previous = conn.execute('SELECT id,checkpoint_json FROM ingest_runs WHERE source_id=? AND status IN (\'running\',\'failed\') ORDER BY id DESC LIMIT 1', (source,)).fetchone()
fingerprint = {'path':str(Path(archive).resolve()), 'bytes':Path(archive).stat().st_size, 'mtime_ns':Path(archive).stat().st_mtime_ns}
start = 0
if previous:
checkpoint = json.loads(previous[1] or '{}')
if checkpoint.get('archive') == fingerprint:
run = previous[0]
start = checkpoint.get('line', 0)
conn.execute("UPDATE ingest_runs SET status='running',error=NULL WHERE id=?", (run,))
else:
raise RuntimeError('Pending run belongs to a different archive; resolve it explicitly')
else:
run = inserted_id(conn,'INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)', (source,json.dumps({'archive':fingerprint,'line':0})))
conn.commit()
line_number = start
last = time.monotonic()
monitored = Path(storage_path) if storage_path else (Path(db).parent if db else Path(archive).parent)
with zipfile.ZipFile(archive) as zipped:
# Reading the member to EOF also checks its CRC.
with zipped.open('arxiv-metadata-oai-snapshot.json') as stream:
for line_number, line in enumerate(stream, 1):
if line_number <= start:
continue
row = json.loads(line)
external_id = row['id']
title = ' '.join(row['title'].split())
abstract = row.get('abstract', '').strip()
authors = row.get('authors_parsed') or row.get('authors') or []
metadata = {k:v for k,v in row.items() if k not in ('title','abstract','authors','authors_parsed')}
paper,record = store.record('arxiv',external_id,'https://arxiv.org/abs/'+external_id,title,abstract,authors,{'arxiv':external_id,'doi':row.get('doi')},metadata,(row.get('versions') or [{}])[0].get('created'))
refresh_categories(store,paper,record,row.get('categories') or '')
if line_number % 500 == 0:
if shutil.disk_usage(monitored).free < 4*1024**3:
raise RuntimeError('Stopped before exhausting disk: less than 4 GiB free')
conn.execute('UPDATE ingest_runs SET records_seen=?,records_written=?,checkpoint_json=? WHERE id=?',(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
conn.commit()
if time.monotonic()-last >= 20:
print(json.dumps({'run':run,'rows':line_number}),flush=True)
last = time.monotonic()
conn.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
conn.commit()
print(json.dumps({'status':'complete','rows':line_number}),flush=True)
except BaseException as exc:
conn.rollback()
if run is not None:
conn.execute("UPDATE ingest_runs SET status='failed',error=? WHERE id=?",(str(exc),run))
conn.commit()
raise
finally:
conn.close()
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
database_arguments(parser)
parser.add_argument('--archive', required=True)
parser.add_argument('--storage-path', help='Local filesystem to monitor for free space; specify the database volume when running beside PostgreSQL')
args = parser.parse_args(argv)
ingest(args.db,args.archive,pg_service=args.pg_service,pg_schema=args.pg_schema,storage_path=args.storage_path)
if __name__ == '__main__':
main()