90 lines
5.0 KiB
Python
90 lines
5.0 KiB
Python
"""Stream the official metadata ZIP into the configured metadata database."""
|
|
import argparse
|
|
import json
|
|
import time
|
|
import shutil
|
|
import zipfile
|
|
from pathlib import Path
|
|
|
|
|
|
from paper_atlas.db.backend import database_arguments, open_store, inserted_id
|
|
from paper_atlas.db.submissions import timestamp
|
|
|
|
|
|
def refresh_categories(store,paper,record,categories):
|
|
"""Keep category evidence for each arXiv record, including merged papers."""
|
|
if not isinstance(categories,str):raise ValueError('arXiv categories must be text')
|
|
store.record_tags(paper,record,'arxiv',[{'namespace':'arxiv:category','value':value} for value in categories.split()])
|
|
store.conn.execute("DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key='arxiv:categories' AND tag_id IN (SELECT id FROM tags WHERE namespace='arxiv:category')",(record,))
|
|
|
|
|
|
def ingest(db, archive, *, pg_service=None, pg_schema=None, storage_path=None):
|
|
store = open_store(db,pg_service=pg_service,pg_schema=pg_schema)
|
|
conn = store.conn
|
|
run = None
|
|
try:
|
|
source = store.source('arxiv','https://arxiv.org')
|
|
previous = conn.execute('SELECT id,checkpoint_json FROM ingest_runs WHERE source_id=? AND status IN (\'running\',\'failed\') ORDER BY id DESC LIMIT 1', (source,)).fetchone()
|
|
fingerprint = {'path':str(Path(archive).resolve()), 'bytes':Path(archive).stat().st_size, 'mtime_ns':Path(archive).stat().st_mtime_ns}
|
|
start = 0
|
|
if previous:
|
|
checkpoint = json.loads(previous[1] or '{}')
|
|
if checkpoint.get('archive') == fingerprint:
|
|
run = previous[0]
|
|
start = checkpoint.get('line', 0)
|
|
conn.execute("UPDATE ingest_runs SET status='running',error=NULL WHERE id=?", (run,))
|
|
else:
|
|
raise RuntimeError('Pending run belongs to a different archive; resolve it explicitly')
|
|
else:
|
|
run = inserted_id(conn,'INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)', (source,json.dumps({'archive':fingerprint,'line':0})))
|
|
conn.commit()
|
|
line_number = start
|
|
last = time.monotonic()
|
|
monitored = Path(storage_path) if storage_path else (Path(db).parent if db else Path(archive).parent)
|
|
with zipfile.ZipFile(archive) as zipped:
|
|
# Reading the member to EOF also checks its CRC.
|
|
with zipped.open('arxiv-metadata-oai-snapshot.json') as stream:
|
|
for line_number, line in enumerate(stream, 1):
|
|
if line_number <= start:
|
|
continue
|
|
row = json.loads(line)
|
|
external_id = row['id']
|
|
title = ' '.join(row['title'].split())
|
|
abstract = row.get('abstract', '').strip()
|
|
authors = row.get('authors_parsed') or row.get('authors') or []
|
|
metadata = {k:v for k,v in row.items() if k not in ('title','abstract','authors','authors_parsed')}
|
|
paper,record = store.record('arxiv',external_id,'https://arxiv.org/abs/'+external_id,title,abstract,authors,{'arxiv':external_id,'doi':row.get('doi')},metadata,(row.get('versions') or [{}])[0].get('created'))
|
|
refresh_categories(store,paper,record,row.get('categories') or '')
|
|
if line_number % 500 == 0:
|
|
if shutil.disk_usage(monitored).free < 4*1024**3:
|
|
raise RuntimeError('Stopped before exhausting disk: less than 4 GiB free')
|
|
conn.execute('UPDATE ingest_runs SET records_seen=?,records_written=?,checkpoint_json=? WHERE id=?',(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
|
|
conn.commit()
|
|
if time.monotonic()-last >= 20:
|
|
print(json.dumps({'run':run,'rows':line_number}),flush=True)
|
|
last = time.monotonic()
|
|
conn.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run))
|
|
conn.commit()
|
|
print(json.dumps({'status':'complete','rows':line_number}),flush=True)
|
|
except BaseException as exc:
|
|
conn.rollback()
|
|
if run is not None:
|
|
conn.execute("UPDATE ingest_runs SET status='failed',error=? WHERE id=?",(str(exc),run))
|
|
conn.commit()
|
|
raise
|
|
finally:
|
|
conn.close()
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
database_arguments(parser)
|
|
parser.add_argument('--archive', required=True)
|
|
parser.add_argument('--storage-path', help='Local filesystem to monitor for free space; specify the database volume when running beside PostgreSQL')
|
|
args = parser.parse_args(argv)
|
|
ingest(args.db,args.archive,pg_service=args.pg_service,pg_schema=args.pg_schema,storage_path=args.storage_path)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|