94 lines
6.0 KiB
Python
94 lines
6.0 KiB
Python
"""Import normalized conference JSONL files with coverage evidence."""
|
|
import argparse
|
|
import json
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from index_core import Store
|
|
from submission_evidence import identities_for_row,timestamp
|
|
|
|
|
|
def ingest_file(db,path):
|
|
path = Path(path)
|
|
manifest_path = path.with_suffix('.manifest.json')
|
|
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
|
|
if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2:
|
|
raise ValueError('Recollect proceedings with parser_version>=2 before identity matching')
|
|
if manifest.get('source')=='iclr_program' and manifest.get('parser_version',0)<2:
|
|
raise ValueError('Recollect ICLR with parser_version>=2 before submission identity matching')
|
|
store = Store(db)
|
|
c = store.conn
|
|
source_name = manifest.get('source')
|
|
if not source_name:
|
|
raise ValueError('A source manifest is required')
|
|
source_id = store.source(source_name,manifest['list_url'])
|
|
run = c.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path)}))).lastrowid
|
|
c.commit()
|
|
count = 0
|
|
missing_abstracts = 0
|
|
seen = set()
|
|
editions = set()
|
|
try:
|
|
with path.open() as stream:
|
|
for line in stream:
|
|
row = json.loads(line)
|
|
if row['source'] != source_name:
|
|
raise ValueError('Source differs from manifest')
|
|
if any(key in manifest and manifest[key]!=row[key] for key in ('venue','year','track')):
|
|
raise ValueError('Conference edition differs from manifest')
|
|
external = row['external_id']
|
|
if external in seen:
|
|
raise ValueError('Duplicate external ID in one collected file: '+external)
|
|
seen.add(external)
|
|
paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata'))
|
|
if 'tags' in row:
|
|
store.record_tags(paper,record,source_name,row['tags'])
|
|
if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):
|
|
identities=identities_for_row(row)
|
|
events=row.get('decision_events')
|
|
if events is None:
|
|
events=[{'status':row['status'],'raw_value':row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),'evidence_url':row.get('evidence_url',row['url'])}]
|
|
if not isinstance(events,list):raise ValueError('Decision events must be a list')
|
|
if not events and row['status']!='unknown':raise ValueError('Known status requires decision evidence')
|
|
if row['status']!='unknown' and not any(event.get('status')==row['status'] for event in events):raise ValueError('Summary status has no corresponding decision evidence')
|
|
row_observed=row.get('observed_at') or timestamp()
|
|
for event in events or [{'status':None}]:
|
|
status=event['status']
|
|
if events and status is None:raise ValueError('Explicit decision event requires a status')
|
|
if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event')
|
|
edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'))
|
|
editions.add(edition)
|
|
else:
|
|
raise ValueError('This importer requires an explicit supported decision status')
|
|
count += 1
|
|
missing_abstracts += not bool((row.get('abstract') or '').strip())
|
|
if not count:
|
|
raise ValueError('Empty metadata file')
|
|
if len(editions) != 1:
|
|
raise ValueError('A manifest must describe exactly one conference edition and track')
|
|
edition = next(iter(editions))
|
|
expected = manifest.get('count')
|
|
if expected is not None and expected != count:
|
|
raise ValueError(f'Manifest count {expected} differs from records {count}')
|
|
complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0) and not missing_abstracts
|
|
c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
|
|
c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run))
|
|
c.commit()
|
|
print(json.dumps({'file':str(path),'status':'imported','records':count,'missing_abstract_count':missing_abstracts}),flush=True)
|
|
except BaseException as exc:
|
|
c.rollback()
|
|
c.execute("UPDATE ingest_runs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(exc),run))
|
|
c.commit()
|
|
raise
|
|
finally:
|
|
c.close()
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument('--db',required=True)
|
|
parser.add_argument('files',nargs='+')
|
|
args = parser.parse_args()
|
|
for filename in args.files:
|
|
ingest_file(args.db,filename)
|