Files
paper-atlas/paper_atlas/ingestion/records.py
T
codex-bot 7035cf33b9
Metadata integrity checks / metadata (push) Successful in 1m59s
Correct existing attempt tracks with bound evidence and refresh guards
2026-11-08 23:46:37 +08:00

201 lines
12 KiB
Python

"""Import normalized conference JSONL files with coverage evidence."""
import argparse
import hashlib
import json
import time
from pathlib import Path
from paper_atlas.db.backend import database_arguments, open_store, inserted_id
from paper_atlas.db.submissions import identities_for_row,timestamp
from paper_atlas.collectors.pmlr_tracks import require_track_manifest, require_track_record
from paper_atlas.collectors.public_openreview import SOURCE as PUBLIC_SOURCE
from paper_atlas.collectors.icml_public_archive import SOURCE as ARCHIVE_SOURCE
def coverage_options(manifest):
mode = manifest.get('coverage_mode', 'snapshot')
if mode not in ('snapshot', 'incremental'):
raise ValueError('Unsupported coverage mode')
if mode == 'snapshot':
return None
required = ('venue', 'year', 'track', 'count', 'coverage_previous_count',
'coverage_previous_expected_count', 'coverage_expected_count')
if any(key not in manifest for key in required) or manifest.get('status') != 'partial':
raise ValueError('Incremental coverage requires a partial, explicitly scoped manifest')
previous = manifest['coverage_previous_count']
expected = manifest['coverage_expected_count']
prior_expected = manifest['coverage_previous_expected_count']
if (type(previous) is not int or previous < 0 or type(expected) is not int or expected < 1
or type(manifest['count']) is not int or manifest['count'] < 1
or expected < max(previous, manifest['count'])
or (prior_expected is not None and (type(prior_expected) is not int or prior_expected < 0
or expected < prior_expected))):
raise ValueError('Invalid incremental coverage counts')
return previous, prior_expected, expected
def source_record_count(c, source, edition):
return c.execute('''SELECT count(DISTINCT ss.source_record_id) FROM submission_sources ss
JOIN submissions s ON s.id=ss.submission_id
JOIN source_records sr ON sr.id=ss.source_record_id
WHERE s.edition_id=? AND sr.source_id=?''', (edition, source)).fetchone()[0]
def require_incremental_parent(c, source, manifest, options):
from paper_atlas.db.events import has_event_key, validate_event_key
key = validate_event_key(manifest.get('event_key'))
enabled = has_event_key(c)
if key and not enabled:
raise ValueError('Event identity migration required for incremental coverage')
field = 'e.event_key' if enabled else "''"
rows = c.execute('SELECT e.id,'+field+""" FROM editions e JOIN venues v ON v.id=e.venue_id
WHERE lower(v.acronym)=lower(?) AND e.year=? AND e.track=?""",
(manifest['venue'], manifest['year'], manifest['track'])).fetchall()
if key is None:
if len(rows) > 1 or any(r[1] for r in rows) or (manifest['venue'] == 'WWW' and manifest['year'] in (1994,1995)):
raise ValueError('Explicit manifest event_key required for incremental coverage')
else:
rows = [r for r in rows if r[1] == key]
if len(rows) > 1:
raise ValueError('Ambiguous incremental coverage event')
edition = rows[0] if rows else None
entries = c.execute('''SELECT list_url,status,expected_count,imported_count
FROM edition_sources WHERE edition_id=? AND source_id=?''',
(edition[0], source)).fetchall() if edition else []
previous, prior_expected, _ = options
if entries:
if len(entries) != 1 or entries[0][0] != manifest['list_url']:
raise ValueError('Incremental coverage requires one unambiguous source listing')
if tuple(entries[0][1:]) != ('partial', prior_expected, previous):
raise ValueError('Incremental coverage parent changed or is not partial')
elif previous != 0 or prior_expected is not None:
raise ValueError('Incremental coverage parent is absent')
actual = source_record_count(c, source, edition[0]) if edition else 0
if actual != previous:
raise ValueError('Incremental coverage parent differs from actual source records')
def ingest_file(db,path, *, pg_service=None, pg_schema=None):
path = Path(path)
manifest_path = path.with_suffix('.manifest.json')
manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
require_track_manifest(manifest)
if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2:
raise ValueError('Recollect proceedings with parser_version>=2 before identity matching')
if manifest.get('source')=='iclr_program' and manifest.get('parser_version',0)<2:
raise ValueError('Recollect ICLR with parser_version>=2 before submission identity matching')
source_name = manifest.get('source')
if not source_name:
raise ValueError('A source manifest is required')
incremental = coverage_options(manifest)
store = open_store(db, pg_service=pg_service, pg_schema=pg_schema)
c = store.conn
run = None
count = 0
missing_abstracts = 0
seen = set()
editions = set()
try:
source_id = store.source(source_name,manifest['list_url'])
run = inserted_id(c,'INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path),'expected_sha256':manifest.get('file_sha256')})))
c.commit()
if incremental is not None:
require_incremental_parent(c, source_id, manifest, incremental)
digest = hashlib.sha256()
with path.open('rb') as stream:
for line in stream:
digest.update(line)
row = json.loads(line)
if row['source'] != source_name:
raise ValueError('Source differs from manifest')
if any(key in manifest and manifest[key]!=row.get(key) for key in ('venue','year','track','event_key')):
raise ValueError('Conference edition differs from manifest')
require_track_record(row, manifest)
from paper_atlas.db.events import validate_event_key
validate_event_key(row.get('event_key'))
external = row['external_id']
if external in seen:
raise ValueError('Duplicate external ID in one collected file: '+external)
seen.add(external)
paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata'))
if 'tags' in row:
store.record_tags(paper,record,source_name,row['tags'])
if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):
identities=identities_for_row(row)
events=row.get('decision_events')
if events is None:
events=[{'status':row['status'],'raw_value':row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),'evidence_url':row.get('evidence_url',row['url'])}]
if not isinstance(events,list):raise ValueError('Decision events must be a list')
if not events and row['status']!='unknown':raise ValueError('Known status requires decision evidence')
if row['status']!='unknown' and not any(event.get('status')==row['status'] for event in events):raise ValueError('Summary status has no corresponding decision evidence')
row_observed=row.get('observed_at') or timestamp()
for event in events or [{'status':None}]:
status=event['status']
if events and status is None:raise ValueError('Explicit decision event requires a status')
if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event')
edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'),guard_status=source_name in (PUBLIC_SOURCE,ARCHIVE_SOURCE),event_key=row.get('event_key'))
from paper_atlas.db.edition_scopes import KEY
scope_metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])
if scope_metadata.get(KEY):
actual_track=c.execute('SELECT track FROM editions WHERE id=?',(edition,)).fetchone()[0]
if row['track']!=actual_track:
raise ValueError('Recollect this whole-volume source with reviewed per-record tracks before refresh')
editions.add(edition)
else:
raise ValueError('This importer requires an explicit supported decision status')
count += 1
missing_abstracts += not bool((row.get('abstract') or '').strip())
if not count:
raise ValueError('Empty metadata file')
if manifest.get('file_sha256') and digest.hexdigest() != manifest['file_sha256']:
raise ValueError('Collected file hash differs from manifest')
if len(editions) != 1:
raise ValueError('A manifest must describe exactly one conference edition and track')
edition = next(iter(editions))
expected = manifest.get('count')
if expected is not None and expected != count:
raise ValueError(f'Manifest count {expected} differs from records {count}')
complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0) and not missing_abstracts
imported = count
condition = ''
condition_params = ()
if incremental is not None:
previous, prior_expected, expected = incremental
imported = source_record_count(c, source_id, edition)
if imported < previous or imported > expected:
raise ValueError('Incremental coverage exceeds the declared source bounds')
complete = False
condition = " WHERE edition_sources.imported_count=? AND COALESCE(edition_sources.expected_count,-1)=? AND edition_sources.status='partial'"
condition_params = (previous, -1 if prior_expected is None else prior_expected)
result = c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL'+condition,(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,imported,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),*condition_params))
if incremental is not None and result.rowcount != 1:
raise ValueError('Incremental coverage parent changed during import')
checkpoint = {'file':str(path),'file_sha256':digest.hexdigest()}
if incremental is not None:
checkpoint['coverage'] = {'mode':'incremental','previous_count':previous,
'expected_count':expected,'imported_count':imported}
c.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),count,count,json.dumps(checkpoint),run))
c.commit()
print(json.dumps({'file':str(path),'status':'imported','records':count,'missing_abstract_count':missing_abstracts}),flush=True)
except BaseException as exc:
c.rollback()
if run is not None:
c.execute("UPDATE ingest_runs SET status='failed',finished_at=?,records_seen=?,records_written=0,error=? WHERE id=?",(timestamp(),count,str(exc),run))
c.commit()
raise
finally:
c.close()
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
database_arguments(parser)
parser.add_argument('files',nargs='+')
args = parser.parse_args(argv)
for filename in args.files:
ingest_file(args.db,filename,pg_service=args.pg_service,pg_schema=args.pg_schema)
if __name__ == '__main__':
main()