135 lines
9.4 KiB
Python
135 lines
9.4 KiB
Python
"""Canonical submission events with independent source and decision evidence."""
|
|
from datetime import datetime, timezone
|
|
import re
|
|
from paper_atlas.io.publication_ids import mlsys_id
|
|
|
|
|
|
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp'}
|
|
|
|
|
|
def timestamp(value=None):
|
|
date=datetime.now(timezone.utc) if value is None else datetime.fromisoformat(value.replace('Z','+00:00'))
|
|
if date.tzinfo is None:raise ValueError('Decision timestamp requires a timezone')
|
|
return date.astimezone(timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z')
|
|
|
|
|
|
def identities_for_row(row):
|
|
identities=list(row.get('submission_identifiers') or [])
|
|
own=row.get('identifiers') or {}
|
|
if row['source']=='mlsys':
|
|
value=own.get('publication_mlsys')
|
|
if (not isinstance(value,str) or not re.fullmatch(str(row['year'])+r'/[a-f0-9]{32}',value)
|
|
or row['venue']!='MLSys'):
|
|
raise ValueError('Invalid authoritative MLSys publisher identity')
|
|
if own.get('openreview'):
|
|
identities.append({'scheme':'openreview','value':own['openreview']})
|
|
if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and own.get('doi'):
|
|
doi=re.sub(r'^https?://(?:dx\.)?doi\.org/','',str(own['doi']),flags=re.I).strip().lower()
|
|
if re.fullmatch(r'10\.\d{4,9}/\S+',doi):identities.append({'scheme':'publication_doi','value':doi})
|
|
publication_mlsys = own.get('publication_mlsys')
|
|
if not publication_mlsys and row['source']=='dblp' and row['venue']=='MLSys':
|
|
# Older normalized DBLP files retain their own publisher links even
|
|
# when a newly supported historical URL label was not yet parsed.
|
|
links = (row.get('metadata') or {}).get('bibliography', {}).get('ee', [])
|
|
candidates = {identity for link in links if (identity := mlsys_id(link))}
|
|
if len(candidates)==1:
|
|
publication_mlsys = candidates.pop()
|
|
if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and publication_mlsys:
|
|
identities.append({'scheme':'publication_mlsys','value':publication_mlsys})
|
|
meta=row.get('metadata') or {}
|
|
if row['source']=='iclr_program' and row['venue']=='ICLR' and meta.get('program_identity_type')=='submission' and meta.get('program_submission_number_verified') is True:
|
|
identities.append({'scheme':'iclr_number','value':str(meta['program_source_id'])})
|
|
result=set()
|
|
for identity in identities:
|
|
scheme,value=identity['scheme'],str(identity['value']).strip()
|
|
if scheme=='openreview':
|
|
valid=re.fullmatch(r'[A-Za-z0-9_-]+',value) and not re.match(r'20\d{2}-Oral-',value)
|
|
elif scheme=='iclr_number':valid=re.fullmatch(r'[1-9]\d*',value) and row['venue']=='ICLR'
|
|
elif scheme=='publication_doi':valid=re.fullmatch(r'10\.\d{4,9}/\S+',value)
|
|
elif scheme=='publication_mlsys':
|
|
valid=(re.fullmatch(str(row['year'])+r'/[a-f0-9]{32}',value)
|
|
and row['venue']=='MLSys' and row['source'] in PUBLICATIONS and row.get('status')=='accepted')
|
|
else:valid=False
|
|
if not valid:raise ValueError('Invalid authoritative submission identity')
|
|
result.add((scheme,value))
|
|
return sorted(result)
|
|
|
|
|
|
def migrate_sources(c):
|
|
if c.execute("SELECT 1 FROM schema_migrations WHERE name='submission_source_baseline_v1'").fetchone():return
|
|
with c:
|
|
c.execute('INSERT OR IGNORE INTO submission_sources SELECT id,source_record_id,source_submission_id FROM submissions')
|
|
c.execute("INSERT OR IGNORE INTO decision_sources SELECT d.id,s.source_record_id,'' FROM decisions d JOIN submissions s ON s.id=d.submission_id")
|
|
c.execute("INSERT INTO schema_migrations(name) VALUES('submission_source_baseline_v1')")
|
|
|
|
|
|
def next_id(c,table,audit,former):
|
|
# Merged IDs are kept in audit; never allocate them to a later unrelated event.
|
|
return max(c.execute('SELECT COALESCE(max(id),0) FROM '+table).fetchone()[0],c.execute('SELECT COALESCE(max('+former+'),0) FROM '+audit).fetchone()[0])+1
|
|
|
|
|
|
def merge_submissions(c,target,other):
|
|
if target==other:return
|
|
editions=c.execute('SELECT id,edition_id FROM submissions WHERE id IN (?,?)',(target,other)).fetchall()
|
|
if len(editions)!=2 or editions[0][1]!=editions[1][1]:raise ValueError('Cannot combine submissions from different editions')
|
|
c.execute('INSERT OR IGNORE INTO submission_sources SELECT ?,source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(target,other))
|
|
c.execute('DELETE FROM submission_sources WHERE submission_id=?',(other,))
|
|
for decision,status,raw,url,observed,date in c.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=?',(other,)).fetchall():
|
|
duplicate=c.execute('SELECT id,decision_date FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND observed_at=?',(target,status,raw,url,observed)).fetchone()
|
|
if duplicate:
|
|
surviving=duplicate[0]
|
|
if date and duplicate[1] and date!=duplicate[1]:raise ValueError('Conflicting dates for the same decision observation')
|
|
if date and not duplicate[1]:c.execute('UPDATE decisions SET decision_date=? WHERE id=?',(date,surviving))
|
|
c.execute('INSERT OR IGNORE INTO decision_sources SELECT ?,source_record_id,source_decision_id FROM decision_sources WHERE decision_id=?',(surviving,decision))
|
|
c.execute('DELETE FROM decision_sources WHERE decision_id=?',(decision,))
|
|
c.execute('UPDATE decision_merges SET surviving_decision_id=? WHERE surviving_decision_id=?',(surviving,decision))
|
|
c.execute("INSERT INTO decision_merges(former_decision_id,surviving_decision_id,reason) VALUES(?,?,'same decision observation on merged submission')",(decision,surviving))
|
|
c.execute('DELETE FROM decisions WHERE id=?',(decision,))
|
|
else:c.execute('UPDATE decisions SET submission_id=? WHERE id=?',(target,decision))
|
|
c.execute('UPDATE submission_identifiers SET submission_id=? WHERE submission_id=?',(target,other))
|
|
c.execute('UPDATE submission_merges SET surviving_submission_id=? WHERE surviving_submission_id=?',(target,other))
|
|
c.execute("INSERT INTO submission_merges(former_submission_id,surviving_submission_id,reason) VALUES(?,?,'shared authoritative submission identity')",(other,target))
|
|
c.execute('DELETE FROM submissions WHERE id=?',(other,))
|
|
|
|
|
|
def attach_submission(store,edition,paper,record,external_id,identities):
|
|
c=store.conn
|
|
matches={r[0] for r in c.execute('SELECT s.id FROM submission_sources p JOIN submissions s ON s.id=p.submission_id WHERE p.source_record_id=? AND p.source_submission_id=? AND s.edition_id=?',(record,external_id,edition))}
|
|
matches.update(r[0] for r in c.execute('SELECT id FROM submissions WHERE source_record_id=? AND source_submission_id=? AND edition_id=?',(record,external_id,edition)))
|
|
for scheme,value in identities:
|
|
found=c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?',(edition,scheme,value)).fetchone()
|
|
if found:matches.add(found[0])
|
|
for scheme in ('openreview','iclr_number'):
|
|
values={value for kind,value in identities if kind==scheme}
|
|
for sid in matches:
|
|
values.update(value for value, in c.execute('SELECT value FROM submission_identifiers WHERE submission_id=? AND scheme=?',(sid,scheme)))
|
|
if len(values)>1:raise ValueError('Conflicting authoritative submission identities: '+scheme)
|
|
if matches:
|
|
papers={paper}|{c.execute('SELECT paper_id FROM submissions WHERE id=?',(sid,)).fetchone()[0] for sid in matches}
|
|
surviving=min(papers)
|
|
for other in sorted(papers-{surviving}):store.merge(surviving,other,'shared authoritative conference submission identity')
|
|
submission=min(matches)
|
|
for other in sorted(matches-{submission}):merge_submissions(c,submission,other)
|
|
c.execute('UPDATE submissions SET paper_id=? WHERE id=?',(surviving,submission))
|
|
else:
|
|
submission=next_id(c,'submissions','submission_merges','former_submission_id')
|
|
c.execute('INSERT INTO submissions(id,paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?,?)',(submission,paper,edition,record,external_id))
|
|
c.execute('INSERT OR IGNORE INTO submission_sources VALUES(?,?,?)',(submission,record,external_id))
|
|
for scheme,value in identities:
|
|
c.execute('INSERT OR IGNORE INTO submission_identifiers VALUES(?,?,?,?)',(edition,scheme,value,submission))
|
|
return submission
|
|
|
|
|
|
def write_decision(store,submission,record,status,raw,url,observed_at=None,decision_date=None,source_decision_id=''):
|
|
c=store.conn
|
|
observed=timestamp(observed_at)
|
|
date=timestamp(decision_date) if decision_date else None
|
|
# Re-observing unchanged evidence adds provenance, not a new decision event.
|
|
old=c.execute('SELECT id FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND COALESCE(decision_date,\'\')=COALESCE(?,\'\')',(submission,status,raw,url,date)).fetchone()
|
|
if old:decision=old[0]
|
|
else:
|
|
decision=next_id(c,'decisions','decision_merges','former_decision_id')
|
|
c.execute('INSERT INTO decisions(id,submission_id,status,raw_value,evidence_url,observed_at,decision_date) VALUES(?,?,?,?,?,?,?)',(decision,submission,status,raw,url,observed,date))
|
|
c.execute('INSERT OR IGNORE INTO decision_sources VALUES(?,?,?)',(decision,record,source_decision_id or ''))
|
|
return decision
|