204 lines
14 KiB
Python
204 lines
14 KiB
Python
"""Move existing PMLR submission events to verified tracks without reimporting."""
|
|
import argparse
|
|
from contextlib import closing
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
|
|
from paper_atlas.collectors.pmlr_tracks import require_track_manifest, require_track_record
|
|
from paper_atlas.db.submissions import identities_for_row, timestamp
|
|
|
|
|
|
def save(path, result):
|
|
part = path.with_suffix('.part')
|
|
part.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
|
|
part.replace(path)
|
|
|
|
|
|
def load_inventories(paths):
|
|
scopes, seen, forums = [], set(), set()
|
|
for filename in paths:
|
|
path = Path(filename).resolve()
|
|
inventory = json.loads(path.read_text())
|
|
require_track_manifest(inventory)
|
|
if inventory.get('source') != 'pmlr' or inventory.get('venue') != 'ICML' or inventory.get('year', 0) < 2024:
|
|
raise ValueError('Only verified modern ICML PMLR volumes may be reclassified')
|
|
if not inventory.get('count') or not inventory.get('files'):
|
|
raise ValueError('A nonempty complete publication inventory is required')
|
|
rows, manifests = [], []
|
|
tracks = set()
|
|
for manifest in inventory['files']:
|
|
require_track_manifest(manifest)
|
|
if any(manifest.get(key) != inventory.get(key) for key in ('source', 'venue', 'year', 'volume', 'track_program')):
|
|
raise ValueError('Track manifest differs from volume inventory')
|
|
if manifest['track'] not in ('main', 'position') or manifest['track'] in tracks:
|
|
raise ValueError('Invalid or repeated track in volume inventory')
|
|
tracks.add(manifest['track'])
|
|
data = Path(manifest['file']).read_bytes()
|
|
if hashlib.sha256(data).hexdigest() != manifest.get('file_sha256'):
|
|
raise ValueError('Track file differs from its verified inventory')
|
|
chunk = [json.loads(line) for line in data.splitlines()]
|
|
if not chunk or len(chunk) != manifest['count']:
|
|
raise ValueError('Track inventory count mismatch')
|
|
for row in chunk:
|
|
if any(row.get(key) != manifest.get(key) for key in ('source', 'venue', 'year', 'track')):
|
|
raise ValueError('Record differs from its track manifest')
|
|
if not row['external_id'].startswith('v' + str(inventory['volume']) + '/') or row['status'] != 'accepted':
|
|
raise ValueError('Record is outside the accepted publication scope')
|
|
if row['external_id'] in seen:
|
|
raise ValueError('Repeated publication in reclassification inventory')
|
|
seen.add(row['external_id'])
|
|
require_track_record(row, manifest)
|
|
own_forum = row.get('identifiers', {}).get('openreview')
|
|
if not own_forum or (row['year'], own_forum) in forums:
|
|
raise ValueError('Missing or repeated forum identity in one ICML edition')
|
|
forums.add((row['year'], own_forum))
|
|
rows.extend(chunk); manifests.append(manifest)
|
|
if len(rows) != inventory['count']:
|
|
raise ValueError('Volume count differs from its track files')
|
|
scopes.append({'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
'inventory': inventory, 'manifests': manifests, 'rows': rows})
|
|
if not scopes:
|
|
raise ValueError('At least one complete volume inventory is required')
|
|
return scopes
|
|
|
|
|
|
def plan(c, scopes):
|
|
updates, changes = [], []
|
|
for scope in scopes:
|
|
volume = scope['inventory']['volume']
|
|
source_rows = c.execute('''SELECT r.id,r.external_id FROM source_records r JOIN sources src ON src.id=r.source_id
|
|
WHERE src.name='pmlr' AND r.external_id LIKE ?''', ('v' + str(volume) + '/%',)).fetchall()
|
|
if {r[1] for r in source_rows} != {r['external_id'] for r in scope['rows']}:
|
|
raise ValueError('Existing PMLR volume differs from the complete audited publication inventory')
|
|
for row in scope['rows']:
|
|
record = c.execute('''SELECT r.id,r.paper_id,r.title,r.abstract,r.authors_json,r.metadata_json FROM source_records r
|
|
JOIN sources src ON src.id=r.source_id WHERE src.name='pmlr' AND r.external_id=?''', (row['external_id'],)).fetchone()
|
|
if (record[2] != row['title'] or record[3] != row.get('abstract')
|
|
or json.loads(record[4]) != row.get('authors', [])):
|
|
raise ValueError('Reclassification must preserve existing publisher text and authors: ' + row['external_id'])
|
|
metadata = json.loads(record[5])
|
|
forum = str(metadata.get('openreview') or (metadata.get('record_identifiers') or {}).get('openreview') or '')
|
|
forum = forum.split('id=')[-1].split('&')[0]
|
|
if not forum or forum != row['identifiers'].get('openreview'):
|
|
raise ValueError('Publisher forum identity differs from retained source metadata')
|
|
events = c.execute('''SELECT s.id,s.paper_id,s.edition_id,e.venue_id,e.year,e.track,v.acronym,s.source_record_id,s.source_submission_id
|
|
FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id JOIN editions e ON e.id=s.edition_id
|
|
JOIN venues v ON v.id=e.venue_id WHERE ss.source_record_id=? AND ss.source_submission_id=?''',
|
|
(record[0], row['external_id'])).fetchall()
|
|
if len(events) != 1:
|
|
raise ValueError('Publication must have exactly one existing submission event')
|
|
event = events[0]
|
|
sid, paper, old_edition, venue, year, old_track, acronym, owner, external = event
|
|
if paper != record[1] or year != row['year'] or acronym.upper() != 'ICML' or old_track not in ('main', 'position'):
|
|
raise ValueError('Existing submission is outside this publication scope')
|
|
identities = c.execute('SELECT scheme,value FROM submission_identifiers WHERE submission_id=? ORDER BY scheme,value', (sid,)).fetchall()
|
|
own = identities_for_row(row)
|
|
if {value for scheme, value in identities if scheme == 'openreview'} - {forum}:
|
|
raise ValueError('Existing event has a conflicting forum identity')
|
|
target = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?', (venue, year, row['track'])).fetchone()
|
|
if target:
|
|
for scheme, value in sorted(set(identities) | set(own)):
|
|
collision = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target[0], scheme, value)).fetchone()
|
|
if collision and collision[0] != sid:
|
|
raise ValueError('Target track already has another event for this identity; reconcile sources before moving')
|
|
if c.execute('SELECT 1 FROM submissions WHERE source_record_id=? AND source_submission_id=? AND edition_id=? AND id<>?', (owner, external, target[0], sid)).fetchone():
|
|
raise ValueError('Target track already has this publication event')
|
|
if old_track != row['track']:
|
|
links = c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?', (sid,)).fetchall()
|
|
if links != [(record[0], row['external_id'])] or owner != record[0] or external != row['external_id']:
|
|
raise ValueError('Moving an event shared by multiple sources requires a separate reconciliation')
|
|
decisions = c.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY id', (sid,)).fetchall()
|
|
if not decisions or any(d[1] != 'accepted' for d in decisions):
|
|
raise ValueError('Moving a publication with conflicting decision history requires reconciliation')
|
|
changes.append({'submission_id': sid, 'source_record_id': record[0], 'external_id': row['external_id'],
|
|
'paper_id': paper, 'from_edition_id': old_edition, 'from_track': old_track,
|
|
'to_track': row['track'], 'year': year, 'before_identifiers': identities,
|
|
'before_decisions': decisions, 'track_evidence': row['metadata']['track_evidence']})
|
|
updates.append((row, record, event, own))
|
|
return updates, changes
|
|
|
|
|
|
def reclassify(db, inventories, out, apply=False):
|
|
db, out = Path(db).resolve(), Path(out).resolve()
|
|
if apply:
|
|
from paper_atlas.db.backend import require_sqlite_writable
|
|
require_sqlite_writable(db)
|
|
if not db.is_file() or out == db:
|
|
raise ValueError('Existing database and distinct report path required')
|
|
scopes = load_inventories(inventories)
|
|
inputs = {Path(scope['path']) for scope in scopes}
|
|
inputs.update(Path(m['file']).resolve() for scope in scopes for m in scope['manifests'])
|
|
inputs.update(Path(m['file']).with_suffix('.manifest.json').resolve() for scope in scopes for m in scope['manifests'])
|
|
if out in inputs:
|
|
raise ValueError('Report must preserve evidence inputs')
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
with Path(str(db) + '.maintenance.lock').open('a') as lock, Path(str(out) + '.lock').open('a') as output_lock:
|
|
fcntl.flock(lock, (fcntl.LOCK_EX if apply else fcntl.LOCK_SH) | fcntl.LOCK_NB)
|
|
fcntl.flock(output_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if out.exists():
|
|
raise ValueError('Reclassification report exists; use a new path')
|
|
mode = 'rw' if apply else 'ro'
|
|
with closing(sqlite3.connect(db.as_uri() + '?mode=' + mode, uri=True)) as c:
|
|
c.execute('PRAGMA foreign_keys=ON'); c.execute('PRAGMA cache_size=-32768')
|
|
c.execute('BEGIN IMMEDIATE' if apply else 'BEGIN')
|
|
updates, changes = plan(c, scopes)
|
|
result = {'status': 'planned', 'db': str(db), 'started_at': timestamp(),
|
|
'records_verified': len(updates), 'events_to_move': len(changes), 'changes': changes,
|
|
'inputs': [{key: scope[key] for key in ('path', 'sha256')} for scope in scopes],
|
|
'note': 'Preserves paper/source/submission/decision IDs and all decision observations; no entity merges or text reimport.'}
|
|
save(out, result)
|
|
if not apply:
|
|
return result
|
|
try:
|
|
for row, record, event, own in updates:
|
|
sid, _, old_edition, venue, year, old_track, _, _, _ = event
|
|
c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)', (venue, year, row['track']))
|
|
target = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?', (venue, year, row['track'])).fetchone()[0]
|
|
if target != old_edition:
|
|
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (target, sid))
|
|
c.execute('UPDATE submissions SET edition_id=? WHERE id=?', (target, sid))
|
|
for scheme, value in own:
|
|
c.execute('INSERT OR IGNORE INTO submission_identifiers VALUES(?,?,?,?)', (target, scheme, value, sid))
|
|
metadata = json.loads(record[5]); metadata.update(row['metadata'])
|
|
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?', (json.dumps(metadata, ensure_ascii=False), record[0]))
|
|
for scope in scopes:
|
|
for m in scope['manifests']:
|
|
edition = c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? COLLATE NOCASE AND e.year=? AND e.track=?', ('ICML', m['year'], m['track'])).fetchone()[0]
|
|
source = c.execute("SELECT id FROM sources WHERE name='pmlr'").fetchone()[0]
|
|
c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at)
|
|
VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
|
|
expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL''',
|
|
(edition, source, m['list_url'], 'partial' if m.get('missing_abstract_count') or m.get('status') == 'partial' else 'complete', m['count'], m['count'], timestamp()))
|
|
_, remaining = plan(c, scopes)
|
|
if remaining:
|
|
raise ValueError('Post-migration track comparison still has mismatches')
|
|
print(json.dumps({'stage': 'checking_foreign_keys', 'events_moved': len(changes)}), flush=True)
|
|
if c.execute('PRAGMA foreign_key_check').fetchone():
|
|
raise ValueError('Foreign key violations after track reclassification')
|
|
result.update(status='prepared', events_moved=len(changes))
|
|
save(out, result)
|
|
c.commit()
|
|
except BaseException as error:
|
|
c.rollback(); result.update(status='error', error=str(error)); save(out, result); raise
|
|
result.update(status='complete', finished_at=timestamp())
|
|
save(out, result)
|
|
return result
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--db', required=True)
|
|
parser.add_argument('--inventory', action='append', required=True)
|
|
parser.add_argument('--out', required=True)
|
|
parser.add_argument('--apply', action='store_true', help='Apply the verified migration under an exclusive maintenance lock')
|
|
args = parser.parse_args(argv)
|
|
result = reclassify(args.db, args.inventory, args.out, args.apply)
|
|
print(json.dumps({key: value for key, value in result.items() if key != 'changes'}), flush=True)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|