Files
paper-atlas/paper_atlas/ingestion/pmlr_tracks.py
codex-bot a3b677bdf2
Metadata integrity checks / metadata (push) Successful in 46s
Add verified PostgreSQL migration and shared research queries
2026-11-07 02:09:19 +08:00

204 lines
14 KiB
Python

"""Move existing PMLR submission events to verified tracks without reimporting."""
import argparse
from contextlib import closing
import fcntl
import hashlib
import json
from pathlib import Path
import sqlite3
from paper_atlas.collectors.pmlr_tracks import require_track_manifest, require_track_record
from paper_atlas.db.submissions import identities_for_row, timestamp
def save(path, result):
part = path.with_suffix('.part')
part.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
part.replace(path)
def load_inventories(paths):
scopes, seen, forums = [], set(), set()
for filename in paths:
path = Path(filename).resolve()
inventory = json.loads(path.read_text())
require_track_manifest(inventory)
if inventory.get('source') != 'pmlr' or inventory.get('venue') != 'ICML' or inventory.get('year', 0) < 2024:
raise ValueError('Only verified modern ICML PMLR volumes may be reclassified')
if not inventory.get('count') or not inventory.get('files'):
raise ValueError('A nonempty complete publication inventory is required')
rows, manifests = [], []
tracks = set()
for manifest in inventory['files']:
require_track_manifest(manifest)
if any(manifest.get(key) != inventory.get(key) for key in ('source', 'venue', 'year', 'volume', 'track_program')):
raise ValueError('Track manifest differs from volume inventory')
if manifest['track'] not in ('main', 'position') or manifest['track'] in tracks:
raise ValueError('Invalid or repeated track in volume inventory')
tracks.add(manifest['track'])
data = Path(manifest['file']).read_bytes()
if hashlib.sha256(data).hexdigest() != manifest.get('file_sha256'):
raise ValueError('Track file differs from its verified inventory')
chunk = [json.loads(line) for line in data.splitlines()]
if not chunk or len(chunk) != manifest['count']:
raise ValueError('Track inventory count mismatch')
for row in chunk:
if any(row.get(key) != manifest.get(key) for key in ('source', 'venue', 'year', 'track')):
raise ValueError('Record differs from its track manifest')
if not row['external_id'].startswith('v' + str(inventory['volume']) + '/') or row['status'] != 'accepted':
raise ValueError('Record is outside the accepted publication scope')
if row['external_id'] in seen:
raise ValueError('Repeated publication in reclassification inventory')
seen.add(row['external_id'])
require_track_record(row, manifest)
own_forum = row.get('identifiers', {}).get('openreview')
if not own_forum or (row['year'], own_forum) in forums:
raise ValueError('Missing or repeated forum identity in one ICML edition')
forums.add((row['year'], own_forum))
rows.extend(chunk); manifests.append(manifest)
if len(rows) != inventory['count']:
raise ValueError('Volume count differs from its track files')
scopes.append({'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'inventory': inventory, 'manifests': manifests, 'rows': rows})
if not scopes:
raise ValueError('At least one complete volume inventory is required')
return scopes
def plan(c, scopes):
updates, changes = [], []
for scope in scopes:
volume = scope['inventory']['volume']
source_rows = c.execute('''SELECT r.id,r.external_id FROM source_records r JOIN sources src ON src.id=r.source_id
WHERE src.name='pmlr' AND r.external_id LIKE ?''', ('v' + str(volume) + '/%',)).fetchall()
if {r[1] for r in source_rows} != {r['external_id'] for r in scope['rows']}:
raise ValueError('Existing PMLR volume differs from the complete audited publication inventory')
for row in scope['rows']:
record = c.execute('''SELECT r.id,r.paper_id,r.title,r.abstract,r.authors_json,r.metadata_json FROM source_records r
JOIN sources src ON src.id=r.source_id WHERE src.name='pmlr' AND r.external_id=?''', (row['external_id'],)).fetchone()
if (record[2] != row['title'] or record[3] != row.get('abstract')
or json.loads(record[4]) != row.get('authors', [])):
raise ValueError('Reclassification must preserve existing publisher text and authors: ' + row['external_id'])
metadata = json.loads(record[5])
forum = str(metadata.get('openreview') or (metadata.get('record_identifiers') or {}).get('openreview') or '')
forum = forum.split('id=')[-1].split('&')[0]
if not forum or forum != row['identifiers'].get('openreview'):
raise ValueError('Publisher forum identity differs from retained source metadata')
events = c.execute('''SELECT s.id,s.paper_id,s.edition_id,e.venue_id,e.year,e.track,v.acronym,s.source_record_id,s.source_submission_id
FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id JOIN editions e ON e.id=s.edition_id
JOIN venues v ON v.id=e.venue_id WHERE ss.source_record_id=? AND ss.source_submission_id=?''',
(record[0], row['external_id'])).fetchall()
if len(events) != 1:
raise ValueError('Publication must have exactly one existing submission event')
event = events[0]
sid, paper, old_edition, venue, year, old_track, acronym, owner, external = event
if paper != record[1] or year != row['year'] or acronym.upper() != 'ICML' or old_track not in ('main', 'position'):
raise ValueError('Existing submission is outside this publication scope')
identities = c.execute('SELECT scheme,value FROM submission_identifiers WHERE submission_id=? ORDER BY scheme,value', (sid,)).fetchall()
own = identities_for_row(row)
if {value for scheme, value in identities if scheme == 'openreview'} - {forum}:
raise ValueError('Existing event has a conflicting forum identity')
target = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?', (venue, year, row['track'])).fetchone()
if target:
for scheme, value in sorted(set(identities) | set(own)):
collision = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target[0], scheme, value)).fetchone()
if collision and collision[0] != sid:
raise ValueError('Target track already has another event for this identity; reconcile sources before moving')
if c.execute('SELECT 1 FROM submissions WHERE source_record_id=? AND source_submission_id=? AND edition_id=? AND id<>?', (owner, external, target[0], sid)).fetchone():
raise ValueError('Target track already has this publication event')
if old_track != row['track']:
links = c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?', (sid,)).fetchall()
if links != [(record[0], row['external_id'])] or owner != record[0] or external != row['external_id']:
raise ValueError('Moving an event shared by multiple sources requires a separate reconciliation')
decisions = c.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY id', (sid,)).fetchall()
if not decisions or any(d[1] != 'accepted' for d in decisions):
raise ValueError('Moving a publication with conflicting decision history requires reconciliation')
changes.append({'submission_id': sid, 'source_record_id': record[0], 'external_id': row['external_id'],
'paper_id': paper, 'from_edition_id': old_edition, 'from_track': old_track,
'to_track': row['track'], 'year': year, 'before_identifiers': identities,
'before_decisions': decisions, 'track_evidence': row['metadata']['track_evidence']})
updates.append((row, record, event, own))
return updates, changes
def reclassify(db, inventories, out, apply=False):
db, out = Path(db).resolve(), Path(out).resolve()
if apply:
from paper_atlas.db.backend import require_sqlite_writable
require_sqlite_writable(db)
if not db.is_file() or out == db:
raise ValueError('Existing database and distinct report path required')
scopes = load_inventories(inventories)
inputs = {Path(scope['path']) for scope in scopes}
inputs.update(Path(m['file']).resolve() for scope in scopes for m in scope['manifests'])
inputs.update(Path(m['file']).with_suffix('.manifest.json').resolve() for scope in scopes for m in scope['manifests'])
if out in inputs:
raise ValueError('Report must preserve evidence inputs')
out.parent.mkdir(parents=True, exist_ok=True)
with Path(str(db) + '.maintenance.lock').open('a') as lock, Path(str(out) + '.lock').open('a') as output_lock:
fcntl.flock(lock, (fcntl.LOCK_EX if apply else fcntl.LOCK_SH) | fcntl.LOCK_NB)
fcntl.flock(output_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if out.exists():
raise ValueError('Reclassification report exists; use a new path')
mode = 'rw' if apply else 'ro'
with closing(sqlite3.connect(db.as_uri() + '?mode=' + mode, uri=True)) as c:
c.execute('PRAGMA foreign_keys=ON'); c.execute('PRAGMA cache_size=-32768')
c.execute('BEGIN IMMEDIATE' if apply else 'BEGIN')
updates, changes = plan(c, scopes)
result = {'status': 'planned', 'db': str(db), 'started_at': timestamp(),
'records_verified': len(updates), 'events_to_move': len(changes), 'changes': changes,
'inputs': [{key: scope[key] for key in ('path', 'sha256')} for scope in scopes],
'note': 'Preserves paper/source/submission/decision IDs and all decision observations; no entity merges or text reimport.'}
save(out, result)
if not apply:
return result
try:
for row, record, event, own in updates:
sid, _, old_edition, venue, year, old_track, _, _, _ = event
c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)', (venue, year, row['track']))
target = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?', (venue, year, row['track'])).fetchone()[0]
if target != old_edition:
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (target, sid))
c.execute('UPDATE submissions SET edition_id=? WHERE id=?', (target, sid))
for scheme, value in own:
c.execute('INSERT OR IGNORE INTO submission_identifiers VALUES(?,?,?,?)', (target, scheme, value, sid))
metadata = json.loads(record[5]); metadata.update(row['metadata'])
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?', (json.dumps(metadata, ensure_ascii=False), record[0]))
for scope in scopes:
for m in scope['manifests']:
edition = c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? COLLATE NOCASE AND e.year=? AND e.track=?', ('ICML', m['year'], m['track'])).fetchone()[0]
source = c.execute("SELECT id FROM sources WHERE name='pmlr'").fetchone()[0]
c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at)
VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL''',
(edition, source, m['list_url'], 'partial' if m.get('missing_abstract_count') or m.get('status') == 'partial' else 'complete', m['count'], m['count'], timestamp()))
_, remaining = plan(c, scopes)
if remaining:
raise ValueError('Post-migration track comparison still has mismatches')
print(json.dumps({'stage': 'checking_foreign_keys', 'events_moved': len(changes)}), flush=True)
if c.execute('PRAGMA foreign_key_check').fetchone():
raise ValueError('Foreign key violations after track reclassification')
result.update(status='prepared', events_moved=len(changes))
save(out, result)
c.commit()
except BaseException as error:
c.rollback(); result.update(status='error', error=str(error)); save(out, result); raise
result.update(status='complete', finished_at=timestamp())
save(out, result)
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--db', required=True)
parser.add_argument('--inventory', action='append', required=True)
parser.add_argument('--out', required=True)
parser.add_argument('--apply', action='store_true', help='Apply the verified migration under an exclusive maintenance lock')
args = parser.parse_args(argv)
result = reclassify(args.db, args.inventory, args.out, args.apply)
print(json.dumps({key: value for key, value in result.items() if key != 'changes'}), flush=True)
if __name__ == '__main__':
main()