Files
paper-atlas/paper_atlas/audits/public_openreview.py
codex-bot bf8178354f
Metadata integrity checks / metadata (push) Successful in 2m42s
Preserve Unicode separators in public metadata preflight
2026-11-07 16:34:54 +08:00

111 lines
6.0 KiB
Python

"""Check curated negative assertions against existing submission decisions."""
import argparse
from contextlib import closing
import hashlib
import json
from pathlib import Path
from paper_atlas.collectors.public_openreview import SOURCE, REVISION
from paper_atlas.collectors.icml_public_archive import SOURCE as ARCHIVE_SOURCE, REVISION as ARCHIVE_REVISION
from paper_atlas.db.submissions import timestamp, latest_status
from paper_atlas.db.backend import database_arguments, open_read_connection
def audit(db, inventory, out, *, pg_service=None, pg_schema=None):
inventory = Path(inventory).resolve()
result = json.loads(inventory.read_text())
specs = {SOURCE: (REVISION, 'curated_public_status_assertion'),
ARCHIVE_SOURCE: (ARCHIVE_REVISION, 'public_submission_venueid')}
source = result.get('source')
if source not in specs or result.get('revision') != specs[source][0]:
raise ValueError('Unsupported public source inventory')
revision, evidence_kind = specs[source]
conflicts = []
seen = set()
checked = 0
existing_attempts = 0
files = []
checksums = {}
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
if pg_service:
identity = c.execute('SELECT current_database(),current_schema()').fetchone()
database, actual_schema = identity[0], identity[1]
else:
database, actual_schema = str(Path(db).resolve()), None
for manifest in result['files']:
path = Path(manifest['file'])
if hashlib.sha256(path.read_bytes()).hexdigest() != manifest['file_sha256']:
raise ValueError('Collected file differs from pinned inventory')
if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest:
raise ValueError('Collected manifest differs from inventory')
count = 0
# Unicode paragraph/line separators are valid inside JSON strings;
# split physical byte lines, not Unicode text boundaries.
for line in path.read_bytes().splitlines():
row = json.loads(line)
if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')):
raise ValueError('Record edition differs from inventory')
meta = row['metadata']
if meta.get('evidence_kind') != evidence_kind or meta.get('source_revision') != revision:
raise ValueError('Missing explicit public source provenance')
if meta.get('original_decision_signature_verified') is not False:
raise ValueError('Curated metadata must not claim original signatures')
if row['status'] not in ('rejected', 'desk_rejected', 'withdrawn'):
raise ValueError('Unsupported negative assertion')
forum = row['identifiers']['openreview']
if forum in seen:
raise ValueError('Repeated forum in normalized inventory')
seen.add(forum)
existing = c.execute('''SELECT s.id FROM submission_identifiers i
JOIN submissions s ON s.id=i.submission_id JOIN editions e ON e.id=i.edition_id
JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? AND e.year=? AND e.track=?
AND i.scheme='openreview' AND i.value=?''',
(row['venue'], row['year'], row['track'], forum)).fetchall()
if len(existing) > 1:
raise ValueError('Multiple canonical attempts for one forum')
if existing:
existing_attempts += 1
latest = latest_status(c, existing[0][0])
if latest and latest not in ('unknown', 'under_review', row['status']):
conflicts.append({'forum': forum, 'venue': row['venue'], 'year': row['year'],
'track': row['track'], 'submission_id': existing[0][0],
'existing_status': latest, 'curated_status': row['status']})
count += 1
if count != manifest['count']:
raise ValueError('Collected record count differs from inventory')
checked += count
files.append(str(path))
checksums[str(path)] = manifest['file_sha256']
if checked != result['normalized_records']:
raise ValueError('Normalized inventory total differs from checked records')
report = {'source': source, 'revision': revision, 'database': database,
'engine': 'postgresql' if pg_service else 'sqlite', 'schema': actual_schema, 'pg_service': pg_service,
'inventory': str(inventory), 'checked_at': timestamp(), 'records_checked': checked,
'existing_same_edition_attempts': existing_attempts, 'conflicts': conflicts,
'status': 'blocked' if conflicts else 'ready_for_serial_import',
'files': [] if conflicts else files, 'file_sha256': {} if conflicts else checksums,
'note': 'Read-only preflight. Other-conference acceptances remain separate history; '
'this is not an entity deduplication or complete rejection-coverage proof.'}
out = Path(out)
out.parent.mkdir(parents=True, exist_ok=True)
part = out.with_suffix('.part')
part.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
part.replace(out)
print(json.dumps(report), flush=True)
return report
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
database_arguments(parser)
parser.add_argument('--inventory', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
report = audit(args.db, args.inventory, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
if report['status'] == 'blocked':
raise SystemExit('Conflicting existing decisions require review before import')
if __name__ == '__main__':
main()