111 lines
6.0 KiB
Python
111 lines
6.0 KiB
Python
"""Check curated negative assertions against existing submission decisions."""
|
|
import argparse
|
|
from contextlib import closing
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from paper_atlas.collectors.public_openreview import SOURCE, REVISION
|
|
from paper_atlas.collectors.icml_public_archive import SOURCE as ARCHIVE_SOURCE, REVISION as ARCHIVE_REVISION
|
|
from paper_atlas.db.submissions import timestamp, latest_status
|
|
from paper_atlas.db.backend import database_arguments, open_read_connection
|
|
|
|
|
|
def audit(db, inventory, out, *, pg_service=None, pg_schema=None):
|
|
inventory = Path(inventory).resolve()
|
|
result = json.loads(inventory.read_text())
|
|
specs = {SOURCE: (REVISION, 'curated_public_status_assertion'),
|
|
ARCHIVE_SOURCE: (ARCHIVE_REVISION, 'public_submission_venueid')}
|
|
source = result.get('source')
|
|
if source not in specs or result.get('revision') != specs[source][0]:
|
|
raise ValueError('Unsupported public source inventory')
|
|
revision, evidence_kind = specs[source]
|
|
conflicts = []
|
|
seen = set()
|
|
checked = 0
|
|
existing_attempts = 0
|
|
files = []
|
|
checksums = {}
|
|
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
|
|
if pg_service:
|
|
identity = c.execute('SELECT current_database(),current_schema()').fetchone()
|
|
database, actual_schema = identity[0], identity[1]
|
|
else:
|
|
database, actual_schema = str(Path(db).resolve()), None
|
|
for manifest in result['files']:
|
|
path = Path(manifest['file'])
|
|
if hashlib.sha256(path.read_bytes()).hexdigest() != manifest['file_sha256']:
|
|
raise ValueError('Collected file differs from pinned inventory')
|
|
if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest:
|
|
raise ValueError('Collected manifest differs from inventory')
|
|
count = 0
|
|
# Unicode paragraph/line separators are valid inside JSON strings;
|
|
# split physical byte lines, not Unicode text boundaries.
|
|
for line in path.read_bytes().splitlines():
|
|
row = json.loads(line)
|
|
if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')):
|
|
raise ValueError('Record edition differs from inventory')
|
|
meta = row['metadata']
|
|
if meta.get('evidence_kind') != evidence_kind or meta.get('source_revision') != revision:
|
|
raise ValueError('Missing explicit public source provenance')
|
|
if meta.get('original_decision_signature_verified') is not False:
|
|
raise ValueError('Curated metadata must not claim original signatures')
|
|
if row['status'] not in ('rejected', 'desk_rejected', 'withdrawn'):
|
|
raise ValueError('Unsupported negative assertion')
|
|
forum = row['identifiers']['openreview']
|
|
if forum in seen:
|
|
raise ValueError('Repeated forum in normalized inventory')
|
|
seen.add(forum)
|
|
existing = c.execute('''SELECT s.id FROM submission_identifiers i
|
|
JOIN submissions s ON s.id=i.submission_id JOIN editions e ON e.id=i.edition_id
|
|
JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? AND e.year=? AND e.track=?
|
|
AND i.scheme='openreview' AND i.value=?''',
|
|
(row['venue'], row['year'], row['track'], forum)).fetchall()
|
|
if len(existing) > 1:
|
|
raise ValueError('Multiple canonical attempts for one forum')
|
|
if existing:
|
|
existing_attempts += 1
|
|
latest = latest_status(c, existing[0][0])
|
|
if latest and latest not in ('unknown', 'under_review', row['status']):
|
|
conflicts.append({'forum': forum, 'venue': row['venue'], 'year': row['year'],
|
|
'track': row['track'], 'submission_id': existing[0][0],
|
|
'existing_status': latest, 'curated_status': row['status']})
|
|
count += 1
|
|
if count != manifest['count']:
|
|
raise ValueError('Collected record count differs from inventory')
|
|
checked += count
|
|
files.append(str(path))
|
|
checksums[str(path)] = manifest['file_sha256']
|
|
if checked != result['normalized_records']:
|
|
raise ValueError('Normalized inventory total differs from checked records')
|
|
report = {'source': source, 'revision': revision, 'database': database,
|
|
'engine': 'postgresql' if pg_service else 'sqlite', 'schema': actual_schema, 'pg_service': pg_service,
|
|
'inventory': str(inventory), 'checked_at': timestamp(), 'records_checked': checked,
|
|
'existing_same_edition_attempts': existing_attempts, 'conflicts': conflicts,
|
|
'status': 'blocked' if conflicts else 'ready_for_serial_import',
|
|
'files': [] if conflicts else files, 'file_sha256': {} if conflicts else checksums,
|
|
'note': 'Read-only preflight. Other-conference acceptances remain separate history; '
|
|
'this is not an entity deduplication or complete rejection-coverage proof.'}
|
|
out = Path(out)
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
part = out.with_suffix('.part')
|
|
part.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
|
part.replace(out)
|
|
print(json.dumps(report), flush=True)
|
|
return report
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
database_arguments(parser)
|
|
parser.add_argument('--inventory', required=True)
|
|
parser.add_argument('--out', required=True)
|
|
args = parser.parse_args(argv)
|
|
report = audit(args.db, args.inventory, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
|
if report['status'] == 'blocked':
|
|
raise SystemExit('Conflicting existing decisions require review before import')
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|