Zayden-Jung
ec6076700b
Metadata integrity checks / metadata (push) Successful in 1m14s
Reviewed-on: #22
304 lines
18 KiB
Python
304 lines
18 KiB
Python
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import tempfile
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
from paper_atlas.collectors.public_openreview import (REVISION, SOURCE, collect,
|
|
negative_status, normalize)
|
|
from paper_atlas.db.store import Store
|
|
from paper_atlas.ingestion.records import ingest_file
|
|
from paper_atlas.audits.public_openreview import audit
|
|
|
|
|
|
class PublicOpenReviewTests(unittest.TestCase):
|
|
def row(self, **overrides):
|
|
return {'id': 'RealForum01', 'title': 'A public paper', 'abstract': 'An abstract',
|
|
'author': 'Anonymous;Jane Doe;private@example.org', 'track': 'main',
|
|
'status': 'Reject', 'keywords': 'planning;reasoning',
|
|
'profiles': {'private': 'secret'}, 'reviews': ['private review'],
|
|
'gender': 'irrelevant', **overrides}
|
|
|
|
def provenance(self):
|
|
return {'source_file': 'https://example.org/pinned.jsonl', 'source_line': 7,
|
|
'source_revision': REVISION, 'source_blob_sha1': 'a' * 40}
|
|
|
|
def fetcher(self, files):
|
|
tree = []
|
|
payloads = {}
|
|
for name, rows in files.items():
|
|
data = ''.join(json.dumps(row) + '\n' for row in rows).encode()
|
|
sha = hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()
|
|
tree.append({'path': name, 'type': 'blob', 'size': len(data), 'sha': sha})
|
|
payloads[name] = data
|
|
|
|
def fetch(url, limit):
|
|
if '?recursive=1' in url:
|
|
return json.dumps({'sha': REVISION, 'truncated': False, 'tree': tree}).encode()
|
|
return payloads[url.split('/' + REVISION + '/')[1]]
|
|
return fetch
|
|
|
|
def collect(self, out, files, **kwargs):
|
|
with patch('paper_atlas.collectors.public_openreview.shutil.disk_usage', return_value=type('Space', (), {'free': 20 * 1024**3})()):
|
|
return collect(out, fetcher=self.fetcher(files), **kwargs)
|
|
|
|
def test_only_explicit_decisions_and_verified_scope_become_events(self):
|
|
for value in ('Pending', 'Active', 'Submitted to NeurIPS 2025', 'TBD', '',
|
|
'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject',
|
|
'Workshop', 'Poster Workshop', 'Oral Workshop'):
|
|
self.assertIsNone(negative_status(value), value)
|
|
self.assertEqual(negative_status('Reject'), 'rejected')
|
|
self.assertEqual(negative_status('Desk rejected'), 'desk_rejected')
|
|
self.assertEqual(negative_status('Withdraw'), 'withdrawn')
|
|
self.assertEqual(negative_status('NeurIPS 2023 Conference Withdrawn Submission'), 'withdrawn')
|
|
for raw, expected in [('main', 'main'), ('Position', 'position'),
|
|
('Datasets & Benchmarks', 'datasets_benchmarks')]:
|
|
row, reason = normalize(self.row(track=raw), 'nips', 2025, self.provenance())
|
|
self.assertIsNone(reason)
|
|
self.assertEqual(row['track'], expected)
|
|
for raw in ('workshop', 'workshop_mlsys', ''):
|
|
self.assertIsNone(normalize(self.row(track=raw), 'icml', 2025, self.provenance())[0])
|
|
with self.assertRaisesRegex(ValueError, 'conference year'):
|
|
normalize(self.row(status='NeurIPS 2023 Conference Withdrawn Submission'), 'icml', 2025, self.provenance())
|
|
|
|
def test_metadata_whitelist_retains_curated_provenance_without_profiles_or_reviews(self):
|
|
row, _ = normalize(self.row(), 'icml', 2025, self.provenance())
|
|
self.assertEqual(row['authors'], ['Jane Doe'])
|
|
self.assertEqual(row['identifiers'], {'openreview': 'RealForum01'})
|
|
self.assertEqual(row['metadata']['evidence_kind'], 'curated_public_status_assertion')
|
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
|
self.assertEqual(row['evidence_url'], 'https://example.org/pinned.jsonl#L7')
|
|
for private in ('secret', 'private review', 'gender', 'private@example.org'):
|
|
self.assertNotIn(private, json.dumps(row))
|
|
|
|
def test_pinned_collection_separates_tracks_and_holds_retraction(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
files = {'venues/icml/icml2025.jsonl': [self.row(), self.row(id='Position01', track='Position'),
|
|
self.row(id='Retracted1', status='ICML 2025 Retracted by Authors'),
|
|
self.row(id='Pending001', status='Submitted to ICML 2025')]}
|
|
result = self.collect(tmp, files)
|
|
self.assertEqual(result['input_rows'], 4)
|
|
self.assertEqual(result['normalized_records'], 2)
|
|
self.assertEqual({f['track'] for f in result['files']}, {'main', 'position'})
|
|
held = [json.loads(line) for line in Path(result['held_file']).read_text().splitlines()]
|
|
self.assertEqual([(r['id'], r['reason']) for r in held], [('Retracted1', 'retracted')])
|
|
self.assertTrue(all(f['status'] == 'partial' for f in result['files']))
|
|
with self.assertRaisesRegex(ValueError, 'Completed inventory'):
|
|
self.collect(tmp, files)
|
|
|
|
def test_hash_failure_or_duplicate_forum_publishes_no_ingestible_files(self):
|
|
files = {'venues/icml/icml2025.jsonl': [self.row()]}
|
|
fetch = self.fetcher(files)
|
|
def corrupted(url, limit):
|
|
result = fetch(url, limit)
|
|
return result if '?recursive' in url else result.replace(b'Reject', b'Accept')
|
|
for fetcher in (corrupted, self.fetcher({'venues/icml/icml2025.jsonl': [self.row(), self.row()]})):
|
|
with tempfile.TemporaryDirectory() as tmp, patch('paper_atlas.collectors.public_openreview.shutil.disk_usage', return_value=type('Space', (), {'free': 20 * 1024**3})()):
|
|
with self.assertRaises(ValueError):
|
|
collect(tmp, fetcher=fetcher)
|
|
self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
|
|
self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
|
|
|
|
def test_missing_title_is_held_without_losing_valid_negative_records(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
result = self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [
|
|
self.row(id='SJUdkecgx', title=''),
|
|
self.row(id='NoTitle001', title=None),
|
|
self.row(id='Whitespace', title=' '),
|
|
self.row(id='ValidReject'),
|
|
self.row(id='Withdraw01', status='Withdraw'),
|
|
self.row(id='Workshop01', status='Workshop'),
|
|
]}, families=['iclr'], years=[2017])
|
|
self.assertEqual(result['input_rows'], 6)
|
|
self.assertEqual(result['normalized_records'], 2)
|
|
self.assertEqual(result['source_files'][0]['excluded_or_held'],
|
|
{'missing_title': 3, 'not_explicit_negative': 1})
|
|
held_path = Path(result['held_file'])
|
|
self.assertEqual(hashlib.sha256(held_path.read_bytes()).hexdigest(),
|
|
result['held_file_sha256'])
|
|
held = [json.loads(line) for line in held_path.read_text().splitlines()]
|
|
self.assertEqual([row['id'] for row in held], ['SJUdkecgx', 'NoTitle001', 'Whitespace'])
|
|
self.assertTrue(all(row['reason'] == 'missing_title' and row['status'] == 'Reject'
|
|
for row in held))
|
|
self.assertEqual([row['provenance']['source_line'] for row in held], [1, 2, 3])
|
|
self.assertNotIn('private@example.org', held_path.read_text())
|
|
self.assertNotIn('private review', held_path.read_text())
|
|
manifest = result['files'][0]
|
|
self.assertEqual((manifest['parser_version'], manifest['status']), (3, 'partial'))
|
|
rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
|
|
self.assertEqual([(row['identifiers']['openreview'], row['status']) for row in rows],
|
|
[('ValidReject', 'rejected'), ('Withdraw01', 'withdrawn')])
|
|
for row in held:
|
|
self.assertEqual(row['provenance']['source_revision'], REVISION)
|
|
|
|
def test_nontext_title_still_fails_collection(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
with self.assertRaisesRegex(ValueError, 'lacks a title'):
|
|
self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [self.row(title=['invalid'])]},
|
|
families=['iclr'], years=[2017])
|
|
self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
|
|
self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
|
|
|
|
def workshop_cases(self):
|
|
return [
|
|
(2024, 'workshop_me_fomo', 'ICLR 2024 Workshop MEFoMo Withdrawn Submission', 'withdrawn', 'ME-FoMo'),
|
|
(2024, 'workshop_genai4dm', 'ICLR 2024 Workshop GenAI4DM Withdrawn Submission', 'withdrawn', 'GenAI4DM'),
|
|
(2025, 'workshop_agenticai', 'ICLR 2025 Workshop AgenticAI Reject', 'rejected', 'AgenticAI'),
|
|
(2025, 'workshop_lmrl', 'ICLR 2025 Workshop LMRL Withdrawn Submission', 'withdrawn', 'LMRL'),
|
|
(2026, 'workshop_llm_reasoning', 'ICLR 2026 Workshop LLM Reasoning Withdrawn Submission', 'withdrawn', 'LLM_Reasoning'),
|
|
]
|
|
|
|
def test_iclr_workshop_labels_require_opt_in_and_retain_identity_evidence(self):
|
|
for year, track, label, status, group in self.workshop_cases():
|
|
with self.subTest(label=label):
|
|
raw = self.row(track=track, status=label)
|
|
self.assertEqual(normalize(raw, 'iclr', year, self.provenance()),
|
|
(None, 'separate_workshop'))
|
|
row, reason = normalize(raw, 'iclr', year, self.provenance(), True)
|
|
self.assertIsNone(reason)
|
|
self.assertEqual((row['venue'], row['year'], row['track'], row['status']),
|
|
('ICLR', year, track, status))
|
|
self.assertEqual(row['raw_decision'], label)
|
|
self.assertEqual(row['metadata']['workshop_venueid'],
|
|
f'ICLR.cc/{year}/Workshop/{group}')
|
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
|
self.assertEqual(row['evidence_url'], 'https://example.org/pinned.jsonl#L7')
|
|
self.assertNotIn('private review', json.dumps(row))
|
|
|
|
def test_workshop_label_scope_conflicts_never_become_main_events(self):
|
|
for year, track, label, _, _ in self.workshop_cases():
|
|
for enabled in (False, True):
|
|
for wrong_track in ('main', 'workshop_other'):
|
|
self.assertEqual(normalize(self.row(track=wrong_track, status=label),
|
|
'iclr', year, self.provenance(), enabled),
|
|
(None, 'conflicting_workshop_track'))
|
|
for family, wrong_year in [('icml', year), ('iclr', year + 1)]:
|
|
with self.assertRaisesRegex(ValueError, 'conference year'):
|
|
normalize(self.row(track=track, status=label), family,
|
|
wrong_year, self.provenance(), enabled)
|
|
row, reason = normalize(self.row(track='workshop_unknown',
|
|
status='ICLR 2026 Workshop Unknown Reject'),
|
|
'iclr', 2026, self.provenance(), True)
|
|
self.assertIsNone(row)
|
|
self.assertEqual(reason, 'unmapped_negative_label')
|
|
|
|
def test_workshop_collection_and_import_keep_five_tracks_separate_from_main(self):
|
|
files = {}
|
|
for index, (year, track, label, _, _) in enumerate(self.workshop_cases()):
|
|
files.setdefault(f'venues/iclr/iclr{year}.jsonl', []).append(
|
|
self.row(id=f'Workshop{index:02}', track=track, status=label))
|
|
files['venues/iclr/iclr2024.jsonl'].append(self.row())
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
default = self.collect(root / 'default', files, families=['iclr'])
|
|
self.assertEqual(default['normalized_records'], 1)
|
|
held = [json.loads(line) for line in Path(default['held_file']).read_bytes().split(b'\n') if line]
|
|
self.assertEqual(len(held), 5)
|
|
self.assertTrue(all(row['reason'] == 'separate_workshop' for row in held))
|
|
selected = self.collect(root / 'selected', files, families=['iclr'], include_workshops=True)
|
|
self.assertEqual(selected['normalized_records'], 6)
|
|
self.assertEqual(len(selected['files']), 6)
|
|
self.assertEqual(Path(selected['held_file']).read_bytes(), b'')
|
|
db = root / 'db.sqlite'
|
|
for manifest in selected['files']:
|
|
self.assertEqual(manifest['parser_version'], 3)
|
|
ingest_file(db, manifest['file'])
|
|
store = Store(db)
|
|
try:
|
|
scopes = set(store.conn.execute('SELECT e.year,e.track,d.status FROM submissions s '
|
|
'JOIN editions e ON e.id=s.edition_id '
|
|
'JOIN decisions d ON d.submission_id=s.id'))
|
|
expected = {(year, track, status) for year, track, _, status, _ in self.workshop_cases()}
|
|
self.assertEqual(scopes, expected | {(2024, 'main', 'rejected')})
|
|
self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(), [])
|
|
finally:
|
|
store.conn.close()
|
|
|
|
def test_mirror_reimport_preserves_one_attempt_and_later_acceptance_elsewhere(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
result = self.collect(root / 'source', {'venues/icml/icml2025.jsonl': [self.row(author='Jane Doe')]})
|
|
db = root / 'db.sqlite'
|
|
path = result['files'][0]['file']
|
|
ingest_file(db, path)
|
|
ingest_file(db, path)
|
|
store = Store(db)
|
|
try:
|
|
paper = store.conn.execute('SELECT paper_id FROM source_records').fetchone()[0]
|
|
store.conn.execute('INSERT INTO identifiers VALUES(?,?,?)', ('arxiv', '2501.00001', paper))
|
|
same, record = store.record('official_program', 'neurips-later', 'https://example.org/accepted',
|
|
'A revised public paper', 'An abstract', ['Jane Doe'],
|
|
{'openreview': 'LaterForum1', 'arxiv': '2501.00001'})
|
|
self.assertEqual(same, paper)
|
|
store.decision(paper, record, 'NeurIPS', 2026, 'main', 'neurips-later',
|
|
'https://example.org/accepted', 'accepted', 'Accept',
|
|
identities=[('openreview', 'LaterForum1')])
|
|
store.conn.commit()
|
|
self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0], 2)
|
|
self.assertEqual(store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0], 2)
|
|
self.assertEqual(set(store.conn.execute('SELECT status FROM decisions')), {('accepted',), ('rejected',)})
|
|
self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
|
|
self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(), [])
|
|
finally:
|
|
store.conn.close()
|
|
|
|
def test_preimport_audit_blocks_same_attempt_conflict_but_allows_later_other_venue_acceptance(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
self.collect(root / 'source', {'venues/icml/icml2025.jsonl': [self.row(author='Jane Doe')]})
|
|
db = root / 'db.sqlite'
|
|
store = Store(db)
|
|
paper, record = store.record('program', 'later', 'https://example.org/later',
|
|
'A public paper', 'An abstract', ['Jane Doe'], {'openreview': 'LaterForum1'})
|
|
store.decision(paper, record, 'NeurIPS', 2026, 'main', 'later',
|
|
'https://example.org/later', 'accepted', 'Accept', identities=[('openreview', 'LaterForum1')])
|
|
store.conn.commit()
|
|
inventory = root / 'source/public-openreview-inventory.json'
|
|
first = audit(db, inventory, root / 'audit.json')
|
|
self.assertEqual(first['status'], 'ready_for_serial_import')
|
|
self.assertEqual(len(first['files']), 1)
|
|
store.decision(paper, record, 'ICML', 2025, 'main', 'same-attempt',
|
|
'https://example.org/same', 'accepted', 'Accept', identities=[('openreview', 'RealForum01')])
|
|
store.conn.commit()
|
|
try:
|
|
second = audit(db, inventory, root / 'audit.json')
|
|
self.assertEqual(second['status'], 'blocked')
|
|
self.assertEqual(second['files'], [])
|
|
self.assertEqual(second['conflicts'][0]['existing_status'], 'accepted')
|
|
finally:
|
|
store.conn.close()
|
|
|
|
def test_unicode_separators_in_abstract_survive_audit_and_import(self):
|
|
for separator in ('\u0085', '\u2028', '\u2029'):
|
|
with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
abstract = 'First paragraph' + separator + 'Second paragraph'
|
|
result = self.collect(root / 'source', {
|
|
'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]})
|
|
path = Path(result['files'][0]['file'])
|
|
data = path.read_bytes()
|
|
self.assertIn(separator.encode(), data)
|
|
self.assertEqual(data.count(b'\n'), 1)
|
|
db = root / 'db.sqlite'
|
|
store = Store(db)
|
|
store.conn.close()
|
|
inventory = root / 'source/public-openreview-inventory.json'
|
|
before = audit(db, inventory, root / 'before.json')
|
|
self.assertEqual(before['status'], 'ready_for_serial_import')
|
|
self.assertEqual(before['records_checked'], 1)
|
|
ingest_file(db, path)
|
|
after = audit(db, inventory, root / 'after.json')
|
|
self.assertEqual(after['status'], 'ready_for_serial_import')
|
|
self.assertEqual(after['existing_same_edition_attempts'], 1)
|
|
store = Store(db)
|
|
try:
|
|
self.assertEqual(store.conn.execute(
|
|
'SELECT abstract FROM source_records').fetchone()[0], abstract)
|
|
finally:
|
|
store.conn.close()
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|