Files
paper-atlas/tests/test_source_revisions.py
T
codex-bot 2000d340fa
Metadata integrity checks / metadata (push) Successful in 7m16s
Add snapshot-bound publication source scope revisions
2026-11-18 06:59:21 +08:00

210 lines
14 KiB
Python

"""Whole-source scope correction, stale refresh, full history and atomicity on both engines."""
from contextlib import closing, redirect_stdout
import copy
import hashlib
import io
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import uuid
from paper_atlas.db.backend import open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.source_revisions import available, require_row, require_manifest
from paper_atlas.db.store import Store
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
from paper_atlas.ingestion.event_identity import database_identity, digest, values
from paper_atlas.ingestion.records import coverage_options, ingest_file
from paper_atlas.postgres.client import connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
URL = 'https://example.org/volume'
def seed(path):
old = []
for i in range(4):
doi = '10.1234/publication'+str(i)
old.append({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'main' if i == 0 else 'unclassified',
'external_id': doi, 'url': 'https://doi.org/'+doi, 'title': 'Distinct publication '+str(i),
'authors': ['Alice Author', 'Bob Writer'], 'abstract': None if i == 1 else 'Source abstract '+str(i),
'identifiers': {'doi': doi}, 'submission_identifiers': {'publication_doi': doi},
'status': 'accepted', 'raw_decision': 'Publication observation', 'evidence_url': URL,
'observed_at': '2026-06-01T00:00:00Z', 'tags': [{'namespace': 'fixture:track', 'value': 'main' if i == 0 else 'unclassified'}],
'metadata': {'original': i, 'program': None, 'scientific_work_identity_approved': False}})
current = copy.deepcopy(old)
current[1].update(track='main', abstract='Reviewed program abstract with NUL\x00 and SOH\x01')
current[1]['metadata']['program'] = {'id': 'p1', 'evidence_url': URL}
current[1]['tags'][0]['value'] = 'main'
store = Store(path)
for row in old[:3]:
p, r = store.record(row['source'], row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
store.decision(p, r, 'MM', 2026, row['track'], row['external_id'], URL, 'accepted', row['raw_decision'],
identities=[('publication_doi', row['external_id'])], observed_at=row['observed_at'])
store.record_tags(p, r, 'fixture', row['tags'])
if row == old[1]:
store.conn.execute('UPDATE papers SET abstract=? WHERE id=?', ('Canonical preprint abstract', p))
p, other = store.record('prior', 'old-forum', 'https://example.org/prior', row['title'], 'Canonical preprint abstract', row['authors'], row['identifiers'])
store.decision(p, other, 'PRIOR', 2025, 'main', 'old-forum', 'https://example.org/prior', 'rejected', 'Reject', observed_at='2025-01-01T00:00:00Z')
# An unrelated source already has an attempt in the populated target edition.
p, r = store.record('other', 'other', 'https://example.org/other', 'Unrelated target paper', 'Other abstract', ['Someone Else'])
store.decision(p, r, 'MM', 2026, 'main', 'other', 'https://example.org/other', 'accepted', 'Published')
sid = store.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
for track, expected, imported in [('main', 1, 1), ('unclassified', 3, 2)]:
eid = store.conn.execute('SELECT id FROM editions WHERE year=2026 AND track=?', (track,)).fetchone()[0]
store.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'partial',?,?,?,'2026-06-01',NULL)", (eid, sid, URL, expected, imported, 'checkpoint'))
store.conn.commit(); store.conn.close()
return {'format': 1, 'source': 'fixture', 'venue': 'MM', 'year': 2026, 'event_key': '', 'list_url': URL,
'previous_rows': old, 'rows': current, 'evidence': [{'url': URL, 'sha256': 'a'*64}]}
class RevisionCases:
def store(self):
return open_store(**self.target)
def plan(self, c):
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
with closing(self.store().conn) as c:
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
self.assertEqual(len(plan['before']['packet']['submissions']), 5)
before_papers = plan['before']['packet']['papers']
before_tags = plan['before']['packet']['paper_tags']
result = apply_plan(c, plan); c.commit()
self.assertEqual((result['events_moved'], result['records_changed']), (1, 1))
self.assertEqual(result['protected_decisions'], 5)
self.assertEqual(values(c.execute('SELECT * FROM papers ORDER BY id')), before_papers)
self.assertEqual(values(c.execute('SELECT * FROM paper_tags ORDER BY paper_id,tag_id,evidence_key')), before_tags)
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status='rejected'").fetchone()[0], 1)
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources ORDER BY edition_id')),
[['partial', 2, 2, 'checkpoint', '2026-06-01', None], ['partial', 2, 1, 'checkpoint', '2026-06-01', None]])
saved = json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0])
self.assertEqual(saved, plan)
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
def test_stale_direct_store_and_jsonl_refresh_fail_but_current_refresh_succeeds(self):
with closing(self.store().conn) as c:
plan = self.plan(c); apply_plan(c, plan); c.commit()
with closing(self.store().conn) as c:
store = object.__new__(Store); store.conn = c; store.event_identity = True
old = self.inventory['previous_rows'][1]
with self.assertRaisesRegex(ValueError, 'Stale source'):
store.record('fixture', old['external_id'], old['url'], old['title'], old['abstract'], old['authors'], old['identifiers'], old['metadata'])
c.rollback()
row = self.inventory['rows'][1]
p, r = store.record('fixture', row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (p,)).fetchone()[0], 'Canonical preprint abstract')
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?', (r,)).fetchone()[0])[KEY][0]['target']['track'], 'main')
c.rollback()
path = self.root/'old.jsonl'; raw = (json.dumps(old)+'\n').encode(); path.write_bytes(raw)
path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'fixture', 'venue': 'MM', 'year': 2026,
'track': 'unclassified', 'list_url': URL, 'status': 'partial', 'count': 1, 'file_sha256': hashlib.sha256(raw).hexdigest()}))
with self.assertRaisesRegex(ValueError, 'source snapshot'):
ingest_file(self.target.get('db'), path, **{k: v for k, v in self.target.items() if k != 'db'})
def test_unregistered_member_and_changed_program_metadata_are_rejected(self):
with closing(self.store().conn) as c:
apply_plan(c, self.plan(c)); c.commit()
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
cache = {}; unknown = copy.deepcopy(self.inventory['rows'][0]); unknown['external_id'] = '10.1234/unlisted'
with self.assertRaisesRegex(ValueError, 'absent from current'):
require_row(c, sid, unknown, {'list_url': URL}, cache)
bad = copy.deepcopy(self.inventory['rows'][1]); bad['metadata']['program'] = None
with self.assertRaisesRegex(ValueError, 'Stale source'):
require_row(c, sid, bad, {'list_url': URL}, cache)
good = copy.deepcopy(self.inventory['rows'][1]); good['metadata']['verified_work_anchor'] = {'reason': 'extra evidence'}
require_row(c, sid, good, {'list_url': URL}, cache)
with self.assertRaisesRegex(ValueError, 'coverage manifest'):
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 1, 'status': 'partial'}, cache)
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 2, 'status': 'partial'}, cache)
def test_stale_plan_and_target_identifier_collision_prevent_writes(self):
with closing(self.store().conn) as c:
plan = self.plan(c)
c.execute("UPDATE decisions SET raw_value='Additional evidence' WHERE status='rejected'")
with self.assertRaisesRegex(ValueError, 'Stale'):
apply_plan(c, plan)
c.rollback()
move = plan['changes'][0]
target_sub = c.execute('SELECT id FROM submissions WHERE edition_id=? LIMIT 1', (move['target_edition_id'],)).fetchone()[0]
c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?)', (move['target_edition_id'], 'publication_doi', move['external_id'], target_sub))
with self.assertRaisesRegex(ValueError, 'collision'):
self.plan(c)
c.rollback()
self.assertFalse(available(c))
def test_failed_postcheck_rolls_back_rows_extension_and_coverage(self):
with closing(self.store().conn) as c:
plan = self.plan(c)
with patch('paper_atlas.ingestion.source_revisions.verify', side_effect=ValueError('Injected verification failure')):
with self.assertRaisesRegex(ValueError, 'Injected'):
apply_plan(c, plan)
c.rollback()
self.assertFalse(available(c))
self.assertEqual(self.plan(c), plan)
def test_cli_binds_database_and_file_hash_without_writing_during_preflight(self):
inventory = self.root/'inventory.json'; inventory.write_text(json.dumps(self.inventory))
path = self.root/'plan.json'
args = {k: v for k, v in self.target.items() if k != 'db'}
run(self.target.get('db'), path, inventory=inventory, **args)
with closing(self.store().conn) as c:
self.assertFalse(available(c))
with self.assertRaisesRegex(ValueError, 'hash'):
run(self.target.get('db'), self.root/'bad.json', plan=path, plan_sha256='0'*64, **args)
report = run(self.target.get('db'), self.root/'applied.json', plan=path, plan_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), **args)
self.assertEqual(report['events_moved'], 1)
def test_membership_identity_changes_and_incremental_shrink_remain_forbidden(self):
for mutate in [lambda x: x['rows'].pop(), lambda x: x['rows'][0].update(title='Different work'),
lambda x: x['rows'][0].update(status='rejected')]:
bad = copy.deepcopy(self.inventory); mutate(bad)
with self.assertRaises(ValueError):
validate(bad)
with self.assertRaisesRegex(ValueError, 'incremental'):
coverage_options({'coverage_mode': 'incremental', 'venue': 'MM', 'year': 2026, 'track': 'unclassified',
'status': 'partial', 'count': 1, 'coverage_previous_count': 2,
'coverage_previous_expected_count': 3, 'coverage_expected_count': 2})
class SQLiteSourceRevisionTests(RevisionCases, unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
db = self.root/'source.sqlite'; self.inventory = seed(db); self.target = {'db': db}
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
def test_optional_revision_table_migrates_with_full_history(self):
with closing(self.store().conn) as c:
plan = self.plan(c); apply_plan(c, plan); c.commit()
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_copy_'+uuid.uuid4().hex[:10]
self.addCleanup(drop_schema, service, schema)
with redirect_stdout(io.StringIO()):
migrate(self.target['db'], service, self.root/'copied.json', schema=schema, min_free_gib=0)
with closing(open_store(pg_service=service, pg_schema=schema).conn) as c:
self.assertEqual(json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]), plan)
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
def drop_schema(service, schema):
with connect(service) as c:
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
class PostgreSQLSourceRevisionTests(RevisionCases, unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
db = self.root/'source.sqlite'; self.inventory = seed(db)
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_'+uuid.uuid4().hex[:12]
self.addCleanup(drop_schema, service, schema)
with redirect_stdout(io.StringIO()):
migrate(db, service, self.root/'migration.json', schema=schema, min_free_gib=0)
self.target = {'pg_service': service, 'pg_schema': schema}