Add snapshot-bound publication source scope revisions
Metadata integrity checks / metadata (push) Successful in 7m16s
Metadata integrity checks / metadata (push) Successful in 7m16s
This commit is contained in:
@@ -0,0 +1,209 @@
|
||||
"""Whole-source scope correction, stale refresh, full history and atomicity on both engines."""
|
||||
from contextlib import closing, redirect_stdout
|
||||
import copy
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
import uuid
|
||||
|
||||
from paper_atlas.db.backend import open_store
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
from paper_atlas.db.source_revisions import available, require_row, require_manifest
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
|
||||
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||
from paper_atlas.ingestion.records import coverage_options, ingest_file
|
||||
from paper_atlas.postgres.client import connect
|
||||
from paper_atlas.postgres.migration import migrate
|
||||
from paper_atlas.postgres.schema import quote
|
||||
|
||||
URL = 'https://example.org/volume'
|
||||
|
||||
|
||||
def seed(path):
|
||||
old = []
|
||||
for i in range(4):
|
||||
doi = '10.1234/publication'+str(i)
|
||||
old.append({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'main' if i == 0 else 'unclassified',
|
||||
'external_id': doi, 'url': 'https://doi.org/'+doi, 'title': 'Distinct publication '+str(i),
|
||||
'authors': ['Alice Author', 'Bob Writer'], 'abstract': None if i == 1 else 'Source abstract '+str(i),
|
||||
'identifiers': {'doi': doi}, 'submission_identifiers': {'publication_doi': doi},
|
||||
'status': 'accepted', 'raw_decision': 'Publication observation', 'evidence_url': URL,
|
||||
'observed_at': '2026-06-01T00:00:00Z', 'tags': [{'namespace': 'fixture:track', 'value': 'main' if i == 0 else 'unclassified'}],
|
||||
'metadata': {'original': i, 'program': None, 'scientific_work_identity_approved': False}})
|
||||
current = copy.deepcopy(old)
|
||||
current[1].update(track='main', abstract='Reviewed program abstract with NUL\x00 and SOH\x01')
|
||||
current[1]['metadata']['program'] = {'id': 'p1', 'evidence_url': URL}
|
||||
current[1]['tags'][0]['value'] = 'main'
|
||||
store = Store(path)
|
||||
for row in old[:3]:
|
||||
p, r = store.record(row['source'], row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
|
||||
store.decision(p, r, 'MM', 2026, row['track'], row['external_id'], URL, 'accepted', row['raw_decision'],
|
||||
identities=[('publication_doi', row['external_id'])], observed_at=row['observed_at'])
|
||||
store.record_tags(p, r, 'fixture', row['tags'])
|
||||
if row == old[1]:
|
||||
store.conn.execute('UPDATE papers SET abstract=? WHERE id=?', ('Canonical preprint abstract', p))
|
||||
p, other = store.record('prior', 'old-forum', 'https://example.org/prior', row['title'], 'Canonical preprint abstract', row['authors'], row['identifiers'])
|
||||
store.decision(p, other, 'PRIOR', 2025, 'main', 'old-forum', 'https://example.org/prior', 'rejected', 'Reject', observed_at='2025-01-01T00:00:00Z')
|
||||
# An unrelated source already has an attempt in the populated target edition.
|
||||
p, r = store.record('other', 'other', 'https://example.org/other', 'Unrelated target paper', 'Other abstract', ['Someone Else'])
|
||||
store.decision(p, r, 'MM', 2026, 'main', 'other', 'https://example.org/other', 'accepted', 'Published')
|
||||
sid = store.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||
for track, expected, imported in [('main', 1, 1), ('unclassified', 3, 2)]:
|
||||
eid = store.conn.execute('SELECT id FROM editions WHERE year=2026 AND track=?', (track,)).fetchone()[0]
|
||||
store.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'partial',?,?,?,'2026-06-01',NULL)", (eid, sid, URL, expected, imported, 'checkpoint'))
|
||||
store.conn.commit(); store.conn.close()
|
||||
return {'format': 1, 'source': 'fixture', 'venue': 'MM', 'year': 2026, 'event_key': '', 'list_url': URL,
|
||||
'previous_rows': old, 'rows': current, 'evidence': [{'url': URL, 'sha256': 'a'*64}]}
|
||||
|
||||
|
||||
class RevisionCases:
|
||||
def store(self):
|
||||
return open_store(**self.target)
|
||||
|
||||
def plan(self, c):
|
||||
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
|
||||
|
||||
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
|
||||
self.assertEqual(len(plan['before']['packet']['submissions']), 5)
|
||||
before_papers = plan['before']['packet']['papers']
|
||||
before_tags = plan['before']['packet']['paper_tags']
|
||||
result = apply_plan(c, plan); c.commit()
|
||||
self.assertEqual((result['events_moved'], result['records_changed']), (1, 1))
|
||||
self.assertEqual(result['protected_decisions'], 5)
|
||||
self.assertEqual(values(c.execute('SELECT * FROM papers ORDER BY id')), before_papers)
|
||||
self.assertEqual(values(c.execute('SELECT * FROM paper_tags ORDER BY paper_id,tag_id,evidence_key')), before_tags)
|
||||
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status='rejected'").fetchone()[0], 1)
|
||||
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources ORDER BY edition_id')),
|
||||
[['partial', 2, 2, 'checkpoint', '2026-06-01', None], ['partial', 2, 1, 'checkpoint', '2026-06-01', None]])
|
||||
saved = json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0])
|
||||
self.assertEqual(saved, plan)
|
||||
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
|
||||
|
||||
def test_stale_direct_store_and_jsonl_refresh_fail_but_current_refresh_succeeds(self):
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c); apply_plan(c, plan); c.commit()
|
||||
with closing(self.store().conn) as c:
|
||||
store = object.__new__(Store); store.conn = c; store.event_identity = True
|
||||
old = self.inventory['previous_rows'][1]
|
||||
with self.assertRaisesRegex(ValueError, 'Stale source'):
|
||||
store.record('fixture', old['external_id'], old['url'], old['title'], old['abstract'], old['authors'], old['identifiers'], old['metadata'])
|
||||
c.rollback()
|
||||
row = self.inventory['rows'][1]
|
||||
p, r = store.record('fixture', row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
|
||||
self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (p,)).fetchone()[0], 'Canonical preprint abstract')
|
||||
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?', (r,)).fetchone()[0])[KEY][0]['target']['track'], 'main')
|
||||
c.rollback()
|
||||
path = self.root/'old.jsonl'; raw = (json.dumps(old)+'\n').encode(); path.write_bytes(raw)
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'fixture', 'venue': 'MM', 'year': 2026,
|
||||
'track': 'unclassified', 'list_url': URL, 'status': 'partial', 'count': 1, 'file_sha256': hashlib.sha256(raw).hexdigest()}))
|
||||
with self.assertRaisesRegex(ValueError, 'source snapshot'):
|
||||
ingest_file(self.target.get('db'), path, **{k: v for k, v in self.target.items() if k != 'db'})
|
||||
|
||||
def test_unregistered_member_and_changed_program_metadata_are_rejected(self):
|
||||
with closing(self.store().conn) as c:
|
||||
apply_plan(c, self.plan(c)); c.commit()
|
||||
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||
cache = {}; unknown = copy.deepcopy(self.inventory['rows'][0]); unknown['external_id'] = '10.1234/unlisted'
|
||||
with self.assertRaisesRegex(ValueError, 'absent from current'):
|
||||
require_row(c, sid, unknown, {'list_url': URL}, cache)
|
||||
bad = copy.deepcopy(self.inventory['rows'][1]); bad['metadata']['program'] = None
|
||||
with self.assertRaisesRegex(ValueError, 'Stale source'):
|
||||
require_row(c, sid, bad, {'list_url': URL}, cache)
|
||||
good = copy.deepcopy(self.inventory['rows'][1]); good['metadata']['verified_work_anchor'] = {'reason': 'extra evidence'}
|
||||
require_row(c, sid, good, {'list_url': URL}, cache)
|
||||
with self.assertRaisesRegex(ValueError, 'coverage manifest'):
|
||||
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 1, 'status': 'partial'}, cache)
|
||||
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 2, 'status': 'partial'}, cache)
|
||||
|
||||
def test_stale_plan_and_target_identifier_collision_prevent_writes(self):
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c)
|
||||
c.execute("UPDATE decisions SET raw_value='Additional evidence' WHERE status='rejected'")
|
||||
with self.assertRaisesRegex(ValueError, 'Stale'):
|
||||
apply_plan(c, plan)
|
||||
c.rollback()
|
||||
move = plan['changes'][0]
|
||||
target_sub = c.execute('SELECT id FROM submissions WHERE edition_id=? LIMIT 1', (move['target_edition_id'],)).fetchone()[0]
|
||||
c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?)', (move['target_edition_id'], 'publication_doi', move['external_id'], target_sub))
|
||||
with self.assertRaisesRegex(ValueError, 'collision'):
|
||||
self.plan(c)
|
||||
c.rollback()
|
||||
self.assertFalse(available(c))
|
||||
|
||||
def test_failed_postcheck_rolls_back_rows_extension_and_coverage(self):
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c)
|
||||
with patch('paper_atlas.ingestion.source_revisions.verify', side_effect=ValueError('Injected verification failure')):
|
||||
with self.assertRaisesRegex(ValueError, 'Injected'):
|
||||
apply_plan(c, plan)
|
||||
c.rollback()
|
||||
self.assertFalse(available(c))
|
||||
self.assertEqual(self.plan(c), plan)
|
||||
|
||||
def test_cli_binds_database_and_file_hash_without_writing_during_preflight(self):
|
||||
inventory = self.root/'inventory.json'; inventory.write_text(json.dumps(self.inventory))
|
||||
path = self.root/'plan.json'
|
||||
args = {k: v for k, v in self.target.items() if k != 'db'}
|
||||
run(self.target.get('db'), path, inventory=inventory, **args)
|
||||
with closing(self.store().conn) as c:
|
||||
self.assertFalse(available(c))
|
||||
with self.assertRaisesRegex(ValueError, 'hash'):
|
||||
run(self.target.get('db'), self.root/'bad.json', plan=path, plan_sha256='0'*64, **args)
|
||||
report = run(self.target.get('db'), self.root/'applied.json', plan=path, plan_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), **args)
|
||||
self.assertEqual(report['events_moved'], 1)
|
||||
|
||||
def test_membership_identity_changes_and_incremental_shrink_remain_forbidden(self):
|
||||
for mutate in [lambda x: x['rows'].pop(), lambda x: x['rows'][0].update(title='Different work'),
|
||||
lambda x: x['rows'][0].update(status='rejected')]:
|
||||
bad = copy.deepcopy(self.inventory); mutate(bad)
|
||||
with self.assertRaises(ValueError):
|
||||
validate(bad)
|
||||
with self.assertRaisesRegex(ValueError, 'incremental'):
|
||||
coverage_options({'coverage_mode': 'incremental', 'venue': 'MM', 'year': 2026, 'track': 'unclassified',
|
||||
'status': 'partial', 'count': 1, 'coverage_previous_count': 2,
|
||||
'coverage_previous_expected_count': 3, 'coverage_expected_count': 2})
|
||||
|
||||
|
||||
class SQLiteSourceRevisionTests(RevisionCases, unittest.TestCase):
|
||||
def setUp(self):
|
||||
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
|
||||
db = self.root/'source.sqlite'; self.inventory = seed(db); self.target = {'db': db}
|
||||
|
||||
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
|
||||
def test_optional_revision_table_migrates_with_full_history(self):
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c); apply_plan(c, plan); c.commit()
|
||||
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_copy_'+uuid.uuid4().hex[:10]
|
||||
self.addCleanup(drop_schema, service, schema)
|
||||
with redirect_stdout(io.StringIO()):
|
||||
migrate(self.target['db'], service, self.root/'copied.json', schema=schema, min_free_gib=0)
|
||||
with closing(open_store(pg_service=service, pg_schema=schema).conn) as c:
|
||||
self.assertEqual(json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]), plan)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
|
||||
|
||||
|
||||
def drop_schema(service, schema):
|
||||
with connect(service) as c:
|
||||
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
|
||||
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
|
||||
class PostgreSQLSourceRevisionTests(RevisionCases, unittest.TestCase):
|
||||
def setUp(self):
|
||||
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
|
||||
db = self.root/'source.sqlite'; self.inventory = seed(db)
|
||||
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_'+uuid.uuid4().hex[:12]
|
||||
self.addCleanup(drop_schema, service, schema)
|
||||
with redirect_stdout(io.StringIO()):
|
||||
migrate(db, service, self.root/'migration.json', schema=schema, min_free_gib=0)
|
||||
self.target = {'pg_service': service, 'pg_schema': schema}
|
||||
Reference in New Issue
Block a user