Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ae8bcd4c14 | |||
| 3acb738aa8 | |||
| cbf7bab8ca |
@@ -151,4 +151,4 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
|||||||
|
|
||||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||||
|
|
||||||
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。
|
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。同届节目投稿可按独立审核计划附加出版证据,保留原投稿 ID 和既往决定。
|
||||||
|
|||||||
@@ -14,3 +14,18 @@ paper-atlas collect eccv2026-publications \
|
|||||||
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
||||||
|
|
||||||
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
||||||
|
|
||||||
|
## 已有节目投稿的出版证据
|
||||||
|
|
||||||
|
先在只读快照上复核节目 ID、章节 DOI、标题、完整有序作者、摘要版本与当前投稿历史。保存计划和原始证据,计划应列出每条出版观察、保留的论文与投稿 ID、节目来源和决定 ID、当前与目标届次及完整历史包的 SHA-256。作者或版本仍有疑点的条目继续留在待核清单。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas maintain eccv-publication-reconciliation \
|
||||||
|
--pg-service <writer-service> --pg-schema research \
|
||||||
|
--plan <reviewed-plan.json> --plan-sha256 <reviewed-plan-sha256> \
|
||||||
|
--springer-previous-count <last-verified-count>
|
||||||
|
```
|
||||||
|
|
||||||
|
命令接收 `readonly_same_submission_reconciliation_plan` 格式的已复核计划。它会在同一事务内核对实时历史和来源覆盖,把原节目投稿归入主会,追加出版来源与观测决定,并分别调整节目与 Springer 的覆盖数。节目主会覆盖保持 partial,因为其目录的主会分母尚未独立核定。随后通过只读连接核验投稿、决定及覆盖,保存计划和回执。相同计划只允许成功提交一次。若执行中失败,先查同一计划的提交标记、覆盖和投稿来源,再决定是否重新执行。
|
||||||
|
|
||||||
|
旧节目快照的全量未分类 manifest 无法刷新已有逐条更正的投稿。继续刷新时需按已复核轨道分别整理输出并生成 manifest。
|
||||||
|
|||||||
@@ -173,34 +173,40 @@ def find_candidates(connection, records):
|
|||||||
records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
|
records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
|
||||||
|
|
||||||
|
|
||||||
|
def _named_rows(connection, sql, params):
|
||||||
|
cursor = connection.execute(sql, params)
|
||||||
|
columns = [field[0] for field in cursor.description]
|
||||||
|
return [{name: row[i] for i, name in enumerate(columns)} for row in cursor]
|
||||||
|
|
||||||
|
|
||||||
def entity_packet(connection, paper_id):
|
def entity_packet(connection, paper_id):
|
||||||
paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone()
|
papers = _named_rows(connection, 'SELECT * FROM papers WHERE id=?', (paper_id,))
|
||||||
if paper is None:
|
if not papers:
|
||||||
raise RuntimeError('Candidate disappeared from read-only snapshot')
|
raise RuntimeError('Candidate disappeared from read-only snapshot')
|
||||||
sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source
|
sources = _named_rows(connection, '''SELECT r.*,s.name AS source
|
||||||
FROM source_records r JOIN sources s ON s.id=r.source_id
|
FROM source_records r JOIN sources s ON s.id=r.source_id
|
||||||
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))]
|
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))
|
||||||
attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
|
attempts = _named_rows(connection, '''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
|
||||||
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
||||||
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))]
|
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))
|
||||||
packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute(
|
packet = {'paper': papers[0], 'identifiers': _named_rows(connection,
|
||||||
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))],
|
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,)),
|
||||||
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
|
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
|
||||||
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
|
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
|
||||||
for attempt in packet['attempts']:
|
for attempt in packet['attempts']:
|
||||||
observations = [dict(r) for r in connection.execute(
|
observations = _named_rows(connection,
|
||||||
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))]
|
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))
|
||||||
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
|
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
|
||||||
provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source
|
provenance = _named_rows(connection, '''SELECT ss.*,r.external_id,s.name AS source
|
||||||
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
|
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
|
||||||
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
|
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
|
||||||
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))]
|
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))
|
||||||
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
|
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
|
||||||
for decision in attempt['decisions']:
|
for decision in attempt['decisions']:
|
||||||
provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source
|
provenance = _named_rows(connection, '''SELECT ds.*,r.external_id,s.name AS source
|
||||||
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
|
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
|
||||||
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
|
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
|
||||||
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))]
|
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))
|
||||||
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
|
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
|
||||||
return packet
|
return packet
|
||||||
|
|
||||||
|
|||||||
@@ -49,6 +49,7 @@ COMMANDS = {
|
|||||||
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
||||||
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
||||||
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
||||||
|
('maintain','eccv-publication-reconciliation'): ('paper_atlas.ingestion.publication_reconciliation', {}, '向既有 ECCV 2026 投稿附加已复核的正式出版证据'),
|
||||||
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
|
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
|
||||||
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
||||||
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
||||||
|
|||||||
@@ -0,0 +1,222 @@
|
|||||||
|
"""Attach reviewed ECCV proceedings evidence to an existing programme attempt."""
|
||||||
|
import argparse
|
||||||
|
from contextlib import closing
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.audits.source_identity import entity_packet
|
||||||
|
from paper_atlas.collectors.eccv2026 import SOURCE as PROGRAMME
|
||||||
|
from paper_atlas.collectors.eccv2026_publications import SOURCE as PUBLISHER
|
||||||
|
from paper_atlas.db.backend import database_arguments, open_store
|
||||||
|
from paper_atlas.db.edition_scopes import KEY
|
||||||
|
from paper_atlas.db.submissions import identities_for_row, timestamp
|
||||||
|
from paper_atlas.ingestion.edition_tracks import curation
|
||||||
|
from paper_atlas.ingestion.records import source_record_count
|
||||||
|
|
||||||
|
MARKER = 'eccv_publication_reconciliation_v1:'
|
||||||
|
PUBLISHER_COUNT = 2835
|
||||||
|
|
||||||
|
|
||||||
|
def require(condition, message):
|
||||||
|
if not condition:
|
||||||
|
raise ValueError(message)
|
||||||
|
|
||||||
|
|
||||||
|
def packet_digest(packet):
|
||||||
|
return sha256(json.dumps(packet, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def validate_plan(plan):
|
||||||
|
require(plan.get('status') == 'readonly_same_submission_reconciliation_plan'
|
||||||
|
and plan.get('database_written') is False, 'A reviewed read-only plan is required')
|
||||||
|
rows = plan['plans']
|
||||||
|
require(0 < len(rows) <= 1000 and len(rows) == plan['guarded_same_submission_plans'],
|
||||||
|
'Unexpected approved plan size')
|
||||||
|
for key in ('doi', 'retained_paper_id', 'retained_submission_id', 'retained_programme_record_id'):
|
||||||
|
require(len({entry[key] for entry in rows}) == len(rows), 'Repeated reviewed ' + key)
|
||||||
|
for entry in rows:
|
||||||
|
doi = entry['doi']
|
||||||
|
row = entry['source_row']
|
||||||
|
url = 'https://link.springer.com/chapter/' + doi
|
||||||
|
evidence = row['metadata']['publisher_chapter_evidence']
|
||||||
|
require(re.fullmatch(r'10\.1007/978-3-032-\d{5}-[0-9Xx]_[1-9]\d*', doi)
|
||||||
|
and (row['source'], row['external_id'], row['url'], row['evidence_url']) ==
|
||||||
|
(PUBLISHER, doi, url, url)
|
||||||
|
and (row['venue'], row['year'], row['track'], row['status']) ==
|
||||||
|
('ECCV', 2026, 'main', 'accepted')
|
||||||
|
and row['identifiers'] == {'doi': doi}
|
||||||
|
and identities_for_row(row) == [('publication_doi', doi)]
|
||||||
|
and isinstance(row['abstract'], str) and row['abstract'].strip()
|
||||||
|
and re.fullmatch(r'[0-9a-f]{64}', evidence['sha256'])
|
||||||
|
and re.fullmatch(r'[0-9a-f]{64}', entry['before_packet_sha256']),
|
||||||
|
'Invalid reviewed publisher observation')
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def existing_coverage(conn, edition, source):
|
||||||
|
rows = conn.execute('''SELECT list_url,status,expected_count,imported_count,cursor,last_success_at
|
||||||
|
FROM edition_sources WHERE edition_id=? AND source_id=?''', (edition, source)).fetchall()
|
||||||
|
require(len(rows) <= 1, 'Source coverage is ambiguous')
|
||||||
|
actual = source_record_count(conn, source, edition)
|
||||||
|
if not rows:
|
||||||
|
require(actual == 0, 'Linked records have no coverage listing')
|
||||||
|
return None
|
||||||
|
row = rows[0]
|
||||||
|
require(row[1] == 'partial' and row[3] == actual, 'Coverage differs from linked records')
|
||||||
|
return tuple(row)
|
||||||
|
|
||||||
|
|
||||||
|
def reconcile_attempt(store, entry, original, target, venue_id, event_key, venue_names, publisher, signature):
|
||||||
|
conn = store.conn
|
||||||
|
doi = entry['doi']
|
||||||
|
canonical_doi = doi.lower()
|
||||||
|
paper = entry['retained_paper_id']
|
||||||
|
submission = entry['retained_submission_id']
|
||||||
|
programme_record = entry['retained_programme_record_id']
|
||||||
|
row = entry['source_row']
|
||||||
|
before = entity_packet(conn, paper)
|
||||||
|
require(packet_digest(before) == entry['before_packet_sha256'], 'Reviewed paper history changed: ' + doi)
|
||||||
|
require(not before['source_records_truncated'] and not before['attempts_truncated']
|
||||||
|
and all(not attempt['decisions_truncated'] and not attempt['sources_truncated']
|
||||||
|
and all(not decision['sources_truncated'] for decision in attempt['decisions'])
|
||||||
|
for attempt in before['attempts']), 'Reviewed paper history exceeds verification limits')
|
||||||
|
same_event = [a for a in before['attempts'] if a['venue'].casefold() == 'eccv' and a['year'] == 2026]
|
||||||
|
require(len(same_event) == 1, 'Paper has multiple or missing ECCV 2026 attempts')
|
||||||
|
attempt = same_event[0]
|
||||||
|
require((attempt['id'], attempt['edition_id'], attempt['source_record_id']) ==
|
||||||
|
(submission, original, programme_record)
|
||||||
|
and [(d['id'], d['status']) for d in attempt['decisions']] ==
|
||||||
|
[(entry['retained_programme_decision_id'], 'accepted')]
|
||||||
|
and [(s['source_record_id'], s['source']) for s in attempt['sources']] ==
|
||||||
|
[(programme_record, PROGRAMME)], 'Original programme attempt differs')
|
||||||
|
programme = next(record for record in before['source_records'] if record['id'] == programme_record)
|
||||||
|
metadata = json.loads(programme['metadata_json'])
|
||||||
|
require(KEY not in metadata and attempt['source_submission_id'] == programme['external_id'],
|
||||||
|
'Programme observation already curated or ambiguous')
|
||||||
|
scope = curation({
|
||||||
|
'external_id': programme['external_id'], 'venue_names': venue_names, 'from_edition_id': original,
|
||||||
|
'year': 2026, 'from_track': 'program_unclassified', 'event_key': event_key,
|
||||||
|
'venue_id': venue_id, 'track': 'main', 'section': 'published ECCV refereed proceedings',
|
||||||
|
'evidence_url': row['evidence_url'], 'evidence_sha256': row['metadata']['publisher_chapter_evidence']['sha256'],
|
||||||
|
}, target, signature)
|
||||||
|
require(not conn.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=?', (submission,)).fetchone()
|
||||||
|
and not conn.execute("SELECT 1 FROM submission_identifiers WHERE edition_id=? AND scheme='publication_doi' AND value=?",
|
||||||
|
(target, doi)).fetchone()
|
||||||
|
and not conn.execute('SELECT 1 FROM source_records WHERE source_id=? AND external_id=?',
|
||||||
|
(publisher, doi)).fetchone(), 'Publication observation or submission identity already exists')
|
||||||
|
require(not conn.execute('SELECT 1 FROM identifiers WHERE scheme=? AND value=?',
|
||||||
|
(PUBLISHER, doi)).fetchone(), 'Publisher identity already exists')
|
||||||
|
owner = conn.execute("SELECT paper_id FROM identifiers WHERE scheme='doi' AND value=?", (canonical_doi,)).fetchone()
|
||||||
|
require(owner is None or owner[0] == paper, 'Publication DOI belongs to another paper')
|
||||||
|
if owner is None:
|
||||||
|
conn.execute("INSERT INTO identifiers(scheme,value,paper_id) VALUES('doi',?,?)", (canonical_doi, paper))
|
||||||
|
before['identifiers'].append({'scheme': 'doi', 'value': canonical_doi, 'paper_id': paper})
|
||||||
|
before['identifiers'].append({'scheme': PUBLISHER, 'value': doi, 'paper_id': paper})
|
||||||
|
before['identifiers'].sort(key=lambda identifier: (identifier['scheme'], identifier['value']))
|
||||||
|
require(conn.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',
|
||||||
|
(target, submission, original)).rowcount == 1, 'Programme attempt changed')
|
||||||
|
conn.execute("INSERT INTO submission_identifiers VALUES(?,'publication_doi',?,?)", (target, doi, submission))
|
||||||
|
metadata[KEY] = [scope]
|
||||||
|
programme['metadata_json'] = json.dumps(metadata, ensure_ascii=False)
|
||||||
|
conn.execute('UPDATE source_records SET metadata_json=? WHERE id=?',
|
||||||
|
(programme['metadata_json'], programme_record))
|
||||||
|
actual_paper, record = store.record(PUBLISHER, doi, row['url'], row['title'], row['abstract'],
|
||||||
|
row['authors'], row['identifiers'], row['metadata'])
|
||||||
|
require(actual_paper == paper, 'Unexpected canonical paper')
|
||||||
|
store.record_tags(paper, record, PUBLISHER, row['tags'])
|
||||||
|
actual_edition = store.decision(paper, record, 'ECCV', 2026, 'main', doi, row['evidence_url'],
|
||||||
|
'accepted', row['raw_decision'], identities=identities_for_row(row),
|
||||||
|
observed_at=row['observed_at'], event_key=event_key)
|
||||||
|
require(actual_edition == target, 'Unexpected target edition')
|
||||||
|
|
||||||
|
after = entity_packet(conn, paper)
|
||||||
|
published = [r for r in after['source_records'] if r['id'] == record]
|
||||||
|
require(len(published) == 1 and published[0]['external_id'] == doi
|
||||||
|
and published[0]['abstract'] == row['abstract'], 'Publication source is missing or changed')
|
||||||
|
after['source_records'].remove(published[0])
|
||||||
|
moved = next(a for a in after['attempts'] if a['id'] == submission)
|
||||||
|
attached = [s for s in moved['sources'] if s['source_record_id'] == record]
|
||||||
|
observed = [d for d in moved['decisions'] if any(s['source_record_id'] == record for s in d['sources'])]
|
||||||
|
require(len(attached) == len(observed) == 1 and attached[0]['source_submission_id'] == doi
|
||||||
|
and observed[0]['status'] == 'accepted' and observed[0]['evidence_url'] == row['evidence_url']
|
||||||
|
and len(observed[0]['sources']) == 1, 'Publisher evidence did not attach to the existing submission')
|
||||||
|
moved['sources'].remove(attached[0])
|
||||||
|
moved['decisions'].remove(observed[0])
|
||||||
|
attempt.update(edition_id=target, track='main')
|
||||||
|
require(before == after, 'Existing paper or submission history changed')
|
||||||
|
|
||||||
|
|
||||||
|
def apply(store, plan, signature, previous_count):
|
||||||
|
rows = validate_plan(plan)
|
||||||
|
conn = store.conn
|
||||||
|
marker = MARKER + signature
|
||||||
|
require(not conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (marker,)).fetchone(),
|
||||||
|
'Reconciliation already applied')
|
||||||
|
edition_rows = conn.execute('''SELECT e.track,e.id,e.venue_id,e.event_key
|
||||||
|
FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym='ECCV' AND e.year=2026
|
||||||
|
AND e.track IN ('main','program_unclassified')''').fetchall()
|
||||||
|
require(len(edition_rows) == 2 and {r[0] for r in edition_rows} == {'main', 'program_unclassified'},
|
||||||
|
'Expected two unambiguous reviewed editions')
|
||||||
|
editions = {r[0]: (r[1], r[2], r[3]) for r in edition_rows}
|
||||||
|
original, venue, event = editions['program_unclassified']
|
||||||
|
target, target_venue, target_event = editions['main']
|
||||||
|
require(venue == target_venue and event == target_event
|
||||||
|
and all((r['move_from_edition_id'], r['move_to_edition_id']) == (original, target) for r in rows),
|
||||||
|
'Reviewed edition identity differs')
|
||||||
|
names = ['ECCV', *(r[0] for r in conn.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (venue,)))]
|
||||||
|
sources = {r[0]: r[1] for r in conn.execute('SELECT name,id FROM sources WHERE name IN (?,?)', (PROGRAMME, PUBLISHER))}
|
||||||
|
programme, publisher = sources[PROGRAMME], sources[PUBLISHER]
|
||||||
|
old = existing_coverage(conn, original, programme)
|
||||||
|
main = existing_coverage(conn, target, programme)
|
||||||
|
published = existing_coverage(conn, target, publisher)
|
||||||
|
require(old is not None and old[3] >= len(rows) and (main is None or main[0] == old[0])
|
||||||
|
and published is not None and published[2:4] == (PUBLISHER_COUNT, previous_count)
|
||||||
|
and previous_count + len(rows) <= PUBLISHER_COUNT, 'Reviewed coverage parent differs')
|
||||||
|
for entry in rows:
|
||||||
|
reconcile_attempt(store, entry, original, target, venue, event, names, publisher, signature)
|
||||||
|
|
||||||
|
remaining = old[3] - len(rows)
|
||||||
|
imported = previous_count + len(rows)
|
||||||
|
main_count = (main[3] if main else 0) + len(rows)
|
||||||
|
for edition, source, expected in ((original, programme, remaining),
|
||||||
|
(target, programme, main_count), (target, publisher, imported)):
|
||||||
|
require(source_record_count(conn, source, edition) == expected, 'Reconciled coverage differs from linked records')
|
||||||
|
conn.execute('UPDATE edition_sources SET imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
|
||||||
|
(remaining, original, programme, old[0]))
|
||||||
|
conn.execute('''INSERT INTO edition_sources
|
||||||
|
(edition_id,source_id,list_url,status,imported_count,cursor,last_success_at) VALUES(?,?,?,'partial',?,?,?)
|
||||||
|
ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET imported_count=excluded.imported_count''',
|
||||||
|
(target, programme, old[0], main_count, old[4], old[5]))
|
||||||
|
conn.execute('''UPDATE edition_sources SET imported_count=?,last_success_at=?
|
||||||
|
WHERE edition_id=? AND source_id=? AND list_url=?''', (imported, timestamp(), target, publisher, published[0]))
|
||||||
|
conn.execute('INSERT INTO schema_migrations(name) VALUES(?)', (marker,))
|
||||||
|
return {'status': 'applied', 'plan_sha256': signature, 'programme_main': main_count,
|
||||||
|
'programme_unclassified': remaining, 'springer_main': imported, 'new_papers': 0, 'new_submissions': 0}
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog)
|
||||||
|
database_arguments(parser)
|
||||||
|
parser.add_argument('--plan', type=Path, required=True)
|
||||||
|
parser.add_argument('--plan-sha256', required=True)
|
||||||
|
parser.add_argument('--springer-previous-count', type=int, required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
raw = args.plan.read_bytes()
|
||||||
|
signature = sha256(raw).hexdigest()
|
||||||
|
require(signature == args.plan_sha256, 'Plan hash differs')
|
||||||
|
plan = json.loads(raw)
|
||||||
|
validate_plan(plan)
|
||||||
|
store = open_store(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
||||||
|
with closing(store.conn) as conn:
|
||||||
|
try:
|
||||||
|
if not getattr(conn, 'postgres', False):
|
||||||
|
conn.commit()
|
||||||
|
conn.execute('BEGIN IMMEDIATE')
|
||||||
|
result = apply(store, plan, signature, args.springer_previous_count)
|
||||||
|
conn.commit()
|
||||||
|
except BaseException:
|
||||||
|
conn.rollback()
|
||||||
|
raise
|
||||||
|
print(json.dumps(result), flush=True)
|
||||||
@@ -26,6 +26,10 @@ class Result:
|
|||||||
def rowcount(self):
|
def rowcount(self):
|
||||||
return self.cursor.rowcount
|
return self.cursor.rowcount
|
||||||
|
|
||||||
|
@property
|
||||||
|
def description(self):
|
||||||
|
return self.cursor.description
|
||||||
|
|
||||||
|
|
||||||
class WriteConnection:
|
class WriteConnection:
|
||||||
postgres = True
|
postgres = True
|
||||||
|
|||||||
@@ -0,0 +1,140 @@
|
|||||||
|
from contextlib import closing, redirect_stdout
|
||||||
|
from hashlib import sha256
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from paper_atlas.audits.source_identity import entity_packet
|
||||||
|
from paper_atlas.cli import main
|
||||||
|
from paper_atlas.db.edition_scopes import resolve
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.ingestion.publication_reconciliation import PROGRAMME, PUBLISHER, packet_digest
|
||||||
|
from paper_atlas.postgres.client import ReadConnection, connect
|
||||||
|
from paper_atlas.postgres.migration import migrate
|
||||||
|
from paper_atlas.postgres.schema import quote
|
||||||
|
|
||||||
|
|
||||||
|
class PublicationReconciliationTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
directory = TemporaryDirectory()
|
||||||
|
self.addCleanup(directory.cleanup)
|
||||||
|
self.root = Path(directory.name)
|
||||||
|
self.db = self.root / 'db.sqlite'
|
||||||
|
self.store = Store(self.db)
|
||||||
|
self.conn = self.store.conn
|
||||||
|
self.addCleanup(self.conn.close)
|
||||||
|
self.programme_url = 'https://eccv.ecva.net/Conferences/2026/AcceptedPapers'
|
||||||
|
self.publisher_url = 'https://link.springer.com/conference/eccv'
|
||||||
|
programme = self.store.source(PROGRAMME, self.programme_url)
|
||||||
|
publisher = self.store.source(PUBLISHER, self.publisher_url)
|
||||||
|
self.entries = [self.programme_attempt(str(i)) for i in range(1, 4)]
|
||||||
|
title = 'A distinct chapter already integrated in the main track'
|
||||||
|
paper, record = self.store.record(PUBLISHER, 'prior', self.publisher_url, title, 'Text', [])
|
||||||
|
self.target = self.store.accepted(paper, record, 'ECCV', 2026, 'main', 'prior', self.publisher_url)
|
||||||
|
self.original = self.entries[0]['move_from_edition_id']
|
||||||
|
for entry in self.entries:
|
||||||
|
entry['move_to_edition_id'] = self.target
|
||||||
|
self.conn.executemany('''INSERT INTO edition_sources
|
||||||
|
(edition_id,source_id,list_url,status,expected_count,imported_count) VALUES(?,?,?,'partial',?,?)''',
|
||||||
|
[(self.original, programme, self.programme_url, 2834, 3),
|
||||||
|
(self.target, publisher, self.publisher_url, 2835, 1)])
|
||||||
|
self.conn.commit()
|
||||||
|
|
||||||
|
def programme_attempt(self, number):
|
||||||
|
title = 'A programme observation for continuing research ' + number
|
||||||
|
url = self.programme_url + '/' + number
|
||||||
|
authors = ['Ada Example', 'Sam Example']
|
||||||
|
paper, record = self.store.record(PROGRAMME, number, url, title, 'Programme text', authors)
|
||||||
|
original = self.store.accepted(paper, record, 'ECCV', 2026, 'program_unclassified', number, url)
|
||||||
|
other, old_record = self.store.record('openreview', 'test-' + number, url, title, 'Programme text', authors)
|
||||||
|
self.assertEqual(other, paper)
|
||||||
|
self.store.decision(paper, old_record, 'ICLR', 2025, 'main', 'test-' + number, url, 'rejected', 'Reject')
|
||||||
|
packet = entity_packet(self.conn, paper)
|
||||||
|
attempt = next(row for row in packet['attempts'] if row['source_record_id'] == record)
|
||||||
|
doi = '10.1007/978-3-032-00000-1_' + number
|
||||||
|
chapter = 'https://link.springer.com/chapter/' + doi
|
||||||
|
row = {'source': PUBLISHER, 'external_id': doi, 'url': chapter, 'evidence_url': chapter,
|
||||||
|
'venue': 'ECCV', 'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||||
|
'title': title, 'authors': authors, 'abstract': 'Publisher text',
|
||||||
|
'observed_at': '2026-11-18T00:00:00Z', 'raw_decision': 'Publication observation',
|
||||||
|
'identifiers': {'doi': doi},
|
||||||
|
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'metadata': {'publisher_chapter_evidence': {'sha256': 'a' * 64}},
|
||||||
|
'tags': [{'namespace': PUBLISHER + ':format', 'value': 'chapter'}]}
|
||||||
|
return {'doi': doi, 'retained_paper_id': paper, 'retained_programme_record_id': record,
|
||||||
|
'retained_submission_id': attempt['id'], 'retained_programme_decision_id': attempt['decisions'][0]['id'],
|
||||||
|
'move_from_edition_id': original, 'source_row': row, 'before_packet_sha256': packet_digest(packet)}
|
||||||
|
|
||||||
|
def run_plan(self, entries, *database):
|
||||||
|
path = self.root / 'plan.json'
|
||||||
|
raw = json.dumps({'status': 'readonly_same_submission_reconciliation_plan', 'database_written': False,
|
||||||
|
'plans': entries, 'guarded_same_submission_plans': len(entries)}).encode()
|
||||||
|
path.write_bytes(raw)
|
||||||
|
main(['maintain', 'eccv-publication-reconciliation', *(database or ('--db', str(self.db))),
|
||||||
|
'--plan', str(path), '--plan-sha256', sha256(raw).hexdigest(), '--springer-previous-count', '1'])
|
||||||
|
|
||||||
|
def test_preserves_history_and_updates_all_three_coverage_scopes(self):
|
||||||
|
entry = self.entries[0]
|
||||||
|
before = entity_packet(self.conn, entry['retained_paper_id'])
|
||||||
|
self.run_plan([entry])
|
||||||
|
packet = entity_packet(self.conn, entry['retained_paper_id'])
|
||||||
|
self.assertEqual(packet['paper'], before['paper'])
|
||||||
|
programme, other = packet['attempts']
|
||||||
|
self.assertEqual((programme['id'], programme['edition_id']), (entry['retained_submission_id'], self.target))
|
||||||
|
self.assertEqual(other, before['attempts'][1])
|
||||||
|
self.assertEqual(programme['decisions'][0], before['attempts'][0]['decisions'][0])
|
||||||
|
self.assertEqual([row['status'] for row in programme['decisions']], ['accepted', 'accepted'])
|
||||||
|
self.assertEqual({r['source'] for r in programme['sources']}, {PROGRAMME, PUBLISHER})
|
||||||
|
self.assertEqual(len(packet['source_records']), 3)
|
||||||
|
self.assertEqual(resolve(self.conn, entry['retained_programme_record_id'], '1',
|
||||||
|
'ECCV', 2026, 'program_unclassified', None), self.target)
|
||||||
|
coverage = {(edition, source): (status, expected, imported)
|
||||||
|
for edition, source, status, expected, imported in self.conn.execute('''SELECT
|
||||||
|
e.track,s.name,c.status,c.expected_count,c.imported_count FROM edition_sources c
|
||||||
|
JOIN editions e ON e.id=c.edition_id JOIN sources s ON s.id=c.source_id''')}
|
||||||
|
self.assertEqual(coverage, {('program_unclassified', PROGRAMME): ('partial', 2834, 2),
|
||||||
|
('main', PROGRAMME): ('partial', None, 1),
|
||||||
|
('main', PUBLISHER): ('partial', 2835, 2)})
|
||||||
|
with self.assertRaisesRegex(ValueError, 'already applied'):
|
||||||
|
self.run_plan([entry])
|
||||||
|
|
||||||
|
def test_changed_history_rolls_back_earlier_observation(self):
|
||||||
|
first, second = self.entries[:2]
|
||||||
|
stale = {**second, 'before_packet_sha256': 'b' * 64}
|
||||||
|
packet = entity_packet(self.conn, first['retained_paper_id'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Reviewed paper history changed'):
|
||||||
|
self.run_plan([first, stale])
|
||||||
|
self.assertEqual(entity_packet(self.conn, first['retained_paper_id']), packet)
|
||||||
|
self.assertIsNone(self.conn.execute(
|
||||||
|
"SELECT 1 FROM identifiers WHERE scheme='doi' AND value=?", (first['doi'],)).fetchone())
|
||||||
|
self.assertEqual(self.conn.execute('SELECT count(*) FROM edition_sources').fetchone()[0], 2)
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the remote PostgreSQL test service')
|
||||||
|
def test_postgres_write_verifies_history_from_the_read_backend(self):
|
||||||
|
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
|
||||||
|
schema = 'pa_eccv_' + uuid.uuid4().hex[:12]
|
||||||
|
|
||||||
|
def cleanup():
|
||||||
|
with connect(service) as conn:
|
||||||
|
for name, in conn.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
|
||||||
|
conn.execute('DROP SCHEMA ' + quote(name) + ' CASCADE')
|
||||||
|
|
||||||
|
self.addCleanup(cleanup)
|
||||||
|
self.conn.close()
|
||||||
|
with redirect_stdout(io.StringIO()):
|
||||||
|
migrate(self.db, service, self.root / 'migration.json', schema=schema, min_free_gib=0)
|
||||||
|
entry = self.entries[0]
|
||||||
|
with closing(ReadConnection(service, schema)) as conn:
|
||||||
|
before = entity_packet(conn, entry['retained_paper_id'])
|
||||||
|
reviewed = {**entry, 'before_packet_sha256': packet_digest(before)}
|
||||||
|
self.run_plan([reviewed], '--pg-service', service, '--pg-schema', schema)
|
||||||
|
with closing(ReadConnection(service, schema)) as conn:
|
||||||
|
packet = entity_packet(conn, entry['retained_paper_id'])
|
||||||
|
self.assertEqual(packet['attempts'][1], before['attempts'][1])
|
||||||
|
self.assertEqual(packet['attempts'][0]['id'], entry['retained_submission_id'])
|
||||||
|
self.assertEqual(packet['attempts'][0]['edition_id'], self.target)
|
||||||
|
self.assertEqual(len(packet['attempts'][0]['decisions']), 2)
|
||||||
Reference in New Issue
Block a user