diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index b3809a6..d2522f5 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -55,6 +55,7 @@ jobs: subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','backfill','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True) smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()" diff --git a/README.md b/README.md index dd61ba5..920965a 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,8 @@ PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service fo PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --paper 123 # 列出全部目标会议,包括尚未入库的项目 -paper-atlas coverage --db /data/paper-index/papers.sqlite \ +PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \ + --pg-service foundry-research --pg-schema research \ --out /data/paper-index/collected/venue-coverage-current.json # 采集与导入分开;这里只采集到独立文件 diff --git a/docs/data-sources.md b/docs/data-sources.md index 71b9f62..06b9e17 100644 --- a/docs/data-sources.md +++ b/docs/data-sources.md @@ -93,6 +93,8 @@ paper-atlas audit public-openreview --db /data/paper-index/papers.sqlite \ 输出按会议、届次及 main、position、datasets_benchmarks 分文件。workshop 默认进入单独的 held 清单;有明确 workshop 名称时可通过 `--include-workshops` 单列采集,不合并到主会。Retraction、未映射的轨道或负面标签也保留在 `held-records.jsonl`,不强改为 Reject 或 Withdrawn。当前决定表没有独立 retracted 状态,因此撤回发表记录仍需后续专门建模。 +`parser_version=2` 还会将明确负面标签但标题缺失、空白或 null 的记录放入 `held-records.jsonl`,保留原始状态、forum ID、固定来源版本和行号;不制造标题或导入不可识别的论文,也不让这一条缺失阻断其他可用记录。held 文件哈希与各原因数量保存在 inventory;非文本标题、错误身份、冲突届次和哈希不符仍使整批失败。旧年 ICLR 镜像的明确 `Withdraw` 标签归为 `withdrawn`,`Workshop`、`Poster Workshop`、`Oral Workshop` 均不作为拒稿。采集文件与导入主库仍是两个独立步骤。 + 该来源明确标记为 `curated_public_status_assertion`:保留源 commit、文件哈希、行号、原标签和 forum ID,不声称已核验原始 PC 签名。只保存标题、摘要、公开作者名、关键词、领域、索引与决定来源,不缓存镜像中的个人资料、邮箱、性别、评审内容或 PDF。所有输出清单均为 partial;盘点中的零条记录仅描述这个固定镜像。 入库前审计核验文件哈希、数量、轨道和来源声明,并用真实 forum ID 对比同届同轨道已有投稿。存在不同决定时阻止整批导入,交由来源和时序核查;另一会议的后续录用属于独立投稿历史,不构成该次拒稿的冲突。同一 forum 的镜像不新增投稿事件,但论文实体的跨平台去重仍须在全量结束后核验。审计通过的报告包含带 SHA-256 的导入队列;应在单写入维护锁下及时串行导入,主库变化后重新审计。 diff --git a/docs/operations.md b/docs/operations.md index 7c302e0..ae875f2 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -97,13 +97,16 @@ paper-atlas audit arxiv-tags --db /data/paper-index/papers.sqlite \ 采集文件、已入库的来源记录和去重论文实体是不同的计数。来源可能覆盖同一篇论文,同一篇论文也可能有多次投稿;不能把各批文件数量相加作为独立论文总数。实时统计与覆盖报告使用: ```sh -paper-atlas query --db /data/paper-index/papers.sqlite --stats -paper-atlas coverage --db /data/paper-index/papers.sqlite \ +PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --pg-schema research --stats +PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \ + --pg-service foundry-research --pg-schema research \ --out /data/paper-index/collected/venue-coverage-current.json ``` manifest 和采集器状态清单用于确认文件是否齐全及是否缺摘要;`ingest_runs` 和协调器日志用于确认主库导入是否成功。官方程序、书目目录和公开投稿快照分别保留,未出现在某份清单不等于被拒稿;第三方快照也不能代表平台当前全量状态。代码测试成功不能代替最终全库去重、完整性和逐届覆盖核验。 +`coverage` 的所有计数来自同一个只读快照。`latest_observed_status_counts` 按每次投稿的最新观察决定统计,`decision_observation_counts` 保留所有历史观察;重复来源不增加投稿数,缺少决定的投稿计入最新状态 unknown,但不凭空制造 unknown 决定。`source_record_counts` 统计来源在该届次的实际记录与缺摘要数,和 `sources` 中逐清单的预期/导入数量分开;身份补充清单与原始论文集清单可以描述同一条记录。来源缺摘要与论文实体从其他来源获得摘要也分别报告。报告仍将历史覆盖标为待独立审计,不能用某个清单 complete 或没有拒稿记录推断全会议覆盖。 + 当前代码与合并状态以 [Pull Requests](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls) 为准;CI 结果以对应提交的 Actions 检查为准。工作分支交付后等待用户手动合并。合并代码不自动部署、不自动启动采集,也不代表数据批次已完成。 ## 修正已有 PMLR 投稿的轨道 diff --git a/docs/usage.md b/docs/usage.md index 452f3d0..133a458 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -83,11 +83,14 @@ paper-atlas query --pg-service foundry-research --query '"language model"' --lim paper-atlas query --pg-service foundry-research --tag arxiv:category=cs.AI paper-atlas query --pg-service foundry-research --venue ICLR --year 2025 --status rejected paper-atlas query --pg-service foundry-research --paper 123 -paper-atlas coverage --db /data/paper-index/papers.sqlite --out /data/paper-index/collected/coverage.json +PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \ + --pg-service foundry-research --pg-schema research --out /data/paper-index/collected/coverage.json ``` `--status rejected` 筛选某次投稿的最新已知决定。它并不表示论文在所有会议均未录用;分析转投情况时查看该论文的完整投稿历史。年份表示会议届次,观察时间和决定日期分别保留。 +覆盖报告使用只读一致性快照,标明实际数据库和 schema,并列出零入库目标与注册表外的已入库会议。逐届、逐 track 分别列出最新观察状态的投稿数、全部决定观察数、各来源记录数,以及来源摘要和论文实体摘要的缺失数。摘要缺失指 NULL 或空字符串;有文字不代表内容已经完整核验。同一来源的多个清单 URL 可以重叠,不能把清单计数相加当作来源总量;零条拒稿也不证明该会议没有公开拒稿。SQLite 样本仍可显式使用 `--db`,生产报告应查询当前 PostgreSQL。 + 例如,查找在 ICLR 有拒稿记录、但在其他已知投稿中已有录用的论文: ```sh diff --git a/paper_atlas/audits/coverage.py b/paper_atlas/audits/coverage.py index 8550881..42db2b8 100644 --- a/paper_atlas/audits/coverage.py +++ b/paper_atlas/audits/coverage.py @@ -1,45 +1,136 @@ -"""Show every target venue, including those with no imported records.""" +"""Describe imported venue coverage without implying a complete census.""" import argparse -from paper_atlas.config import DEFAULT_REGISTRY +from collections import defaultdict import json from pathlib import Path import sqlite3 +from paper_atlas.config import DEFAULT_REGISTRY +from paper_atlas.db.backend import database_arguments, open_read_connection from paper_atlas.db.submissions import timestamp +from paper_atlas.querying import LATEST_DECISION -def report(db,registry): - registry=json.loads(Path(registry).read_text());targets=registry['venues'] - names=[row['acronym'].casefold() for row in targets] - if len(names)!=len(set(names)):raise ValueError('Duplicate target venue') - scope='(v.acronym=? COLLATE NOCASE OR EXISTS(SELECT 1 FROM venue_aliases a WHERE a.venue_id=v.id AND a.alias=? COLLATE NOCASE))' - c=sqlite3.connect(Path(db).resolve().as_uri()+'?mode=ro',uri=True);c.row_factory=sqlite3.Row +STATUSES = ('accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown') + + +def _status_counts(connection, *, latest): + if latest: + query = '''SELECT s.edition_id,COALESCE(d.status,'unknown') AS status,count(*) AS count + FROM submissions s LEFT JOIN decisions d ON d.submission_id=s.id AND ''' + LATEST_DECISION + else: + query = '''SELECT s.edition_id,COALESCE(d.status,'unknown') AS status,count(*) AS count + FROM submissions s JOIN decisions d ON d.submission_id=s.id''' + counts = defaultdict(dict) + for row in connection.execute(query + " GROUP BY s.edition_id,COALESCE(d.status,'unknown')"): + counts[row[0]][row[1]] = row[2] + return counts + + +def report(db, registry, *, pg_service=None, pg_schema=None): + started_at = timestamp() + registry = json.loads(Path(registry).read_text()) + targets = registry['venues'] + names = [row['acronym'].casefold() for row in targets] + if len(names) != len(set(names)): + raise ValueError('Duplicate target venue') + scope = '''(lower(v.acronym)=lower(?) OR EXISTS( + SELECT 1 FROM venue_aliases a WHERE a.venue_id=v.id AND lower(a.alias)=lower(?)))''' + connection = open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema) + if not pg_service: + connection.row_factory = sqlite3.Row try: - c.execute('BEGIN');venues=[] + if pg_service: + identity = connection.execute('SELECT current_database(),current_schema()').fetchone() + database, schema = identity[0], identity[1] + else: + database, schema = str(Path(db).resolve()), None + latest = _status_counts(connection, latest=True) + observations = _status_counts(connection, latest=False) + # Aggregate the paper join once for the whole snapshot. Repeating it + # for each registry target caused repeated scans on the production DB. + edition_counts = {} + for row in connection.execute('''SELECT s.edition_id,count(s.id) AS submissions, + count(DISTINCT s.paper_id) AS papers, + count(DISTINCT CASE WHEN p.abstract IS NULL OR p.abstract='' THEN s.paper_id END) + AS papers_without_abstract_text + FROM submissions s LEFT JOIN papers p ON p.id=s.paper_id GROUP BY s.edition_id'''): + entry = dict(row) + edition_counts[entry.pop('edition_id')] = entry + source_counts = defaultdict(list) + for row in connection.execute('''SELECT s.edition_id,so.name AS source, + count(DISTINCT sr.id) AS source_records, + count(DISTINCT CASE WHEN sr.abstract IS NULL OR sr.abstract='' THEN sr.id END) + AS source_records_without_abstract_text + FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id + JOIN source_records sr ON sr.id=ss.source_record_id JOIN sources so ON so.id=sr.source_id + GROUP BY s.edition_id,so.name ORDER BY s.edition_id,so.name'''): + entry = dict(row) + source_counts[entry.pop('edition_id')].append(entry) + venues, covered_venue_ids = [], set() for target in targets: - acronym=target['acronym'];params=(acronym,acronym) - editions=[] - for row in c.execute('SELECT e.id,v.acronym AS canonical_venue,e.year,e.track,count(s.id) AS submissions,count(DISTINCT s.paper_id) AS papers FROM editions e JOIN venues v ON v.id=e.venue_id LEFT JOIN submissions s ON s.edition_id=e.id WHERE '+scope+' GROUP BY e.id ORDER BY e.year,e.track',params): - edition=dict(row) - edition['sources']=[dict(source) for source in c.execute('SELECT so.name,es.status,es.expected_count,es.imported_count,es.list_url,es.last_success_at,es.last_error FROM edition_sources es JOIN sources so ON so.id=es.source_id WHERE es.edition_id=? ORDER BY so.name',(row['id'],))] + acronym = target['acronym'] + params = (acronym, acronym) + editions = [] + for row in connection.execute('''SELECT e.id,v.id AS venue_id,v.acronym AS canonical_venue, + e.year,e.track FROM editions e JOIN venues v ON v.id=e.venue_id + WHERE ''' + scope + ''' + ORDER BY e.year,e.track,v.acronym''', params): + edition = dict(row) + edition.update(edition_counts.get(row['id'], { + 'submissions': 0, 'papers': 0, 'papers_without_abstract_text': 0})) + covered_venue_ids.add(edition['venue_id']) + edition['latest_observed_status_counts'] = {status: latest[row['id']].get(status, 0) for status in STATUSES} + edition['decision_observation_counts'] = {status: observations[row['id']].get(status, 0) for status in STATUSES} + edition['source_record_counts'] = source_counts[row['id']] + edition['sources'] = [dict(source) for source in connection.execute('''SELECT so.name, + es.status,es.expected_count,es.imported_count,es.list_url,es.last_success_at,es.last_error + FROM edition_sources es JOIN sources so ON so.id=es.source_id + WHERE es.edition_id=? ORDER BY so.name,es.list_url''', (row['id'],))] editions.append(edition) - totals=c.execute('SELECT count(s.id),count(DISTINCT s.paper_id) FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+scope,params).fetchone() - venues.append({'acronym':acronym,'field':target['field'],'imported_submissions':totals[0],'distinct_papers':totals[1], - 'history_status':'partial_history_pending_independent_audit' if totals[0] else 'no_imported_records', - 'discovery_sources':target.get('discovery_sources',[]),'editions':editions}) - return {'checked_at':timestamp(),'target_venue_count':len(targets),'venues_with_imported_records':sum(bool(v['imported_submissions']) for v in venues), - 'scope_note':registry['scope_note'],'interpretation':'All registry targets are listed. Edition source complete means that source file passed import checks; it does not prove full historical or abstract coverage. Papers can appear in multiple venues/years, so do not sum venue paper counts as a global total.', - 'venues':venues} - finally:c.close() + totals = connection.execute('''SELECT count(s.id),count(DISTINCT s.paper_id) + FROM submissions s JOIN editions e ON e.id=s.edition_id + JOIN venues v ON v.id=e.venue_id WHERE ''' + scope, params).fetchone() + venues.append({'acronym': acronym, 'field': target['field'], + 'imported_submissions': totals[0], 'distinct_papers': totals[1], + 'history_status': 'partial_history_pending_independent_audit' if totals[0] else 'no_imported_records', + 'discovery_sources': target.get('discovery_sources', []), 'editions': editions}) + outside_registry = [dict(row) for row in connection.execute('''SELECT v.id AS venue_id, + v.acronym,count(s.id) AS submissions,count(DISTINCT s.paper_id) AS papers + FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id + GROUP BY v.id,v.acronym ORDER BY v.acronym''') if row[0] not in covered_venue_ids] + return {'report_started_at': started_at, 'checked_at': timestamp(), + 'engine': 'postgresql' if pg_service else 'sqlite', 'database': database, 'schema': schema, + 'pg_service': pg_service, 'target_venue_count': len(targets), + 'venues_with_imported_records': sum(bool(v['imported_submissions']) for v in venues), + 'scope_note': registry['scope_note'], + 'interpretation': 'All registry targets are listed, including zero imports. ' + 'Latest status counts use the latest observed decision for each attempt; decision observations retain history. ' + 'Source record totals are per source and edition, not per list URL; multiple list entries may describe overlapping records. ' + 'Missing abstract text means NULL or empty string; nonempty text does not prove a complete abstract. ' + 'An edition source marked complete is not a proof of complete conference history or public rejection coverage. ' + 'Zero imported rejections does not establish that no rejections are public. ' + 'Papers can appear in several venues/years; do not sum venue paper counts as a global total.', + 'venues': venues, 'imported_venues_outside_registry': outside_registry} + finally: + connection.close() def main(argv=None, *, prog=None): - parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--db',required=True) - parser.add_argument('--registry',default=str(DEFAULT_REGISTRY));parser.add_argument('--out') - args=parser.parse_args(argv);result=report(args.db,args.registry);text=json.dumps(result,ensure_ascii=False,indent=2) + parser = argparse.ArgumentParser(prog=prog) + database_arguments(parser) + parser.add_argument('--registry', default=str(DEFAULT_REGISTRY)) + parser.add_argument('--out') + args = parser.parse_args(argv) + result = report(args.db, args.registry, pg_service=args.pg_service, pg_schema=args.pg_schema) + text = json.dumps(result, ensure_ascii=False, indent=2) if args.out: - path=Path(args.out);stage=path.with_suffix('.part');stage.write_text(text+'\n');stage.replace(path) - else:print(text) + path = Path(args.out) + stage = path.with_suffix('.part') + stage.write_text(text + '\n') + stage.replace(path) + else: + print(text) if __name__ == '__main__': diff --git a/paper_atlas/collectors/public_openreview.py b/paper_atlas/collectors/public_openreview.py index bde6ab7..56292a4 100644 --- a/paper_atlas/collectors/public_openreview.py +++ b/paper_atlas/collectors/public_openreview.py @@ -27,7 +27,7 @@ RAW_ROOT = 'https://raw.githubusercontent.com/papercopilot/openreview/' + REVISI TREE_URL = 'https://api.github.com/repos/papercopilot/openreview/git/trees/' + REVISION + '?recursive=1' NEGATIVE = {'reject': 'rejected', 'rejected': 'rejected', 'desk reject': 'desk_rejected', 'desk rejected': 'desk_rejected', - 'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'} + 'withdraw': 'withdrawn', 'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'} TRACKS = {'main': 'main', 'Position': 'position', 'Datasets & Benchmarks': 'datasets_benchmarks'} WORKSHOP_REJECT_LABELS = {"aim-fm workshop @ neurips'24 reject", @@ -90,7 +90,9 @@ def normalize(row, family, year, provenance, include_workshops=False): return None, 'unmapped_venue' title = row.get('title') abstract = row.get('abstract') - if not isinstance(title, str) or not title.strip(): + if title is None or (isinstance(title, str) and not title.strip()): + return None, 'missing_title' + if not isinstance(title, str): raise ValueError('Public submission lacks a title') if abstract is not None and not isinstance(abstract, str): raise ValueError('Abstract must be text or null') @@ -215,7 +217,7 @@ def _collect_files(out, selected, include_workshops, fetcher): name = f'public-openreview-{family}-{year}-{track}.jsonl' path = stage / name path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records)) - manifest = {'source': SOURCE, 'parser_version': 1, 'venue': records[0]['venue'], + manifest = {'source': SOURCE, 'parser_version': 2, 'venue': records[0]['venue'], 'year': year, 'track': track, 'count': len(records), 'status': 'partial', 'file': str(out / name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(), 'list_url': REPOSITORY + '/tree/' + REVISION, 'source_revision': REVISION, diff --git a/paper_atlas/collectors/vision.py b/paper_atlas/collectors/vision.py index 68233d1..dc39732 100644 --- a/paper_atlas/collectors/vision.py +++ b/paper_atlas/collectors/vision.py @@ -7,7 +7,7 @@ import re import time from datetime import date from urllib.error import HTTPError -from urllib.parse import parse_qs, unquote, urljoin, urlsplit +from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit from paper_atlas.io.http import fetch from paper_atlas.io.html import Document, Node, clean @@ -147,7 +147,8 @@ def read_index(index,cache): links=set() for node in Document(html).root.walk(): if node.tag!='a':continue - link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[]) + # Fragments select positions within a page, not separate HTTP resources. + link=urldefrag(urljoin(url,node.attrs.get('href','')))[0];day=parse_qs(urlsplit(link).query).get('day',[]) if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue if day[0]!='all': try:valid=date.fromisoformat(day[0]).year==index['year'] diff --git a/tests/test_coverage_report.py b/tests/test_coverage_report.py index d56364a..3d7a4a5 100644 --- a/tests/test_coverage_report.py +++ b/tests/test_coverage_report.py @@ -7,6 +7,34 @@ from paper_atlas.audits.coverage import report from paper_atlas.db.store import Store +def add_coverage_evidence(store): + paper, record = store.record('official_metadata', 'known', 'https://example.org/known', + 'Known paper with two source observations', 'Complete canonical abstract', ['Jane Doe', 'John Smith'], + {'openreview': 'coverageForum', 'arxiv': '2401.98765'}) + edition = store.decision(paper, record, 'COLING+LREC', 2024, 'main', 'known', + 'https://example.org/accept', 'accepted', 'Accept', aliases=['COLING', 'LREC'], + identities=[('openreview', 'coverageForum')], observed_at='2025-05-01T00:00:00Z') + same, mirror = store.record('bibliography', 'mirror', 'https://example.org/mirror', + 'Known paper with two source observations', None, ['Jane Doe', 'John Smith'], {'openreview': 'coverageForum'}) + store.decision(same, mirror, 'COLING+LREC', 2024, 'main', 'mirror', 'https://example.org/older', + 'rejected', 'Reject', identities=[('openreview', 'coverageForum')], + observed_at='2025-05-01T00:00:00Z', decision_date='2025-04-30T00:00:00Z') + unknown, unknown_record = store.record('unknown_catalogue', 'unknown', 'https://example.org/unknown', + 'Indexed work without a public decision', None, []) + store.decision(unknown, unknown_record, 'COLING+LREC', 2024, 'main', 'unknown', + 'https://example.org/unknown', None, '', identities=[('openreview', 'unknownForum')]) + same, later = store.record('official_metadata', 'later', 'https://example.org/later', + 'Known paper with two source observations', 'Complete canonical abstract', ['Jane Doe', 'John Smith'], + {'openreview': 'coverageNext', 'arxiv': '2401.98765'}) + store.decision(same, later, 'COLING+LREC', 2025, 'findings', 'later', 'https://example.org/later', + 'accepted', 'Accept', identities=[('openreview', 'coverageNext')], observed_at='2025-06-01T00:00:00Z') + source = store.source('official_metadata', 'https://example.org') + for url, status in [('https://example.org/list', 'complete'), ('https://example.org/identity-refresh', 'partial')]: + store.conn.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count) + VALUES(?,?,?,?,?,?)''', (edition, source, url, status, 1, 1)) + store.conn.commit() + + class CoverageTests(unittest.TestCase): def setUp(self): self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup) @@ -31,5 +59,46 @@ class CoverageTests(unittest.TestCase): self.store.conn.commit();coling=report(self.db,self.registry)['venues'][0] self.assertEqual(coling['imported_submissions'],2);self.assertEqual(coling['distinct_papers'],1) + def test_observations_attempts_and_missing_source_abstracts_stay_separate(self): + add_coverage_evidence(self.store) + result = report(self.db, self.registry) + self.assertEqual(result['engine'], 'sqlite') + self.assertEqual(result['database'], str(self.db)) + coling = result['venues'][0] + self.assertEqual((coling['imported_submissions'], coling['distinct_papers']), (3, 2)) + edition, findings = coling['editions'] + self.assertEqual((edition['submissions'], edition['papers'], edition['papers_without_abstract_text']), (2, 2, 1)) + self.assertEqual(edition['latest_observed_status_counts']['accepted'], 1) + self.assertEqual(edition['latest_observed_status_counts']['rejected'], 0) + self.assertEqual(edition['latest_observed_status_counts']['unknown'], 1) + self.assertEqual(edition['decision_observation_counts']['accepted'], 1) + self.assertEqual(edition['decision_observation_counts']['rejected'], 1) + self.assertEqual(edition['decision_observation_counts']['unknown'], 0) + counts = {row['source']: row for row in edition['source_record_counts']} + self.assertEqual(counts['bibliography']['source_records_without_abstract_text'], 1) + self.assertEqual(counts['official_metadata']['source_records'], 1) + self.assertEqual(len(edition['sources']), 2) + self.assertEqual(findings['track'], 'findings') + self.assertEqual(result['imported_venues_outside_registry'], []) + + def test_missing_sqlite_audit_does_not_create_a_database(self): + path = self.root / 'absent.sqlite' + with self.assertRaisesRegex(ValueError, 'Existing database'): + report(path, self.registry) + self.assertFalse(path.exists()) + + def test_empty_editions_remain_visible_after_shared_aggregation(self): + self.store.conn.execute("INSERT INTO venues(acronym,name,field,venue_type) VALUES('AAAI','AAAI','general_ai','conference')") + venue = self.store.conn.execute("SELECT id FROM venues WHERE acronym='AAAI'").fetchone()[0] + self.store.conn.execute("INSERT INTO editions(venue_id,year,track) VALUES(?,2026,'main')", (venue,)) + self.store.conn.commit() + aaai = report(self.db, self.registry)['venues'][1] + self.assertEqual(aaai['history_status'], 'no_imported_records') + self.assertEqual(len(aaai['editions']), 1) + edition = aaai['editions'][0] + self.assertEqual((edition['submissions'], edition['papers'], edition['papers_without_abstract_text']), (0, 0, 0)) + self.assertEqual(sum(edition['latest_observed_status_counts'].values()), 0) + self.assertEqual(edition['source_record_counts'], []) + if __name__=='__main__':unittest.main() diff --git a/tests/test_postgres_ingestion.py b/tests/test_postgres_ingestion.py index 1998e1d..95aaf55 100644 --- a/tests/test_postgres_ingestion.py +++ b/tests/test_postgres_ingestion.py @@ -347,6 +347,50 @@ class PostgreSQLIngestionTests(unittest.TestCase): finally: reader.close() + def test_coverage_matches_sqlite_with_history_aliases_and_missing_source_abstracts(self): + from paper_atlas.audits.coverage import report + from tests.test_coverage_report import add_coverage_evidence + registry = self.root / 'venues.json' + registry.write_text(json.dumps({'scope_note': 'Candidate scope', 'venues': [ + {'acronym': 'coling', 'field': 'language'}, {'acronym': 'AAAI', 'field': 'general_ai'}]})) + sample = Store(self.root / 'seed.sqlite') + try: + add_coverage_evidence(sample) + finally: + sample.conn.close() + store = self.store() + try: + add_coverage_evidence(store) + finally: + store.conn.close() + sqlite_report = report(self.root / 'seed.sqlite', registry) + observed = [] + def reader(*args, **kwargs): + connection = open_read_connection(*args, **kwargs) + observed.append((connection.execute('SHOW transaction_read_only').fetchone()[0], + connection.execute('SHOW transaction_isolation').fetchone()[0])) + return connection + with patch('paper_atlas.audits.coverage.open_read_connection', side_effect=reader): + actual = report(None, registry, pg_service=self.service, pg_schema=self.schema) + self.assertEqual(observed, [('on', 'repeatable read')]) + self.assertEqual(actual['engine'], 'postgresql') + self.assertEqual(actual['schema'], self.schema) + self.assertEqual(actual['pg_service'], self.service) + def content(result): + values = json.loads(json.dumps(result['venues'])) + for venue in values: + for edition in venue['editions']: + for key in ('id', 'venue_id'): + edition.pop(key) + return values + self.assertEqual(content(actual), content(sqlite_report)) + self.assertEqual([r['acronym'] for r in actual['imported_venues_outside_registry']], ['ICLR']) + edition = actual['venues'][0]['editions'][0] + self.assertEqual(edition['latest_observed_status_counts']['accepted'], 1) + self.assertEqual(edition['latest_observed_status_counts']['unknown'], 1) + self.assertEqual(edition['decision_observation_counts']['rejected'], 1) + self.assertEqual(edition['papers_without_abstract_text'], 1) + def test_backfill_merges_evidence_without_reimporting_or_losing_history(self): files = [] for source, external, title in [('official', 'legacy', 'First bibliographic name'), ('openreview_snapshot', 'forum', 'Renamed submission')]: diff --git a/tests/test_public_openreview.py b/tests/test_public_openreview.py index b817a1a..68421e0 100644 --- a/tests/test_public_openreview.py +++ b/tests/test_public_openreview.py @@ -45,10 +45,12 @@ class PublicOpenReviewTests(unittest.TestCase): def test_only_explicit_decisions_and_verified_scope_become_events(self): for value in ('Pending', 'Active', 'Submitted to NeurIPS 2025', 'TBD', '', - 'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject'): + 'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject', + 'Workshop', 'Poster Workshop', 'Oral Workshop'): self.assertIsNone(negative_status(value), value) self.assertEqual(negative_status('Reject'), 'rejected') self.assertEqual(negative_status('Desk rejected'), 'desk_rejected') + self.assertEqual(negative_status('Withdraw'), 'withdrawn') self.assertEqual(negative_status('NeurIPS 2023 Conference Withdrawn Submission'), 'withdrawn') for raw, expected in [('main', 'main'), ('Position', 'position'), ('Datasets & Benchmarks', 'datasets_benchmarks')]: @@ -98,6 +100,46 @@ class PublicOpenReviewTests(unittest.TestCase): self.assertEqual(list(Path(tmp).glob('*.jsonl')), []) self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists()) + def test_missing_title_is_held_without_losing_valid_negative_records(self): + with tempfile.TemporaryDirectory() as tmp: + result = self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [ + self.row(id='SJUdkecgx', title=''), + self.row(id='NoTitle001', title=None), + self.row(id='Whitespace', title=' '), + self.row(id='ValidReject'), + self.row(id='Withdraw01', status='Withdraw'), + self.row(id='Workshop01', status='Workshop'), + ]}, families=['iclr'], years=[2017]) + self.assertEqual(result['input_rows'], 6) + self.assertEqual(result['normalized_records'], 2) + self.assertEqual(result['source_files'][0]['excluded_or_held'], + {'missing_title': 3, 'not_explicit_negative': 1}) + held_path = Path(result['held_file']) + self.assertEqual(hashlib.sha256(held_path.read_bytes()).hexdigest(), + result['held_file_sha256']) + held = [json.loads(line) for line in held_path.read_text().splitlines()] + self.assertEqual([row['id'] for row in held], ['SJUdkecgx', 'NoTitle001', 'Whitespace']) + self.assertTrue(all(row['reason'] == 'missing_title' and row['status'] == 'Reject' + for row in held)) + self.assertEqual([row['provenance']['source_line'] for row in held], [1, 2, 3]) + self.assertNotIn('private@example.org', held_path.read_text()) + self.assertNotIn('private review', held_path.read_text()) + manifest = result['files'][0] + self.assertEqual((manifest['parser_version'], manifest['status']), (2, 'partial')) + rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()] + self.assertEqual([(row['identifiers']['openreview'], row['status']) for row in rows], + [('ValidReject', 'rejected'), ('Withdraw01', 'withdrawn')]) + for row in held: + self.assertEqual(row['provenance']['source_revision'], REVISION) + + def test_nontext_title_still_fails_collection(self): + with tempfile.TemporaryDirectory() as tmp: + with self.assertRaisesRegex(ValueError, 'lacks a title'): + self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [self.row(title=['invalid'])]}, + families=['iclr'], years=[2017]) + self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists()) + self.assertEqual(list(Path(tmp).glob('*.jsonl')), []) + def test_mirror_reimport_preserves_one_attempt_and_later_acceptance_elsewhere(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) diff --git a/tests/test_vision.py b/tests/test_vision.py index 6572b30..7a7c743 100644 --- a/tests/test_vision.py +++ b/tests/test_vision.py @@ -37,6 +37,27 @@ class VisionTests(unittest.TestCase): collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp) self.assertFalse(list(Path(tmp).glob('*.jsonl'))) + def test_fragment_variants_share_one_all_papers_resource(self): + root,papers,entry=self.cvf_fixture();all_url=root+'?day=all' + pages={root:''.join(f'All' for fragment in ('','#','#papers')), + all_url:'