diff --git a/README.md b/README.md index 5b35682..861582c 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 目标包括:完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。 -**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。 +**目前仍在开发,不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology 官方元数据采集、带清单计数核验的会议导入和只读查询脚本。服务器上持续运行合成回归检查;完整 ICML 2025 元数据已采集 3,330 条,尚未作为全量交付。arXiv 元数据 ZIP 已完整下载(1,878,831,586 字节),全量导入正在服务器运行。新增 PeerRead 公开 ICLR 2017 明确决定采集,其他会议适配器和全量历史导入尚未完成。其他会议适配器、跨平台疑似重复复核流程、完整覆盖率核验仍待实现。已配置 ces-control:host 对应的远程 CI 工作流,实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围,不代表已完成采集,也不是与 COLM 比较的客观排名。 ## 数据范围与来源 @@ -53,7 +53,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 | `papers` / `papers_fts` | 稳定论文实体、代表性标题摘要、FTS5 全文检索 | | `identifiers` | arXiv ID、DOI 等唯一外部标识 | | `source_records` / `sources` | 平台记录、原平台标题摘要与索引元数据、来源 URL 和抓取时间 | -| `venues` / `editions` | 会议或期刊系列、年份与 track,区分主会、Findings 和 workshop | +| `venues` / `venue_aliases` / `editions` | 会议或期刊系列、年份与 track,区分主会、Findings 和 workshop,联合会议用别名查询,不重复生成事件 | | `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 | | `submissions` / `decisions` | 多次投稿及带证据的决定历史,支持改投、撤稿和决定变更 | | `tags` / `paper_tags` | 平台标签与人工标签,使用命名空间隔离 | @@ -134,12 +134,28 @@ python3 -B collect_proceedings.py --source neurips \ 采集器从官方根目录发现实际发表届次,仅访问索引和 Abstract HTML 页面,不请求 PDF、补充材料或源码。可用重复 `--year` 限定实际存在的届次;不存在的年份会报错,局部重试保留其他届次的已有状态。每个来源串行请求,实际网络响应后等待 1 秒;缓存命中不额外等待。 -COLM 当前目录提供 2024–2026 三届 pre-proceedings,按会议年份保存,保留平台实际出版日期和出版状态,不把晚补刊日期当作会议年份。NeurIPS 当前目录提供 1987–2025,并包含 2025 独立卷及 2021 独立 Datasets and Benchmarks 索引。主会、datasets_benchmarks、position、creative_ai 分开出文件,不混为同一个 track。首次真实样本核对:COLM 2024 官方目录 299 个摘要入口;NeurIPS 1987 为 90 个,作者与摘要均已验证解析。样本成功不代表所有页面采集完成。 +COLM 当前目录提供 2024–2026 三届 pre-proceedings,按会议年份保存,保留平台实际出版日期和出版状态,不把晚补刊日期当作会议年份。NeurIPS 当前目录提供 1987–2025,并包含 2025 独立卷及 2021 独立 Datasets and Benchmarks 索引。主会、datasets_benchmarks、position、creative_ai 分开出文件,不混为同一个 track。跨平台自标识仅从出版方 paper-actions 区域链接与 citation 元数据提取,不把摘要里的引用链接当作当前论文 ID。manifest 的 `parser_version` 至少为 2 才允许入库;旧文件须利用缓存重新采集,避免错误身份合并。首次真实样本核对:COLM 2024 官方目录 299 个摘要入口;NeurIPS 1987 为 90 个,作者与摘要均已验证解析。样本成功不代表所有页面采集完成。 `colm-index-status.json`、`neurips-index-status.json` 保存逐索引状态;对应 `*-last-run.json` 保存本次实际选择范围。每个 JSONL 的 manifest 记录索引入口数、track 记录数和缺失摘要数。论文数量与官方页面声明不一致会失败,缺失摘要会标记 partial;不会把页脚、姓名政策或其他正文当作摘要。 这两个采集器只写采集文件,合并仍通过单写入协调流程。全量历史覆盖和所有未实现会议仍待最终核验。 +### ACL Anthology 权威 XML + +```sh +python3 -B collect_acl.py \ + --checkout /data/paper-index/raw/metadata-checkouts/acl-anthology \ + --out /data/paper-index/collected +``` + +`metadata_checkout.py` 对官方仓库做只包含 `data/xml` 的 partial clone/sparse checkout,不获取 PDF,不执行仓库 hooks 或构建。每批固定到实际 Git commit。遍历 XML 的 volume venue 字段选择 ACL、EMNLP、NAACL、EACL、COLING、CoNLL;Findings 按所属会议独立成 track,不按文件名把其他同场 workshop 冒充主会。 + +新旧 Anthology ID 按官方规则构造,并核对源 DOI。保留 C69、D19-50 至 D19-66、W 系列的历史编号例外,排除编号 0 的前言。XML 混合标记通过 itertext 提取;无摘要保留为空并标记 partial。主会、Findings、workshop、demo、tutorial、industry、shared_task 分开。 + +联合举办会议保存一个会议事件及多个名称别名。例如 LREC-COLING 的论文可用 COLING 或 LREC 查询,仍只有一次该届投稿记录。可用 `--track main` 限定主会,避免把 Findings 或演示算入主会;别名数据在正常单写入合并时应用 schema。 + +`acl-collection-status.json` 记录每个 XML 及其 volume 的处理结果和排除原因;`acl-last-run.json` 记录源 commit 与本批范围。`--collection P17` 可限定一个确实存在的 collection。已验证真实 ACL 2017 长文 195 条、短文 107 条,二者摘要完整;其他 track 按源实际缺失情况记录,不能凭样本宣称整个历史完成。 + ## 长期维护与更新 1. 每次采集先核对来源、许可、可访问性、空闲空间和现有运行批次。保持低并发,遵守来源限流、重试与人机验证要求。 diff --git a/collect_acl.py b/collect_acl.py new file mode 100644 index 0000000..cfd9976 --- /dev/null +++ b/collect_acl.py @@ -0,0 +1,131 @@ +"""Collect authoritative ACL Anthology XML metadata, never PDFs.""" +import argparse +import json +import re +import xml.etree.ElementTree as ET +from pathlib import Path + +from metadata_checkout import checkout_metadata + +VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'} + + +def text(element): + return ' '.join(''.join(element.itertext()).split()) if element is not None else '' + + +def paper_id(collection,volume,number): + if not str(number).isdigit() or not re.fullmatch(r'[A-Za-z0-9_-]+',str(volume)): + raise ValueError('Invalid Anthology paper numbering') + if re.fullmatch(r'\d{4}\.[a-z0-9]+',collection): + return f'{collection}-{volume}.{int(number)}' + if not re.fullmatch(r'[A-Z]\d{2}',collection) or not str(volume).isdigit(): + raise ValueError('Unsupported historical identifier') + small=collection.startswith('W') or collection=='C69' or (collection=='D19' and 50<=int(volume)<=66) + width=2 if small else 1 + digits=2 if small else 3 + if len(str(int(volume)))>width or len(str(int(number)))>digits: + raise ValueError('Historical identifier exceeds documented width') + return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}' + + +def volume_scope(meta,volume): + codes=[text(n).lower() for n in meta.findall('venue')] + book=text(meta.find('booktitle')) + lower=book.lower() + selected=sorted({VENUES[v] for v in codes if v in VENUES}) + if 'findings' in codes: + associated=volume.split('-')[0].lower() + if associated in VENUES:selected=[VENUES[associated]] + if not selected: + return None + joint_names=sorted({(VENUES|{'lrec':'LREC'})[v] for v in codes if v in VENUES or v=='lrec'}) + if len(joint_names)>1:selected=joint_names + track='main' + if 'findings' in codes or 'findings of' in lower:track='findings' + elif 'shared task' in lower:track='shared_task' + elif 'workshop' in lower or 'student research' in lower:track='workshop' + elif 'demonstration' in lower or 'system demonstrations' in lower:track='demo' + elif 'tutorial' in lower:track='tutorial' + elif 'industry' in lower:track='industry' + return '+'.join(selected),selected,track + + +def collect_xml(path,out,commit): + root=ET.parse(path).getroot() + collection=root.attrib['id'] + results=[] + for volume in root.findall('volume'): + vid=volume.attrib['id'] + meta=volume.find('meta') + if meta is None:raise ValueError('Volume metadata missing') + scope=volume_scope(meta,vid) + key=f'{collection}-{vid}' + if scope is None: + results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))}) + continue + venue,aliases,track=scope + year_text=text(meta.find('year')) + if not re.fullmatch(r'\d{4}',year_text):raise ValueError('No explicit official volume year: '+key) + year=int(year_text) + records=[] + frontmatter=0 + for paper in volume.findall('paper'): + pid=paper.attrib['id'] + if int(pid)==0: + frontmatter+=1 + continue + identifier=paper_id(collection,vid,pid) + doi=text(paper.find('doi')) + if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold(): + raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier) + title=text(paper.find('title')) + if not title:raise ValueError('Missing paper title: '+identifier) + authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')] + records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}}) + if not records: + results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'}) + continue + target=Path(out)/f'acl-{key}.jsonl' + target.parent.mkdir(parents=True,exist_ok=True) + part=target.with_suffix('.part') + with part.open('w') as output: + for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n') + part.replace(target) + missing=sum(not row['abstract'] for row in records) + manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'} + target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)) + results.append(manifest) + print(json.dumps(manifest),flush=True) + return results + + +if __name__=='__main__': + parser=argparse.ArgumentParser() + parser.add_argument('--checkout',required=True) + parser.add_argument('--out',required=True) + parser.add_argument('--collection',action='append') + args=parser.parse_args() + commit=checkout_metadata('https://github.com/acl-org/acl-anthology.git',args.checkout,['/data/xml/']) + out=Path(args.out);out.mkdir(parents=True,exist_ok=True) + status_file=out/'acl-collection-status.json' + catalogue=json.loads(status_file.read_text()) if status_file.exists() else {} + paths=sorted((Path(args.checkout)/'data/xml').glob('*.xml')) + if not paths:raise SystemExit('No official XML files found') + if args.collection and set(args.collection)-{p.stem for p in paths}:parser.error('Requested collection is absent') + errors=[] + processed=0 + for path in paths: + if args.collection and path.stem not in args.collection:continue + try:catalogue[path.stem]={'status':'processed','metadata_commit':commit,'volumes':collect_xml(path,args.out,commit)} + except Exception as exc: + catalogue[path.stem]={'status':'blocked','metadata_commit':commit,'error':str(exc)} + errors.append(path.stem) + print(json.dumps({'collection':path.stem,'error':str(exc)}),flush=True) + processed+=1 + if processed%25==0: + part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file) + print(json.dumps({'xml_collections_processed':processed}),flush=True) + part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file) + (out/'acl-last-run.json').write_text(json.dumps({'metadata_commit':commit,'collection_filter':args.collection,'blocked_collections':errors},indent=2)) + if errors:raise SystemExit(1) diff --git a/collect_proceedings.py b/collect_proceedings.py index 19917c4..30e45fa 100644 --- a/collect_proceedings.py +++ b/collect_proceedings.py @@ -120,7 +120,12 @@ def parse_paper(html,url,source,year,index_url): identifiers={} if meta['citation_doi']: identifiers['doi']=meta['citation_doi'][0] - for node in nodes: + action_nodes=[] + for container in nodes: + if 'paper-actions' in container.attrs.get('class','').split(): + action_nodes.extend(container.walk()) + # Abstract links can cite other papers; only labelled self links identify this one. + for node in action_nodes: if node.tag!='a': continue link=urljoin(url,node.attrs.get('href','')) @@ -176,7 +181,7 @@ def collect_index(source,index_url,year,cache,out): for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n') part.replace(path) missing_count=sum(not r['abstract'] for r in records) - result={'source':source,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'} + result={'source':source,'parser_version':2,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'} path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2)) results.append(result) print(json.dumps(result),flush=True) diff --git a/import_records.py b/import_records.py index 2b45792..665ea13 100644 --- a/import_records.py +++ b/import_records.py @@ -11,6 +11,8 @@ def ingest_file(db,path): path = Path(path) manifest_path = path.with_suffix('.manifest.json') manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} + if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2: + raise ValueError('Recollect proceedings with parser_version>=2 before identity matching') store = Store(db) c = store.conn source_name = manifest.get('source') @@ -34,7 +36,7 @@ def ingest_file(db,path): seen.add(external) paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata')) if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'): - edition = store.decision(paper,record,row['venue'],row['year'],row['track'],external,row.get('evidence_url',row['url']),row['status'],row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status'])) + edition = store.decision(paper,record,row['venue'],row['year'],row['track'],external,row.get('evidence_url',row['url']),row['status'],row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),row.get('venue_aliases')) editions.add(edition) else: raise ValueError('This importer requires an explicit supported decision status') diff --git a/index_core.py b/index_core.py index 2c012e1..5c052d8 100644 --- a/index_core.py +++ b/index_core.py @@ -109,10 +109,12 @@ class Store: c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending')) return paper,record - def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value): + def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None): c = self.conn c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified')) venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0] + for alias in aliases or []: + c.execute('INSERT OR IGNORE INTO venue_aliases VALUES(?,?)',(venue_id,alias)) c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track)) edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0] c.execute('INSERT OR IGNORE INTO submissions(paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?)',(paper,edition,record,external_id)) diff --git a/metadata_checkout.py b/metadata_checkout.py new file mode 100644 index 0000000..4d19294 --- /dev/null +++ b/metadata_checkout.py @@ -0,0 +1,29 @@ +"""Public metadata-only Git checkouts, without credentials or project hooks.""" +import os +import subprocess +import tempfile +from pathlib import Path + + +def checkout_metadata(url,destination,patterns,branch='master'): + destination=Path(destination) + destination.parent.mkdir(parents=True,exist_ok=True) + env=dict(os.environ,GIT_TERMINAL_PROMPT='0',GIT_LFS_SKIP_SMUDGE='1') + def git(*args): + result=subprocess.run(['git','-c','credential.helper=','-c','http.extraHeader=','-c','core.hooksPath=/dev/null','-c','core.deltaBaseCacheLimit=32m',*args],env=env,capture_output=True,text=True,timeout=900) + if result.returncode: + raise RuntimeError('Metadata checkout failed: '+result.stderr[-500:]) + return result.stdout.strip() + if not destination.exists(): + with tempfile.TemporaryDirectory(prefix='metadata-clone-',dir=destination.parent) as tmp: + stage=Path(tmp)/'repo' + git('clone','--filter=blob:none','--depth','1','--single-branch','--branch',branch,'--no-checkout',url,str(stage)) + git('-C',str(stage),'sparse-checkout','set','--no-cone',*patterns) + git('-C',str(stage),'checkout',branch) + stage.rename(destination) + else: + if git('-C',str(destination),'status','--porcelain'): + raise RuntimeError('Metadata checkout has local changes; refusing to overwrite') + git('-C',str(destination),'fetch','--filter=blob:none','--depth','1','origin',branch) + git('-C',str(destination),'checkout','--detach','FETCH_HEAD') + return git('-C',str(destination),'rev-parse','HEAD') diff --git a/query.py b/query.py index c322f43..f1ba580 100644 --- a/query.py +++ b/query.py @@ -13,6 +13,7 @@ def main(): parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI') parser.add_argument('--venue', help='Conference acronym') parser.add_argument('--year', type=int) + parser.add_argument('--track',help='Exact track, e.g. main, findings, workshop') parser.add_argument('--status', choices=['accepted','rejected','desk_rejected','withdrawn','under_review','unknown']) parser.add_argument('--limit', type=int, default=20) parser.add_argument('--stats', action='store_true') @@ -35,10 +36,15 @@ def main(): if args.tag: namespace,value = args.tag.split('=',1) where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value]) - if args.venue or args.year or args.status: + if args.venue or args.year or args.status or args.track: clause = ['1=1'] if args.venue: - clause.append('v.acronym=? COLLATE NOCASE'); params.append(args.venue) + if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='venue_aliases'").fetchone(): + clause.append('(v.acronym=? COLLATE NOCASE OR EXISTS(SELECT 1 FROM venue_aliases va WHERE va.venue_id=v.id AND va.alias=? COLLATE NOCASE))'); params.extend([args.venue,args.venue]) + else: + clause.append('v.acronym=? COLLATE NOCASE');params.append(args.venue) + if args.track: + clause.append('e.track=?');params.append(args.track) if args.year: clause.append('e.year=?'); params.append(args.year) if args.status: diff --git a/schema.sql b/schema.sql index 09a73ec..2e5cfb1 100644 --- a/schema.sql +++ b/schema.sql @@ -31,6 +31,10 @@ CREATE TABLE IF NOT EXISTS editions ( year INTEGER NOT NULL, track TEXT NOT NULL DEFAULT 'main', UNIQUE(venue_id,year,track) ); +CREATE TABLE IF NOT EXISTS venue_aliases ( + venue_id INTEGER NOT NULL REFERENCES venues(id), alias TEXT NOT NULL, + PRIMARY KEY(venue_id,alias) +); CREATE TABLE IF NOT EXISTS edition_sources ( edition_id INTEGER NOT NULL REFERENCES editions(id), source_id INTEGER NOT NULL REFERENCES sources(id), list_url TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending' diff --git a/test_acl.py b/test_acl.py new file mode 100644 index 0000000..99775ba --- /dev/null +++ b/test_acl.py @@ -0,0 +1,40 @@ +import json +import tempfile +import unittest +import xml.etree.ElementTree as ET +from pathlib import Path + +from collect_acl import paper_id,volume_scope,collect_xml + + +class AnthologyTests(unittest.TestCase): + def test_documented_identifier_formats_and_exceptions(self): + for args,expected in [(('P17','1','1'),'P17-1001'),(('2025.acl','long','1'),'2025.acl-long.1'),(('C69','1','1'),'C69-0101'),(('D19','50','1'),'D19-5001'),(('W17','1','1'),'W17-0101')]: + self.assertEqual(paper_id(*args),expected) + + def test_findings_uses_associated_conference_and_separate_track(self): + meta=ET.fromstring('findingsFindings of ACL: EMNLP 2025') + self.assertEqual(volume_scope(meta,'emnlp'),('EMNLP',['EMNLP'],'findings')) + + def test_joint_event_has_one_identity_and_two_aliases(self): + meta=ET.fromstring('lreccolingLREC-COLING 2024') + self.assertEqual(volume_scope(meta,'main'),('COLING+LREC',['COLING','LREC'],'main')) + + def test_xml_mixed_text_frontmatter_and_missing_abstract(self): + xml='2017aclACLFront matterA <fixed-case>Mixed</fixed-case> TitleJaneDoe10.18653/v1/P17-1001' + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'P17.xml';path.write_text(xml) + result=collect_xml(path,tmp,'a'*40)[0] + row=json.loads(Path(result['file']).read_text()) + self.assertEqual(result['count'],1) + self.assertEqual(result['excluded_frontmatter'],1) + self.assertEqual(result['status'],'partial') + self.assertEqual(row['external_id'],'P17-1001') + self.assertEqual(row['title'],'A Mixed Title') + + def test_non_target_workshop_is_not_main_conference(self): + meta=ET.fromstring('some-workshopwsWorkshop at ACL') + self.assertIsNone(volume_scope(meta,'1')) + + +if __name__=='__main__':unittest.main() diff --git a/test_index_core.py b/test_index_core.py index 29cb122..87e1fea 100644 --- a/test_index_core.py +++ b/test_index_core.py @@ -61,10 +61,18 @@ class StoreTests(unittest.TestCase): def test_missing_abstract_does_not_become_complete_coverage(self): path=Path(self.tmp.name)/'partial.jsonl' path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n') - path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','list_url':'https://example.org/list','count':1,'status':'partial','missing_abstract_count':1})) + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','parser_version':2,'list_url':'https://example.org/list','count':1,'status':'partial','missing_abstract_count':1})) ingest_file(self.db,path) self.assertEqual(self.store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'partial') + def test_legacy_proceedings_manifest_is_rejected_before_import(self): + path=Path(self.tmp.name)/'old.jsonl' + path.write_text('') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','count':1})) + with self.assertRaisesRegex(ValueError,'parser_version'): + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0) + if __name__=='__main__': unittest.main() diff --git a/test_proceedings.py b/test_proceedings.py index 44c1153..931b34d 100644 --- a/test_proceedings.py +++ b/test_proceedings.py @@ -52,5 +52,10 @@ class ProceedingsTests(unittest.TestCase): self.assertEqual(result[other]['status'],'collected') self.assertEqual(result[url]['files'][0]['count'],1) + def test_abstract_citations_are_not_used_as_self_identifiers(self): + html='
OpenReview

Related work and another paper.

' + row=parse_paper(html,'https://proceedings.colm.cc/paper','colm',2024,'https://proceedings.colm.cc/') + self.assertEqual(row['identifiers'],{'openreview':'selfForum'}) + if __name__=='__main__':unittest.main() diff --git a/test_query.py b/test_query.py index 0b83b50..068d4b9 100644 --- a/test_query.py +++ b/test_query.py @@ -9,6 +9,18 @@ from index_core import Store class QueryTests(unittest.TestCase): + def test_joint_conference_alias_does_not_duplicate_submission(self): + with tempfile.TemporaryDirectory() as tmp: + db=Path(tmp)/'papers.sqlite';store=Store(db);self.addCleanup(store.conn.close) + paper,record=store.record('acl_anthology','paper1','https://example.org/1','Joint event paper','Abstract',[]) + store.decision(paper,record,'COLING+LREC',2024,'main','paper1','https://example.org/1','accepted','official XML',['COLING','LREC']) + store.conn.commit() + for venue in ('COLING','LREC'): + command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--venue',venue,'--track','main'] + rows=json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout) + self.assertEqual([r['id'] for r in rows],[paper]) + self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + def test_venue_filter_is_case_insensitive_and_uses_latest_decision(self): with tempfile.TemporaryDirectory() as tmp: db=Path(tmp)/'papers.sqlite'