From 443a79dce25b53d075dcfc99581cc136a6a3ef0b Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 9 Nov 2026 17:20:27 +0800 Subject: [PATCH] Preserve edition-specific ACL and TINLAP 1978 source evidence --- .gitea/workflows/check.yml | 1 + README.md | 2 + docs/coverage/acl-early-sources-2026-11-09.md | 52 ++++ paper_atlas/collectors/acl.py | 18 +- paper_atlas/collectors/acl_historical.py | 45 +++ tests/fixtures/acl-t78.xml | 273 ++++++++++++++++++ tests/test_acl_historical.py | 114 ++++++++ 7 files changed, 502 insertions(+), 3 deletions(-) create mode 100644 docs/coverage/acl-early-sources-2026-11-09.md create mode 100644 paper_atlas/collectors/acl_historical.py create mode 100644 tests/fixtures/acl-t78.xml create mode 100644 tests/test_acl_historical.py diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 38fe7b3..eb4d6cf 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -51,6 +51,7 @@ jobs: subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index 0d15ce0..759b65c 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追 ## 项目结构 +ACL 1978 的年会与 TINLAP-2 来源关系按精确卷适配，保留原系列和不明逐篇角色；1965 的替代关系不转成全局别名。来源、排除条目和恢复边界见 [ACL 早期来源核查](docs/coverage/acl-early-sources-2026-11-09.md)。 + ```text paper_atlas/ cli.py 统一命令入口 diff --git a/docs/coverage/acl-early-sources-2026-11-09.md b/docs/coverage/acl-early-sources-2026-11-09.md new file mode 100644 index 0000000..a98b3f8 --- /dev/null +++ b/docs/coverage/acl-early-sources-2026-11-09.md @@ -0,0 +1,52 @@ +# ACL 早期年份：会议存在、出版目录与逐篇资料分开核验 + +固定普查中 ACL 1963–1978 仍有年份缺口。不能因为会议年表连续，就假设每年都有独立的正式论文集；也不能因为当前目录始于 1979，就断言此前没有摘要、通知或其他公开资料。本次不修改固定的 1,266 个目标年份基准。 + +## 1978 年：限定 T78-1 的来源关系 + +[同期年会通知](https://aclanthology.org/J77-4001.pdf)的 PDF 第 9–10 页（印刷页码 9–10）明确说明，1978 年 ACL 年会采用第二届 Theoretical Issues in Natural Language Processing 的形式，地点为 Urbana/Champaign，日期为 7 月 25–27 日。通知中的 ACM 联合赞助确认仍处于待确认状态，不能将该预告改写为已完成的行政决定。 + +[1982 年 Donald E. Walker 的历史回顾](https://aclanthology.org/www.mt-archive.info/70/ACL-1982-Walker-1.pdf)第 3 页（印刷页码 91）的年会表及缩写说明，独立对应了 1978 年的时间、地点与 TNLP/TINLAP-2 关系。两份扫描件相关页已渲染并逐页目视核对，不仅依靠 OCR。 + +[官方卷目录](https://aclanthology.org/volumes/T78-1/)与固定 XML `T78.xml` 保留原系列名 TINLAP、完整卷名、1978 年及编辑 David L. Waltz。适配仅接受精确的 collection `T78`、volume `1` 和上述字段组合；关键元数据变化会拒绝处理，等待重新核查。 + +- 规范事件名使用 `ACL+TINLAP`，原系列名 `TINLAP`、原始 XML 的 `venue_codes: [tinlap]` 及证据均保留。 +- ACL/TINLAP 别名只挂在这一规范事件系列上，不把所有 TINLAP 年份都算成 ACL。T75、T87 及 T78 其他卷没有此例外。 +- 通知描述了邀请与提交论文两种参与方式，XML 没有逐篇角色。统一暂存为 `proceedings_unspecified`，不猜测每篇属于 invited、main 或 workshop。 +- XML 共 36 条条目，其中 T78-1036 是无作者的场次讨论问题列表。精确识别并在 manifest 的 `excluded_materials` 保留其标识、标题与原因；其余 35 条作为具名贡献整理。该排除不推广到所有无作者条目，已识别条目发生变化也须重新核查。 +- 全部 35 条 XML 均无摘要。出版目录中的 accepted 是来源观察；通知提及录用／不录用通知时间，不提供任何可识别的拒稿决定，不能制造 rejected 或 withdrawn。 + +采集器仅生成元数据文件，不写业务数据库。真实来源逐条验收、PostgreSQL 身份预检、串行导入与独立验收仍是后续门槛。35 条来源不等于 35 个新增论文实体，更不等于完成全库去重。 + +## 1965 年：替代关系不改写为别名 + +Walker 回顾第 1 页（印刷页码 89）说明，1965 年 ICCL（后来称 COLING）替代了当年的年会。[官方 C65-1 目录](https://aclanthology.org/volumes/C65-1/)及 XML 仍以 COLING 标记该卷。这是限定年份的历史关系，不证明 COLING 与 ACL 是两个可全局互换的名字。 + +本次保留 C65 原有 `COLING` 归属，不添加 ACL 全局别名、不重放已导入记录、不将某一年的日历关系推广到整个系列，也不据此悄悄减少固定普查缺口。其他 1963–1977 年资料仍须逐年核查；本次不能证明这些年份整体属于非归档例外。 + +## 固定来源与恢复 + +服务器证据目录：`/data/paper-index/collected/acl-early-source-evidence-v1`。五份官方响应均 HTTP 200，另有两份来自既有官方 XML checkout 的文件，已与 Git 对象逐字节核对。未下载论文全文集合；两份 PDF 是本次历史／行政取证资料，不进入论文全文库。 + +| 文件 | SHA256 | +|---|---| +| `acl-venue.html` | `c60f64cb4d72e677aeb84a5844cde28a8d50579e1995f08e8c7801eed1e3fac5` | +| `acl-history-1982.pdf` | `35c21bd4983975bcaccbb94bd8ab91c9019fc190feec1791255407cf18a7e49a` | +| `acl-1978-notice.pdf` | `14b5deb471c55afb07d789529b95d0462a2a1a77df3dfba7fa1f5d3ab75eb411` | +| `tinlap1978-volume.html` | `7e20d709f72f415819e4d17e3674b19dc7d871eabcf0cf8b4935f0392d775729` | +| `coling1965-volume.html` | `0d2f2f49f2a311f9c0b80a8832f6476eb32f94f35abf36f6d2f2e10063ac880f` | +| `T78.xml` | `f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84` | +| `C65.xml` | `572791610e20b7c6045d984d968e73a18a144bcb5c3da64112001c781826808e` | + +XML commit：`a7be623d01c48c61405dec31c871121cdbed5a01`。测试 fixture `tests/fixtures/acl-t78.xml` 是该原始文件，不是手工模拟论文；不包含 PDF。PDF 哈希建立完整性，不单独证明语义，语义依据是上述具体页面。 + +在服务器使用既有命令，固定 checkout 并输出到新目录： + +```sh +paper-atlas collect acl \ + --checkout /data/paper-index/raw/metadata-checkouts/acl-anthology \ + --collection T78 \ + --out /data/paper-index/collected/acl-t78-source-v1 +``` + +该命令会校验／更新官方 metadata checkout，实际 commit 必须以生成清单为准；严格重现本次取证时，使用已冻结的 XML 和上述 commit，不把更新后的内容冒充旧快照。不要将无 collection 限制的全部历史输出直接作为增量队列。采集完成不代表导入完成，恢复前先检查当前单元和报告，不因日志安静重启已执行的写入。 diff --git a/paper_atlas/collectors/acl.py b/paper_atlas/collectors/acl.py index 8e53f18..ccbfcfd 100644 --- a/paper_atlas/collectors/acl.py +++ b/paper_atlas/collectors/acl.py @@ -6,6 +6,7 @@ import xml.etree.ElementTree as ET from pathlib import Path from paper_atlas.io.git_metadata import checkout_metadata +from paper_atlas.collectors.acl_historical import event_scope, material_reason VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'} @@ -29,7 +30,10 @@ def paper_id(collection,volume,number): return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}' -def volume_scope(meta,volume): +def volume_scope(meta,volume,collection=None): + historical=event_scope(meta,volume,collection) + if historical: + return historical['canonical_venue'],historical['venue_aliases'],historical['track'] codes=[text(n).lower() for n in meta.findall('venue')] book=text(meta.find('booktitle')) lower=book.lower() @@ -59,7 +63,7 @@ def collect_xml(path,out,commit): vid=volume.attrib['id'] meta=volume.find('meta') if meta is None:raise ValueError('Volume metadata missing') - scope=volume_scope(meta,vid) + scope=volume_scope(meta,vid,collection) key=f'{collection}-{vid}' if scope is None: results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))}) @@ -70,12 +74,18 @@ def collect_xml(path,out,commit): year=int(year_text) records=[] frontmatter=0 + excluded_materials=[] + historical=event_scope(meta,vid,collection) for paper in volume.findall('paper'): pid=paper.attrib['id'] if int(pid)==0: frontmatter+=1 continue identifier=paper_id(collection,vid,pid) + reason=material_reason(collection,vid,paper) + if reason: + excluded_materials.append({'external_id':identifier,'title':text(paper.find('title')),'reason':reason}) + continue doi=text(paper.find('doi')) if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold(): raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier) @@ -83,6 +93,8 @@ def collect_xml(path,out,commit): if not title:raise ValueError('Missing paper title: '+identifier) authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')] records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}}) + if historical: + records[-1]['metadata']['historical_event_scope']=historical if not records: results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'}) continue @@ -93,7 +105,7 @@ def collect_xml(path,out,commit): for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n') part.replace(target) missing=sum(not row['abstract'] for row in records) - manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'} + manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'excluded_materials':excluded_materials,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'} target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)) results.append(manifest) print(json.dumps(manifest),flush=True) diff --git a/paper_atlas/collectors/acl_historical.py b/paper_atlas/collectors/acl_historical.py new file mode 100644 index 0000000..3b19e8e --- /dev/null +++ b/paper_atlas/collectors/acl_historical.py @@ -0,0 +1,45 @@ +"""Edition-specific historical scope; never alias every TINLAP year to ACL.""" + +HISTORY_URL = 'https://aclanthology.org/www.mt-archive.info/70/ACL-1982-Walker-1.pdf' +NOTICE_URL = 'https://aclanthology.org/J77-4001.pdf' +BOOK = 'Theoretical Issues in Natural Language Processing-2' +MATERIAL_TITLE = 'List of questions suggested for consideration in each session' + + +def text(node): + return ' '.join(''.join(node.itertext()).split()) if node is not None else '' + + +def event_scope(meta, volume, collection): + if (collection, volume) != ('T78', '1'): + return None + if ([text(n) for n in meta.findall('venue')] != ['tinlap'] + or [text(n) for n in meta.findall('year')] != ['1978'] + or [text(n) for n in meta.findall('booktitle')] != [BOOK] + or [(text(n.find('first')), text(n.find('last'))) for n in meta.findall('editor')] + != [('David L.', 'Waltz')]): + raise ValueError('Reviewed T78-1 scope metadata changed; independent review required') + return { + 'collection': 'T78', 'volume': '1', 'original_venue': 'TINLAP', + 'event_year': 1978, 'event_dates': ['1978-07-25', '1978-07-27'], + 'relationship': 'ACL annual meeting took the form of TINLAP-2', + 'canonical_venue': 'ACL+TINLAP', 'venue_aliases': ['ACL', 'TINLAP'], + 'track': 'proceedings_unspecified', + 'track_scope_note': 'Invited and submitted contributions are described at volume level; individual roles are not identified in XML.', + 'evidence': [ + {'url': NOTICE_URL, 'sha256': '14b5deb471c55afb07d789529b95d0462a2a1a77df3dfba7fa1f5d3ab75eb411', + 'pdf_pages': [9, 10], 'printed_pages': ['9', '10'], + 'scope': 'Contemporary annual meeting notice, not individual decision notes; ACM confirmation was pending in this notice.'}, + {'url': HISTORY_URL, 'sha256': '35c21bd4983975bcaccbb94bd8ab91c9019fc190feec1791255407cf18a7e49a', + 'pdf_pages': [3], 'printed_pages': ['91'], + 'scope': 'Retrospective annual meeting table and TNLP abbreviation confirm the 1978 event association.'}, + ], + } + + +def material_reason(collection, volume, paper): + if (collection, volume, paper.attrib.get('id')) != ('T78', '1', '36'): + return None + if text(paper.find('title')) != MATERIAL_TITLE or paper.findall('author'): + raise ValueError('Reviewed T78-1036 material changed; independent review required') + return 'Session question list, not an authored research contribution; no decision inferred.' diff --git a/tests/fixtures/acl-t78.xml b/tests/fixtures/acl-t78.xml new file mode 100644 index 0000000..12fb70b --- /dev/null +++ b/tests/fixtures/acl-t78.xml @@ -0,0 +1,273 @@ + + + + + + Theoretical Issues in Natural Language Processing-2 + David L.Waltz + tinlap + 1978 + + + + + tinlap-1978-theoretical + + + + Testing The Psychological Reality of a Representational Model + DedreGentner + + gentner-1978-testing-psychological + + + + What Makes Something “Ad Hoc” + Roger C.Schank + + schank-1978-makes-something + + + + The Relation of Grammar to Cognition–a Synopsis + LeonardTalmy + + talmy-1978-relation-grammar + + + + On primitives, prototypes, and other semantic anomalies + TerryWinograd + + winograd-1978-primitives-prototypes + + + + Taxonomic Lattice Structures for Situation Recognition + William A.Woods + + woods-1978-taxonomic-lattice + + + + Description Formation and Discourse Model Synthesis + Bonnie LynnWebber + + webber-1978-description-formation + + + + The Processing of Referring Expressions within a Semantic Network + John R.Anderson + + anderson-1978-processing-referring + + + + Reference Diaries + Herbert H.Clark + CatherineMarshall + + clark-marshall-1978-reference-diaries + + + + Subsequent Reference: Syntactic and Rhetorical Constraints + David D.McDonald + + mcdonald-1978-subsequent-reference + + + + Some Psycholinguistic Constraints on the Construction and Interpretation of Definite Descriptions + AndrewOrtony + + ortony-1978-psycholinguistic-constraints + + + + Bound Variables and Other Anaphors + Barbara H.Partee + + partee-1978-bound-variables + + + + The Use of Focus as a Tool for Disambiguation of Definite Noun Phrases + Candace L.Sidner + + sidner-1978-use-focus + + + + Focusing in Dialog + Barbara J.Grosz + + grosz-1978-focusing-dialog + + + + Topic Levels + Joseph E.Grimes + + grimes-1978-topic-levels + + + + Toward a Rational Model of Discourse Comprehension + Jerry L.Morgan + + morgan-1978-toward-rational + + + + Assent and Compliance in Children’s Language Comprehension + David R.Olson + + olson-1978-assent-compliance + + + + Speech Acts as a Basis for Understanding Dialogue Coherence + C. RaymondPerrault + James F.Allen + + perrault-allen-1978-speech + + + + A Framework for Comparing Language Experiences (with particular emphasis on: The Effect of Audience on Discourse Models) + AndeeRubin + + rubin-1978-framework-comparing + + + + Intentlonallty and Human Conversations + Jaime G.Carbonell Jr + + carbonell-jr-1978-intentlonallty-human + + + + On the Interdependence of Language and Perception + David L.Waltz + + waltz-1978-interdependence-language + + + + The Problem of Naming Shapes: Vision-Language Interface + R.Bajcsy + A. K.Joshi + + bajcsy-joshi-1978-problem-naming + + + + An Argument about the Composition of Conceptual Structure + RayJackendoff + + jackendoff-1978-argument-composition + + + + On the Ontological Status of Visual Mental Images + Stephen MichaelKosslyn + + kosslyn-1978-ontological-status + + + + What has language to do with perception? Some speculations on the <i>Lingua Mentis</i> + Zenon W.Pylyshyn + + pylyshyn-1978-language-perception + + + + Semantic Primitives in Language and Vision + YorickWilks + + wilks-1978-semantic-primitives + + + + A Note on Partial Match of Descriptions. Can One Simultaneously Question (Retrieve) and Inform (Update)? + Aravind K.Joshi + + joshi-1978-note-partial + + + + With a Spoon in Hand This Must Be the Eating Frame + EugeneCharniak + + charniak-1978-spoon-hand + + + + Fragments of a Theory of Human Plausible Reasoning + AllanCollins + + collins-1978-fragments-theory + + + + Indirect Responses to Loaded Questions + S. JerroldKaplan + + kaplan-1978-indirect-responses + + + + On Reasoning by Default + RaymondReiter + + reiter-1978-reasoning-default + + + + Path-Based and Node-Based Inference in Semantic Networks + Stuart C.Shapiro + + shapiro-1978-path-based + + + + The Representation of Derivable Information in Memory: When What Might Have Been Left Unsaid Is Said + Rand J.Spiro + JosephEsposito + Richard J.Vondruska + + spiro-etal-1978-representation-derivable + + + + A Heuristic for Paradigms + Joseph E.Grimes + + grimes-1978-heuristic-paradigms + + + + A Computational Account of Some Constraints on Language + MitchellMarcus + + marcus-1978-computational-account + + + + Remarks on Processing, Constraints, and the Lexicon + ThomasWasow + + wasow-1978-remarks-processing + + + + List of questions suggested for consideration in each session + + nn-1978-list-questions + + + diff --git a/tests/test_acl_historical.py b/tests/test_acl_historical.py new file mode 100644 index 0000000..c13b219 --- /dev/null +++ b/tests/test_acl_historical.py @@ -0,0 +1,114 @@ +"""Pinned T78 evidence, edition-local aliases and an explicitly identified material.""" +import copy +import hashlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest +import xml.etree.ElementTree as ET + +from paper_atlas.collectors.acl import collect_xml, volume_scope +from paper_atlas.collectors.acl_historical import event_scope, material_reason +from paper_atlas.db.store import Store +from paper_atlas.ingestion.records import ingest_file + +FIXTURE = Path(__file__).parent/'fixtures/acl-t78.xml' +COMMIT = 'a7be623d01c48c61405dec31c871121cdbed5a01' + + +class AclHistoricalTests(unittest.TestCase): + def setUp(self): + self.root = ET.parse(FIXTURE).getroot() + self.volume = self.root.find('volume') + self.meta = self.volume.find('meta') + + def test_frozen_volume_matches_official_snapshot(self): + self.assertEqual(hashlib.sha256(FIXTURE.read_bytes()).hexdigest(), 'f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84') + self.assertEqual(len(self.volume.findall('paper')), 36) + self.assertEqual(volume_scope(self.meta, '1', 'T78'), ('ACL+TINLAP', ['ACL', 'TINLAP'], 'proceedings_unspecified')) + + def test_unknown_year_collection_or_volume_is_not_globally_aliased(self): + for collection, volume in [('T75', '1'), ('T87', '1'), ('T78', '2'), ('C65', '1')]: + with self.subTest(collection=collection, volume=volume): + self.assertIsNone(event_scope(self.meta, volume, collection)) + self.assertIsNone(volume_scope(self.meta, volume, collection)) + self.assertIsNone(volume_scope(self.meta, '1')) + + def test_known_volume_scope_rejects_critical_metadata_drift(self): + for field, replacement in [('venue', 'acl'), ('year', '1979'), ('booktitle', 'Another TINLAP'), ('editor/first', 'Another')]: + changed = copy.deepcopy(self.meta) + changed.find(field).text = replacement + with self.subTest(field=field), self.assertRaises(ValueError): + event_scope(changed, '1', 'T78') + changed = copy.deepcopy(self.meta) + ET.SubElement(changed, 'year').text = '1978' + with self.assertRaises(ValueError): + event_scope(changed, '1', 'T78') + + def test_evidence_retains_original_series_dates_and_role_boundary(self): + scope = event_scope(self.meta, '1', 'T78') + self.assertEqual(scope['original_venue'], 'TINLAP') + self.assertEqual(scope['event_dates'], ['1978-07-25', '1978-07-27']) + self.assertEqual([e['pdf_pages'] for e in scope['evidence']], [[9, 10], [3]]) + self.assertIn('not identified', scope['track_scope_note']) + self.assertIn('pending', scope['evidence'][0]['scope']) + + def test_only_exact_session_questions_material_is_excluded(self): + paper = copy.deepcopy(self.volume.findall('paper')[-1]) + self.assertIn('Session question list', material_reason('T78', '1', paper)) + self.assertIsNone(material_reason('C65', '1', paper)) + self.assertIsNone(material_reason('T78', '2', paper)) + paper.set('id', '37') + self.assertIsNone(material_reason('T78', '1', paper)) + paper.set('id', '36') + ET.SubElement(paper, 'author') + with self.assertRaises(ValueError): + material_reason('T78', '1', paper) + + def test_all_authored_source_fields_and_material_receipt_survive_collection(self): + with tempfile.TemporaryDirectory() as temporary: + manifest, = collect_xml(FIXTURE, temporary, COMMIT) + rows = [json.loads(raw) for raw in Path(manifest['file']).read_text().splitlines()] + self.assertEqual(manifest['count'], 35) + self.assertEqual(manifest['xml_paper_elements'], 36) + self.assertEqual(manifest['missing_abstract_count'], 35) + self.assertEqual(manifest['excluded_materials'][0]['external_id'], 'T78-1036') + for number, row in enumerate(rows, 1): + paper = self.volume.findall('paper')[number-1] + self.assertEqual(row['external_id'], f'T78-1{number:03d}') + self.assertEqual(row['title'], ' '.join(''.join(paper.find('title').itertext()).split())) + self.assertEqual(row['metadata']['venue_codes'], ['tinlap']) + self.assertEqual(row['metadata']['historical_event_scope']['original_venue'], 'TINLAP') + self.assertEqual(row['metadata']['metadata_commit'], COMMIT) + self.assertEqual((row['status'], row['track'], row['abstract']), ('accepted', 'proceedings_unspecified', None)) + + def test_old_coling_scope_is_unchanged_by_calendar_replacement_evidence(self): + meta = ET.fromstring('coling1965COLING 1965') + self.assertEqual(volume_scope(meta, '1', 'C65'), ('COLING', ['COLING'], 'main')) + self.assertIsNone(event_scope(meta, '1', 'C65')) + + def test_native_ingestion_does_not_alias_other_tinlap_years_or_infer_negative_status(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + manifest, = collect_xml(FIXTURE, root/'source', COMMIT) + db = root/'sample.sqlite' + store = Store(db) + pid, rid = store.record('test_control', 'tinlap1975', 'https://example.org/tinlap1975', + 'An earlier TINLAP contribution', None, ['Jane Doe']) + store.decision(pid, rid, 'TINLAP', 1975, 'main', 'tinlap1975', 'https://example.org/tinlap1975', 'accepted', 'test control') + store.conn.commit() + store.conn.close() + ingest_file(db, manifest['file']) + ingest_file(db, manifest['file']) + with sqlite3.connect(db) as c: + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 36) + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 36) + self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0], 36) + self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status!='accepted'").fetchone()[0], 0) + self.assertIsNone(c.execute("SELECT 1 FROM source_records WHERE external_id='T78-1036'").fetchone()) + aliases = c.execute('SELECT v.acronym,a.alias FROM venue_aliases a JOIN venues v ON v.id=a.venue_id ORDER BY a.alias').fetchall() + self.assertEqual(aliases, [('ACL+TINLAP', 'ACL'), ('ACL+TINLAP', 'TINLAP')]) + metadata = json.loads(c.execute("SELECT metadata_json FROM source_records WHERE external_id='T78-1001'").fetchone()[0]) + self.assertEqual(metadata['historical_event_scope']['event_year'], 1978) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), []) -- 2.52.0