Resolve mixed ICML PMLR tracks from verified official programs

docs/data-sources.md

@@ -300,3 +300,13 @@ paper-atlas collect cvf --cache /data/paper-index/raw/http-cache \
 ```
 
 parser_version 2 的清单还保存实际遍历的 index_urls、入口错误及缺失摘要页面。补采结果经过清单与身份核验后才导入，不能因重试任务启动就标记缺口已解决。
+
+## PMLR 中的 ICML 多轨道论文
+
+ICML 2024 起的一个 PMLR 卷可以同时包含研究主轨道与 Position Paper，不能把整卷都标为 `main`。采集器 v3 按轨道输出 `pmlr-v<VOLUME>-main.jsonl`、`pmlr-v<VOLUME>-position.jsonl` 及各自 manifest；卷级 `pmlr-v<VOLUME>.inventory.json` 和目录清单的 `files` 数组列出实际文件。导入队列应展开 `files`，不能假定每卷只有一个 `file`。
+
+2025、2026 年通过出版社元数据的真实 OpenReview forum ID，匹配官方完整节目单中的 `Conference` / `Position_Paper_Track` 来源组；忽略负编号生成的口头报告副本，遇到冲突、缺少身份或未匹配的发表记录停止发布结果。每条记录保存节目单 URL、SHA-256、事件 ID、关联身份及轨道证据。
+
+2024 年的官方节目单把两轨都放在 `Conference`，但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则；同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`：只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合，不进行全库模糊匹配。
+
+导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联；仅重新导入拆分文件会留下旧的主轨道事件，不能作为修复方式。完成迁移前，不把新文件直接导入现有库。

paper_atlas/collectors/pmlr.py

@@ -1,5 +1,7 @@
 """Collect official PMLR metadata, never the PDFs linked by the metadata."""
 import argparse
+from collections import defaultdict
+import hashlib
 from paper_atlas.config import DEFAULT_REGISTRY
 import io
 import json
@@ -8,6 +10,7 @@ import os
 import subprocess
 import tarfile
 import time
+import tempfile
 from pathlib import Path
 
 import yaml
@@ -37,6 +40,7 @@ def venue_key(shortname):
 
 
 from paper_atlas.io.http import fetch
+from paper_atlas.collectors.pmlr_tracks import collect_track_evidence, record_track
 
 
 def metadata_texts(volume,cache):
@@ -98,57 +102,88 @@ def collect_volume(volume,cache,out,allowed=None):
     track = 'workshop' if 'workshop' in title.lower() else 'main'
     path = Path(out)/f'pmlr-v{volume}.jsonl'
     path.parent.mkdir(parents=True,exist_ok=True)
-    part = path.with_suffix('.part')
+    mixed_icml = venue == 'ICML' and year >= 2024
+    mapping, track_program = collect_track_evidence(year, cache) if mixed_icml else ({}, None)
     records = {}
     duplicate_posts = 0
-    with part.open('w') as output:
-        for member_name,content in metadata_texts(volume,cache):
-            parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
-            if len(parts)<3 or parts[0].strip():
-                raise RuntimeError('Missing YAML frontmatter: '+member_name)
-            row = load_metadata(parts[1])
-            if row.get('layout') not in ('inproceedings','article'):
-                continue
-            if not row.get('title'):
-                raise ValueError('Paper metadata has no title: '+member_name)
-            if not row.get('id'):
-                raise ValueError('Paper metadata has no publication ID: '+member_name)
-            external = f"v{volume}/{row['id']}"
-            ids = {}
-            for key in ('doi','openreview','arxiv'):
-                value = row.get(key)
-                if value:
-                    value = str(value)
-                    if key=='arxiv':
-                        value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
-                    if key=='openreview':
-                        value = value.split('id=')[-1].split('&')[0]
-                    ids[key] = value
-            record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
-            version={'path':member_name,'date':row.get('date')}
-            if external in records:
-                previous=records[external]
-                if ({k:v for k,v in previous.items() if k!='metadata'} != {k:v for k,v in record.items() if k!='metadata'}
-                    or {k:v for k,v in previous['metadata'].items() if k not in ('date','source_versions')} != {k:v for k,v in record['metadata'].items() if k!='date'}):
-                    raise ValueError('Conflicting metadata posts share a publication ID: '+external)
-                previous['metadata']['source_versions'].append(version)
-                if str(row.get('date') or '')>str(previous['metadata'].get('date') or ''):
-                    previous['metadata']['date']=row['date']
-                duplicate_posts+=1
-                continue
-            record['metadata']['source_versions']=[version]
-            records[external]=record
-        for record in records.values():
-            output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n')
+    for member_name,content in metadata_texts(volume,cache):
+        parts = re.split(r'(?m)^---\s*$',content,maxsplit=2)
+        if len(parts)<3 or parts[0].strip():
+            raise RuntimeError('Missing YAML frontmatter: '+member_name)
+        row = load_metadata(parts[1])
+        if row.get('layout') not in ('inproceedings','article'):
+            continue
+        if not row.get('title'):
+            raise ValueError('Paper metadata has no title: '+member_name)
+        if not row.get('id'):
+            raise ValueError('Paper metadata has no publication ID: '+member_name)
+        external = f"v{volume}/{row['id']}"
+        ids = {}
+        for key in ('doi','openreview','arxiv'):
+            value = row.get(key)
+            if value:
+                value = str(value)
+                if key=='arxiv':
+                    value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf')
+                if key=='openreview':
+                    value = value.split('id=')[-1].split('&')[0]
+                ids[key] = value
+        record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
+        version={'path':member_name,'date':row.get('date')}
+        if external in records:
+            previous=records[external]
+            if ({k:v for k,v in previous.items() if k!='metadata'} != {k:v for k,v in record.items() if k!='metadata'}
+                or {k:v for k,v in previous['metadata'].items() if k not in ('date','source_versions')} != {k:v for k,v in record['metadata'].items() if k!='date'}):
+                raise ValueError('Conflicting metadata posts share a publication ID: '+external)
+            previous['metadata']['source_versions'].append(version)
+            if str(row.get('date') or '')>str(previous['metadata'].get('date') or ''):
+                previous['metadata']['date']=row['date']
+            duplicate_posts+=1
+            continue
+        record['metadata']['source_versions']=[version]
+        records[external]=record
     count=len(records)
     missing_abstracts=sum(not (record.get('abstract') or '').strip() for record in records.values())
     if count==0:
-        part.unlink(missing_ok=True)
         raise RuntimeError('No papers parsed; refusing an empty successful collection')
-    part.replace(path)
-    result = {'source':'pmlr','parser_version':2,'venue':venue,'year':year,'track':track,'volume':volume,'count':count,'metadata_post_count':count+duplicate_posts,'duplicate_post_count':duplicate_posts,'missing_abstract_count':missing_abstracts,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'partial' if missing_abstracts else 'collected'}
-    path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2))
-    print(json.dumps(result),flush=True)
+    grouped = defaultdict(list)
+    for record in records.values():
+        if mixed_icml:
+            record_track(record, mapping, track_program)
+        grouped[record['track']].append(record)
+    # Parse and verify the entire volume before replacing any published output.
+    manifests = []
+    with tempfile.TemporaryDirectory(prefix=f'.pmlr-v{volume}-', dir=path.parent) as temporary:
+        stage = Path(temporary)
+        for current_track, rows in sorted(grouped.items()):
+            filename = f'pmlr-v{volume}-{current_track}.jsonl' if mixed_icml else path.name
+            output = stage / filename
+            output.write_text(''.join(json.dumps(row, ensure_ascii=False, default=str)+'\n' for row in rows))
+            absent = sum(not (row.get('abstract') or '').strip() for row in rows)
+            manifest = {'source': 'pmlr', 'parser_version': 3, 'venue': venue, 'year': year,
+                        'track': current_track, 'volume': volume, 'count': len(rows),
+                        'missing_abstract_count': absent, 'list_url': f'https://proceedings.mlr.press/v{volume}/',
+                        'file': str(path.parent / filename), 'file_sha256': hashlib.sha256(output.read_bytes()).hexdigest(),
+                        'status': 'partial' if absent else 'collected'}
+            if track_program:
+                manifest['track_program'] = track_program
+            output.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
+            manifests.append(manifest)
+        result = {'source': 'pmlr', 'parser_version': 3, 'venue': venue, 'year': year,
+                  'track': next(iter(grouped)) if len(grouped) == 1 else 'mixed', 'volume': volume,
+                  'count': count, 'metadata_post_count': count + duplicate_posts,
+                  'duplicate_post_count': duplicate_posts, 'missing_abstract_count': missing_abstracts,
+                  'list_url': f'https://proceedings.mlr.press/v{volume}/', 'files': manifests,
+                  'status': 'partial' if missing_abstracts else 'collected'}
+        if len(manifests) == 1:
+            result['file'] = manifests[0]['file']
+        if track_program:
+            result['track_program'] = track_program
+        inventory = stage / f'pmlr-v{volume}.inventory.json'
+        inventory.write_text(json.dumps(result, indent=2)+'\n')
+        for child in sorted(stage.iterdir(), key=lambda child: child == inventory):
+            child.replace(path.parent / child.name)
+    print(json.dumps(result), flush=True)
     return result
 
 

paper_atlas/collectors/pmlr_tracks.py

@@ -0,0 +1,140 @@
+"""Resolve mixed ICML proceedings tracks from the official paper program."""
+from collections import Counter
+import hashlib
+import json
+import re
+from urllib.parse import parse_qs, urlsplit
+
+from paper_atlas.collectors.iclr import identifiers, source_number
+from paper_atlas.db.store import author_keys, normalized
+from paper_atlas.io.html import Document, clean
+from paper_atlas.io.http import fetch
+
+
+POSITION_LIST_POLICY = 'https://icml.cc/Conferences/2025/CallForPositionPapers'
+PUBLISHER_LINK_ALIASES = {'v235/theo24a': 'v235/guyard24a'}
+
+
+def position_list_policy(data):
+    """The organizers explicitly link this title filter as the 2024 position list."""
+    for node in Document(data.decode()).root.walk():
+        if node.tag != 'a' or clean(node.text()).casefold() != 'position papers':
+            continue
+        url = node.attrs.get('href', '')
+        parsed = urlsplit(url)
+        query = parse_qs(parsed.query)
+        if (parsed.netloc in ('', 'icml.cc') and parsed.path == '/virtual/2024/papers.html'
+                and query.get('filter') == ['titles'] and query.get('search') == ['position:']):
+            return {'url': POSITION_LIST_POLICY, 'sha256': hashlib.sha256(data).hexdigest(),
+                    'position_list_url': 'https://icml.cc' + parsed.path + '?' + parsed.query}
+    raise ValueError('Official policy no longer identifies the 2024 position-paper title list')
+
+
+def program_tracks(data, year, policy=None):
+    program = json.loads(data)
+    events = program.get('results') if isinstance(program, dict) else None
+    if (not isinstance(events, list) or program.get('count') != len(events)
+            or program.get('next') or program.get('previous')):
+        raise ValueError('Official ICML program must be a complete counted event list')
+    if len({str(row['id']) for row in events}) != len(events):
+        raise ValueError('Repeated event identity in official ICML program')
+    if year == 2024 and policy is None:
+        raise ValueError('The 2024 position-list policy is required')
+    mapping = {}
+    excluded = Counter()
+    for row in events:
+        number = source_number(row)
+        if number is not None and number < 0:
+            excluded['synthetic_oral_event'] += 1
+            continue
+        parsed = urlsplit(row.get('sourceurl') or '')
+        group = parse_qs(parsed.query).get('id', [])
+        groups = {f'ICML.cc/{year}/Conference': 'main',
+                  f'ICML.cc/{year}/Position_Paper_Track': 'position'}
+        if (parsed.netloc != 'openreview.net' or parsed.path != '/group'
+                or len(group) != 1 or group[0] not in groups):
+            excluded['other_source'] += 1
+            continue
+        if not str(row.get('decision') or '').casefold().startswith('accept'):
+            raise ValueError('Non-accepted event in official ICML paper program')
+        if year == 2024:
+            # The 2024 program flattens both tracks into Conference and links
+            # PMLR publication IDs instead of forums. Use only this year-specific,
+            # organizer-endorsed list rule, never a global title heuristic.
+            link = urlsplit(row.get('paper_pdf_url') or '')
+            if link.netloc != 'proceedings.mlr.press' or not re.fullmatch(r'/v\d+/[^/]+\.html', link.path):
+                raise ValueError('2024 ICML paper has no publisher self-link')
+            key = ('pmlr', link.path.lstrip('/').removesuffix('.html'))
+            track = 'position' if re.match(r'^Position(?: Paper)?:', row.get('name') or '', re.I) else 'main'
+            kind = 'official_2024_position_list'
+        else:
+            forum = identifiers(row).get('openreview')
+            if not forum:
+                raise ValueError('ICML paper has no real OpenReview self-identity')
+            key = ('openreview', forum)
+            track, kind = groups[group[0]], 'official_openreview_track_group'
+        evidence = {'track': track, 'kind': kind, 'event_ids': [row['id']],
+                    'source_group': group[0], 'program_title': row.get('name')}
+        if key[0] == 'pmlr' and key[1] in PUBLISHER_LINK_ALIASES:
+            evidence['publisher_link_alias'] = key[1]
+            evidence['_authors'] = [author['fullname'] for author in row.get('authors', [])]
+            key = ('pmlr', PUBLISHER_LINK_ALIASES[key[1]])
+        if key in mapping:
+            if mapping[key]['track'] != track:
+                raise ValueError('One paper identity has conflicting official tracks')
+            mapping[key]['event_ids'].append(row['id'])
+        else:
+            mapping[key] = evidence
+    if not mapping:
+        raise ValueError('Official ICML program contains no supported paper identities')
+    return mapping, {'event_count': len(events), 'paper_identity_count': len(mapping),
+                     'excluded_events': dict(excluded)}
+
+
+def collect_track_evidence(year, cache):
+    url = f'https://icml.cc/static/virtual/data/icml-{year}-orals-posters.json'
+    data = fetch(url, cache)
+    policy = position_list_policy(fetch(POSITION_LIST_POLICY, cache)) if year == 2024 else None
+    mapping, inventory = program_tracks(data, year, policy)
+    provenance = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), **inventory}
+    if policy:
+        provenance['position_list_policy'] = policy
+    return mapping, provenance
+
+
+def record_track(record, mapping, provenance):
+    key = ('pmlr', record['external_id']) if record['year'] == 2024 else ('openreview', record['identifiers'].get('openreview'))
+    if key not in mapping:
+        raise ValueError('Published ICML paper is absent from the verified official track inventory: ' + record['external_id'])
+    evidence = {**mapping[key], 'identity_scheme': key[0], 'identity_value': key[1],
+                'program_url': provenance['url'], 'program_sha256': provenance['sha256']}
+    if 'publisher_link_alias' in evidence:
+        overlap = author_keys(record.get('authors', [])) & author_keys(evidence.pop('_authors', []))
+        if normalized(record['title']) != normalized(evidence['program_title']) or len(overlap) < 2:
+            raise ValueError('Reviewed publisher-link alias lacks exact title and independent author corroboration')
+        evidence['corroborating_author_count'] = len(overlap)
+    if 'position_list_policy' in provenance:
+        evidence['position_list_policy'] = provenance['position_list_policy']
+    record['track'] = evidence['track']
+    record['metadata']['track_evidence'] = evidence
+
+
+def require_track_manifest(manifest):
+    if (manifest.get('source') == 'pmlr' and str(manifest.get('venue', '')).upper() == 'ICML'
+            and int(manifest.get('year', 0)) >= 2024
+            and (manifest.get('parser_version', 0) < 3 or not manifest.get('track_program'))):
+        raise ValueError('Recollect mixed ICML PMLR proceedings with parser_version>=3 and official track evidence')
+
+
+def require_track_record(row, manifest):
+    require_track_manifest({**manifest, **{key: row.get(key) for key in ('source', 'venue', 'year')}})
+    if manifest.get('track_program'):
+        evidence = row.get('metadata', {}).get('track_evidence', {})
+        program = manifest['track_program']
+        if (evidence.get('track') != row.get('track') or evidence.get('program_url') != program.get('url')
+                or evidence.get('program_sha256') != program.get('sha256')
+                or not evidence.get('event_ids') or evidence.get('identity_scheme') not in ('pmlr', 'openreview')):
+            raise ValueError('PMLR record differs from its official track evidence')
+        own = row['external_id'] if evidence['identity_scheme'] == 'pmlr' else row.get('identifiers', {}).get('openreview')
+        if not own or evidence.get('identity_value') != own:
+            raise ValueError('PMLR track evidence belongs to a different paper identity')

paper_atlas/ingestion/backfill.py

@@ -6,6 +6,7 @@ from pathlib import Path
 
 from paper_atlas.db.store import Store
 from paper_atlas.db.submissions import attach_submission,identities_for_row
+from paper_atlas.collectors.pmlr_tracks import require_track_manifest, require_track_record
 
 
 def queue_files(path):
@@ -21,6 +22,7 @@ def queue_files(path):
 def backfill(db,files):
     for filename in files:
         manifest=json.loads(Path(filename).with_suffix('.manifest.json').read_text())
+        require_track_manifest(manifest)
         if manifest['source'] in ('colm','neurips','iclr_program') and manifest.get('parser_version',0)<2:
             raise ValueError('Recollect old parser output before submission evidence backfill')
     store=Store(db);c=store.conn;count=0
@@ -35,6 +37,7 @@ def backfill(db,files):
                     seen.add(row['external_id'])
                     if any(row.get(key)!=manifest.get(key) for key in ('source','venue','year','track')):
                         raise ValueError('Evidence row differs from edition manifest')
+                    require_track_record(row, manifest)
                     record=c.execute('SELECT r.id,r.paper_id,r.metadata_json FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?',(row['source'],row['external_id'])).fetchone()
                     edition=c.execute('SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym=? COLLATE NOCASE AND e.year=? AND e.track=?',(row['venue'],row['year'],row['track'])).fetchone()
                     if not record or not edition:raise ValueError('Evidence backfill requires an already imported record and edition')

paper_atlas/ingestion/records.py

@@ -6,12 +6,14 @@ from pathlib import Path
 
 from paper_atlas.db.store import Store
 from paper_atlas.db.submissions import identities_for_row,timestamp
+from paper_atlas.collectors.pmlr_tracks import require_track_manifest, require_track_record
 
 
 def ingest_file(db,path):
     path = Path(path)
     manifest_path = path.with_suffix('.manifest.json')
     manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {}
+    require_track_manifest(manifest)
     if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2:
         raise ValueError('Recollect proceedings with parser_version>=2 before identity matching')
     if manifest.get('source')=='iclr_program' and manifest.get('parser_version',0)<2:
@@ -36,6 +38,7 @@ def ingest_file(db,path):
                     raise ValueError('Source differs from manifest')
                 if any(key in manifest and manifest[key]!=row[key] for key in ('venue','year','track')):
                     raise ValueError('Conference edition differs from manifest')
+                require_track_record(row, manifest)
                 external = row['external_id']
                 if external in seen:
                     raise ValueError('Duplicate external ID in one collected file: '+external)

tests/test_index_core.py

@@ -55,7 +55,7 @@ class StoreTests(unittest.TestCase):
 
     def test_manifest_count_mismatch_rolls_back_records(self):
         path=Path(self.tmp.name)/'source.jsonl'
-        path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'})+'\n')
+        path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'})+'\n')
         path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':2}))
         with self.assertRaisesRegex(ValueError,'differs'):
             ingest_file(self.db,path)
@@ -100,7 +100,7 @@ class StoreTests(unittest.TestCase):
 
     def test_missing_abstract_is_checked_even_when_manifest_claims_complete(self):
         path=Path(self.tmp.name)/'incomplete.jsonl'
-        row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':None,'authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'}
+        row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':None,'authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'}
         path.write_text(json.dumps(row)+'\n')
         path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':1,'status':'collected','missing_abstract_count':0}))
         ingest_file(self.db,path)
@@ -108,7 +108,7 @@ class StoreTests(unittest.TestCase):
 
     def test_manifest_edition_mismatch_rolls_back_instead_of_marking_coverage(self):
         path=Path(self.tmp.name)/'wrong-edition.jsonl'
-        row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'}
+        row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'}
         path.write_text(json.dumps(row)+'\n')
         path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':1,'year':2024}))
         with self.assertRaisesRegex(ValueError,'edition differs'):

tests/test_pmlr.py

@@ -25,7 +25,7 @@ class MetadataTests(unittest.TestCase):
         self.assertEqual(venue_key('ECML-PKDD'),'ECML-PKDD')
 
     def test_missing_abstract_is_reported_as_partial_collection(self):
-        config=b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n'
+        config=b'shortname: ICML\nyear: 2023\nbooktitle: ICML\n'
         paper='---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nabstract:\n---\n'
         with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.pmlr.fetch',return_value=config),patch('paper_atlas.collectors.pmlr.metadata_texts',return_value=iter([('paper.md',paper)])):
             result=collect_volume(999,tmp,tmp)

tests/test_pmlr_tracks.py

@@ -0,0 +1,148 @@
+import copy
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+
+from paper_atlas.collectors.pmlr import collect_volume
+from paper_atlas.collectors.pmlr_tracks import program_tracks, record_track, position_list_policy
+from paper_atlas.ingestion.records import ingest_file
+from paper_atlas.ingestion.backfill import backfill
+
+
+def event(forum='realForum1', track='Conference', event_id=1, year=2025, **values):
+    return {'id': event_id, 'sourceid': event_id, 'name': 'A published paper',
+            'sourceurl': f'https://openreview.net/group?id=ICML.cc/{year}/' + track,
+            'paper_url': 'https://openreview.net/forum?id=' + forum,
+            'decision': 'Accept (Poster)', **values}
+
+
+def program(*events, **changes):
+    return json.dumps({'count': len(events), 'results': list(events), **changes}).encode()
+
+
+def record(forum='realForum1', external='v267/paper1', year=2025, title='A published paper'):
+    return {'source': 'pmlr', 'external_id': external, 'venue': 'ICML', 'year': year,
+            'title': title, 'abstract': 'Original abstract', 'authors': ['Jane Doe', 'John Smith'],
+            'identifiers': {'openreview': forum}, 'metadata': {}, 'track': 'main'}
+
+
+class PmlrTrackTests(unittest.TestCase):
+    def provenance(self):
+        return {'url': 'https://icml.cc/static/virtual/data/icml-2025-orals-posters.json', 'sha256': 'a' * 64}
+
+    def test_real_groups_not_title_prefixes_determine_modern_tracks(self):
+        mapping, inventory = program_tracks(program(
+            event(name='Position: still a main-track paper'),
+            event('position1', 'Position_Paper_Track', 2, name='No prefix needed'),
+            event('2025-Oral-fake', event_id=3, sourceid=-1),
+            event('workshop1', 'Workshop', 4)), 2025)
+        main = record(title='Position: still a main-track paper')
+        position = record('position1', 'v267/paper2', title='No prefix needed')
+        record_track(main, mapping, self.provenance()); record_track(position, mapping, self.provenance())
+        self.assertEqual((main['track'], position['track']), ('main', 'position'))
+        self.assertEqual(inventory['paper_identity_count'], 2)
+        self.assertEqual(inventory['excluded_events'], {'synthetic_oral_event': 1, 'other_source': 1})
+
+    def test_conflicting_forums_pagination_and_missing_real_ids_fail(self):
+        for data in [program(event(), event(track='Position_Paper_Track', event_id=2)),
+                     program(event(), next='page2'), program(event(), count=2),
+                     program(event(paper_url='')), program(event(decision='Reject')),
+                     program(event(), event())]:
+            with self.subTest(data=data), self.assertRaises(ValueError):
+                program_tracks(data, 2025)
+
+    def test_missing_publication_identity_has_no_main_track_fallback(self):
+        mapping, _ = program_tracks(program(event()), 2025)
+        with self.assertRaisesRegex(ValueError, 'absent'):
+            record_track(record('unknownForum'), mapping, self.provenance())
+
+    def test_2024_requires_organizer_endorsed_list_and_publisher_self_links(self):
+        html = b'<a href="/virtual/2024/papers.html?filter=titles&amp;search=position%3A">position papers</a>'
+        policy = position_list_policy(html)
+        with self.assertRaises(ValueError):
+            position_list_policy(html.replace(b'2024', b'2023'))
+        e = event(year=2024, name='Position: An explicit position', paper_url='',
+                  paper_pdf_url='https://proceedings.mlr.press/v235/paper1.html')
+        with self.assertRaisesRegex(ValueError, 'policy'):
+            program_tracks(program(e), 2024)
+        mapping, _ = program_tracks(program(e), 2024, policy)
+        row = record(external='v235/paper1', year=2024)
+        record_track(row, mapping, {**self.provenance(), 'position_list_policy': policy})
+        self.assertEqual(row['track'], 'position')
+        self.assertEqual(row['metadata']['track_evidence']['identity_scheme'], 'pmlr')
+
+    def test_reviewed_2024_link_alias_still_needs_title_and_two_authors(self):
+        e = event(year=2024, paper_url='', paper_pdf_url='https://proceedings.mlr.press/v235/theo24a.html',
+                  authors=[{'fullname': 'Jane Doe'}, {'fullname': 'John Smith'}])
+        mapping, _ = program_tracks(program(e), 2024, {'verified': True})
+        row = record(external='v235/guyard24a', year=2024)
+        record_track(row, mapping, self.provenance())
+        self.assertEqual(row['metadata']['track_evidence']['corroborating_author_count'], 2)
+        self.assertNotIn('_authors', row['metadata']['track_evidence'])
+        for changed in [dict(title='Different title'), dict(authors=['Someone Else'])]:
+            bad = record(external='v235/guyard24a', year=2024); bad.update(changed)
+            with self.assertRaisesRegex(ValueError, 'corroboration'):
+                record_track(bad, mapping, self.provenance())
+
+    def test_collector_splits_tracks_preserves_ids_and_imports_one_event_each(self):
+        config = b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n'
+        template = '---\nid: {id}\nlayout: inproceedings\ntitle: {title}\nabstract: Original abstract\nopenreview: {forum}\n---\n'
+        posts = [('a.md', template.format(id='paper1', title='Main paper', forum='realForum1')),
+                 ('b.md', template.format(id='paper2', title='Position paper', forum='position1'))]
+        mapping, inventory = program_tracks(program(event(), event('position1', 'Position_Paper_Track', 2)), 2025)
+        with tempfile.TemporaryDirectory() as tmp, patch('paper_atlas.collectors.pmlr.fetch', return_value=config), \
+                patch('paper_atlas.collectors.pmlr.metadata_texts', return_value=iter(posts)), \
+                patch('paper_atlas.collectors.pmlr.collect_track_evidence', return_value=(mapping, self.provenance())):
+            result = collect_volume(267, tmp, tmp)
+            self.assertNotIn('file', result)
+            self.assertEqual(result['track'], 'mixed')
+            self.assertEqual({m['track']: m['count'] for m in result['files']}, {'main': 1, 'position': 1})
+            self.assertEqual(result['count'], 2)
+            db = Path(tmp) / 'papers.sqlite'
+            for manifest in result['files']:
+                ingest_file(db, manifest['file'])
+            import sqlite3
+            with sqlite3.connect(db) as c:
+                self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 2)
+                self.assertEqual(c.execute('SELECT track,count(*) FROM submissions s JOIN editions e ON s.edition_id=e.id GROUP BY track').fetchall(), [('main', 1), ('position', 1)])
+                self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
+
+    def test_unknown_track_does_not_replace_existing_volume_output(self):
+        config = b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n'
+        post = '---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nopenreview: missing123\n---\n'
+        with tempfile.TemporaryDirectory() as tmp, patch('paper_atlas.collectors.pmlr.fetch', return_value=config), \
+                patch('paper_atlas.collectors.pmlr.metadata_texts', return_value=iter([('a.md', post)])), \
+                patch('paper_atlas.collectors.pmlr.collect_track_evidence', return_value=({}, self.provenance())):
+            old = Path(tmp) / 'pmlr-v267.jsonl'; old.write_text('previous collection')
+            with self.assertRaisesRegex(ValueError, 'absent'):
+                collect_volume(267, tmp, tmp)
+            self.assertEqual(old.read_text(), 'previous collection')
+            self.assertFalse((Path(tmp) / 'pmlr-v267.inventory.json').exists())
+
+    def test_old_mixed_manifest_is_rejected_before_opening_database(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / 'pmlr-v267.jsonl'; path.write_text('')
+            path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'pmlr', 'venue': 'ICML', 'year': 2025, 'parser_version': 2}))
+            db = Path(tmp) / 'absent.sqlite'
+            for action in [lambda: ingest_file(db, path), lambda: backfill(db, [path])]:
+                with self.assertRaisesRegex(ValueError, 'Recollect mixed'):
+                    action()
+            self.assertFalse(db.exists())
+
+    def test_changed_track_or_own_identity_rolls_back_import(self):
+        from paper_atlas.collectors.pmlr_tracks import require_track_record
+        row = record(); mapping, _ = program_tracks(program(event()), 2025)
+        record_track(row, mapping, self.provenance())
+        manifest = {'source': 'pmlr', 'venue': 'ICML', 'year': 2025, 'parser_version': 3,
+                    'track_program': self.provenance()}
+        require_track_record(row, manifest)
+        for change in [dict(track='position'), dict(identifiers={'openreview': 'otherForum'})]:
+            bad = copy.deepcopy(row); bad.update(change)
+            with self.assertRaises(ValueError):
+                require_track_record(bad, manifest)
+
+
+if __name__ == '__main__':
+    unittest.main()