Compare commits

...

1 Commits

Author SHA1 Message Date
codex-bot b1429cd3d8 Preserve the CoRL 2021 Blue Sky publication track
Metadata integrity checks / metadata (push) Successful in 10m39s
2026-11-22 19:31:39 +08:00
2 changed files with 29 additions and 2 deletions
+11 -2
View File
@@ -103,6 +103,8 @@ def collect_volume(volume,cache,out,allowed=None):
path = Path(out)/f'pmlr-v{volume}.jsonl'
path.parent.mkdir(parents=True,exist_ok=True)
mixed_icml = venue == 'ICML' and year >= 2024
corl_blue_sky_volume = volume == 164 and venue == 'CORL' and year == 2021
corl_section_tracks = {'contributed papers': 'main', 'blue sky papers': 'blue_sky'}
mapping, track_program = collect_track_evidence(year, cache) if mixed_icml else ({}, None)
records = {}
prefaces = set()
@@ -120,7 +122,8 @@ def collect_volume(volume,cache,out,allowed=None):
raise ValueError('Paper metadata has no publication ID: '+member_name)
external = f"v{volume}/{row['id']}"
# PMLR also gives volume prefaces the inproceedings layout.
if str(row.get('section') or '').strip().casefold() == 'preface':
section = str(row.get('section') or '').strip()
if section.casefold() == 'preface':
prefaces.add(external)
continue
ids = {}
@@ -134,6 +137,12 @@ def collect_volume(volume,cache,out,allowed=None):
value = value.split('id=')[-1].split('&')[0]
ids[key] = value
record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}}
if corl_blue_sky_volume:
section_track = corl_section_tracks.get(section.casefold())
if section_track is None:
raise ValueError(f'Unrecognized CoRL 2021 section for {external}: {section!r}')
record['track'] = section_track
record['metadata']['pmlr_section'] = section
version={'path':member_name,'date':row.get('date')}
if external in records:
previous=records[external]
@@ -163,7 +172,7 @@ def collect_volume(volume,cache,out,allowed=None):
with tempfile.TemporaryDirectory(prefix=f'.pmlr-v{volume}-', dir=path.parent) as temporary:
stage = Path(temporary)
for current_track, rows in sorted(grouped.items()):
filename = f'pmlr-v{volume}-{current_track}.jsonl' if mixed_icml else path.name
filename = f'pmlr-v{volume}-{current_track}.jsonl' if mixed_icml or current_track != track else path.name
output = stage / filename
output.write_text(''.join(json.dumps(row, ensure_ascii=False, default=str)+'\n' for row in rows))
absent = sum(not (row.get('abstract') or '').strip() for row in rows)
+18
View File
@@ -60,6 +60,24 @@ class MetadataTests(unittest.TestCase):
self.assertEqual(result['files'][0]['count'], 1)
self.assertEqual([row['external_id'] for row in rows], ['v244/paper1'])
def test_corl_2021_sections_keep_blue_sky_apart_from_contributed_papers(self):
config = b'shortname: CoRL\nyear: 2021\nbooktitle: Proceedings of the 5th Conference on Robot Learning\n'
paper = '---\nid: {id}\nlayout: inproceedings\ntitle: Paper\nabstract: Abstract\nsection: {section}\n---\n'
posts = [('one.md', paper.format(id='one', section='Contributed Papers')),
('two.md', paper.format(id='two', section='Blue Sky Papers'))]
with tempfile.TemporaryDirectory() as tmp, patch('paper_atlas.collectors.pmlr.fetch', return_value=config), patch(
'paper_atlas.collectors.pmlr.metadata_texts', return_value=iter(posts),
):
result = collect_volume(164, tmp, tmp)
self.assertEqual(result['track'], 'mixed')
files = {manifest['track']: Path(manifest['file']) for manifest in result['files']}
self.assertEqual({key: path.name for key, path in files.items()},
{'main': 'pmlr-v164.jsonl', 'blue_sky': 'pmlr-v164-blue_sky.jsonl'})
rows = {track: json.loads(path.read_text()) for track, path in files.items()}
self.assertEqual([(rows[t]['external_id'], rows[t]['track'], rows[t]['metadata']['pmlr_section'])
for t in ('main', 'blue_sky')],
[('v164/one', 'main', 'Contributed Papers'), ('v164/two', 'blue_sky', 'Blue Sky Papers')])
def test_conflicting_duplicate_posts_do_not_replace_previous_collection(self):
config=b'shortname: CoRL\nyear: 2023\nbooktitle: CoRL\n'
paper='---\nid: paper1\nlayout: inproceedings\ntitle: {title}\nabstract: Abstract\n---\n'