66 lines
4.2 KiB
Python
66 lines
4.2 KiB
Python
import unittest
|
|
import json
|
|
import tempfile
|
|
from pathlib import Path
|
|
from unittest.mock import patch
|
|
|
|
from paper_atlas.collectors.proceedings import paper_links,parse_paper,main
|
|
|
|
|
|
class ProceedingsTests(unittest.TestCase):
|
|
def test_legacy_empty_class_attribute_does_not_break_a_paper(self):
|
|
html='<div class>Legacy formatting</div><h1>Paper</h1><p class="paper-abstract">Real abstract.</p>'
|
|
row=parse_paper(html,'https://proceedings.neurips.cc/paper/2000/hash/a-Abstract.html','neurips',2000,'https://proceedings.neurips.cc/paper/2000')
|
|
self.assertEqual(row['abstract'],'Real abstract.')
|
|
|
|
def test_colm_nested_abstract_and_citation_authors(self):
|
|
html='''<meta name="citation_title" content="A paper"><meta name="citation_author" content="Doe, Jane"><section><h2>Abstract</h2><p class="paper-abstract"><p>Real <em>abstract</em> text.</p></p></section><h4>Name Change Policy</h4><p>Do not ingest footer.</p>'''
|
|
row=parse_paper(html,'https://proceedings.colm.cc/paper_files/paper/2024/hash/a-Abstract-Conference.html','colm',2024,'https://proceedings.colm.cc/paper_files/paper/2024')
|
|
self.assertEqual(row['abstract'],'Real abstract text.')
|
|
self.assertEqual(row['authors'],['Jane Doe'])
|
|
self.assertEqual(row['track'],'main')
|
|
self.assertEqual(row['year'],2024)
|
|
|
|
def test_declared_index_count_detects_truncation(self):
|
|
html='<p>2 papers</p><a href="/paper/2024/hash/a-Abstract.html">Paper</a>'
|
|
with self.assertRaisesRegex(ValueError,'count differs'):
|
|
paper_links(html,'https://proceedings.colm.cc/paper/2024')
|
|
|
|
def test_missing_abstract_does_not_capture_footer(self):
|
|
row=parse_paper('<h1>Paper</h1><h4>Name Change Policy</h4><p>Footer</p>','https://example.org/a','colm',2024,'https://example.org/')
|
|
self.assertIsNone(row['abstract'])
|
|
|
|
def test_paper_title_is_not_mistaken_for_index_paper_count(self):
|
|
html='<li><a href="/paper/2021/hash/a-Abstract.html">Lessons from 1000 papers</a></li>'
|
|
self.assertEqual(len(paper_links(html,'https://datasets-benchmarks-proceedings.neurips.cc/paper/2021')),1)
|
|
|
|
def test_neurips_track_and_legacy_abstract(self):
|
|
html='<h1>Paper</h1><span class="paper-track">Datasets and Benchmarks Track</span><h4>Abstract</h4><p>Actual abstract.</p><h4>Name Change Policy</h4><p>Footer</p>'
|
|
row=parse_paper(html,'https://proceedings.neurips.cc/a','neurips',2025,'https://proceedings.neurips.cc/')
|
|
self.assertEqual(row['abstract'],'Actual abstract.')
|
|
self.assertEqual(row['track'],'datasets_benchmarks')
|
|
|
|
def test_focused_retry_preserves_other_index_status(self):
|
|
root='https://proceedings.colm.cc/'
|
|
url=root+'paper_files/paper/2024'
|
|
other=root+'paper_files/paper/2025'
|
|
paper=root+'paper_files/paper/2024/hash/a-Abstract-Conference.html'
|
|
documents={root:f'<a href="{url}">2024</a><a href="{other}">2025</a>',url:f'<p>1 papers</p><a href="{paper}">Paper</a>',paper:'<h1>Paper</h1><p class="paper-abstract">Abstract</p>'}
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
status=Path(tmp)/'colm-index-status.json'
|
|
status.write_text(json.dumps({other:{'status':'collected','files':[]}}))
|
|
argv=['paper-atlas','--source','colm','--cache',tmp,'--out',tmp,'--year','2024']
|
|
with patch('sys.argv',argv),patch('paper_atlas.collectors.proceedings.fetch',side_effect=lambda u,*a,**kw:documents[u].encode()):
|
|
main()
|
|
result=json.loads(status.read_text())
|
|
self.assertEqual(result[other]['status'],'collected')
|
|
self.assertEqual(result[url]['files'][0]['count'],1)
|
|
|
|
def test_abstract_citations_are_not_used_as_self_identifiers(self):
|
|
html='<meta name="citation_title" content="Paper"><div class="paper-actions"><a href="https://openreview.net/forum?id=selfForum">OpenReview</a></div><p class="paper-abstract">Related <a href="https://arxiv.org/abs/1706.03762">work</a> and <a href="https://openreview.net/forum?id=otherForum">another paper</a>.</p>'
|
|
row=parse_paper(html,'https://proceedings.colm.cc/paper','colm',2024,'https://proceedings.colm.cc/')
|
|
self.assertEqual(row['identifiers'],{'openreview':'selfForum'})
|
|
|
|
|
|
if __name__=='__main__':unittest.main()
|