Files
paper-atlas/paper_atlas/collectors/aamas.py
T
codex-bot 13408869c2
Metadata integrity checks / metadata (push) Successful in 2m11s
Add reviewed AAMAS 2025 publisher metadata adapter
2026-11-09 20:50:29 +08:00

214 lines
12 KiB
Python

"""Official AAMAS proceedings metadata, with publication format separate from track."""
import argparse
from collections import Counter, defaultdict
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
from urllib.parse import urljoin, urlsplit
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.http import fetch
SOURCE = 'aamas_proceedings'
ROOT = 'https://www.ifaamas.org/Proceedings/aamas2012/'
# Counts independently checked against the frozen publisher sections. This
# adapter deliberately refuses other editions/layouts until they are reviewed.
SECTIONS = {
'fullpapers': ('Full Papers', 'full_paper', 'proceedings_unspecified', 136),
'extabstracts': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 146),
'demos': ('Demos', 'demo', 'demo', 28),
'doctorals': ('Doctorial Consortium Abstracts', 'doctoral_abstract', 'doctoral', 24),
}
def classes(node):
return set(node.attrs.get('class', '').split())
def text_without(node, omitted):
if node.tag in ('script', 'style') or classes(node) & omitted:
return ''
parts = [text_without(child, omitted) if isinstance(child, Node) else child
for child in node.children]
value = ''.join(parts)
return ' '+value+' ' if node.tag in ('p', 'div', 'section', 'li', 'br') else value
def one(nodes, description):
if len(nodes) != 1:
raise ValueError('Expected one '+description)
return nodes[0]
def parse_section(html, section, observed_at, evidence):
heading, publication_format, track, _ = SECTIONS[section]
document = Document(html)
titles = [clean(n.text()) for n in document.root.walk() if n.tag == 'title']
if titles != [heading+' - AAMAS 2012']:
raise ValueError('Unexpected proceedings edition or section title')
headings = [clean(n.text()) for n in document.root.walk() if n.tag == 'h1']
if headings != [heading]:
raise ValueError('Unexpected proceedings section heading')
url = ROOT+'resources/'+section+'.html'
records = []
seen = set()
session = None
for node in document.root.walk():
if node.tag == 'h2' and 'section' in classes(node):
session = {'id': node.attrs.get('id'), 'title': clean(node.text())}
if node.tag != 'div' or 'paper' not in classes(node):
continue
paper_id = node.attrs.get('id', '')
if not re.fullmatch(r'[A-Za-z0-9]+_[0-9]+', paper_id) or paper_id in seen:
raise ValueError('Missing or repeated publisher paper ID')
seen.add(paper_id)
title_node = one([n for n in node.walk() if n.tag == 'span' and 'title' in classes(n)], 'paper title')
link = one([n for n in title_node.children if isinstance(n, Node) and n.tag == 'a'], 'paper link')
paper_url = urljoin(url, link.attrs.get('href', ''))
if paper_url != ROOT+'papers/'+paper_id+'.pdf':
raise ValueError('Paper link differs from publisher ID or official host')
title = clean(text_without(link, {'abstract'}))
abstract_nodes = [n for n in link.walk() if n.tag == 'span' and 'abstract' in classes(n)]
if len(abstract_nodes) > 1:
raise ValueError('Multiple abstracts for one paper')
abstract = (clean(abstract_nodes[0].text()) or None) if abstract_nodes else None
author_node = one([n for n in node.walk() if n.tag == 'span' and 'authors' in classes(n)], 'author list')
author_display = clean(text_without(author_node, {'amount'}))
authors = [clean(name) for name in author_display.split(',')]
if not title or not author_display or any(not name for name in authors):
raise ValueError('Empty paper title or author')
tags = [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]
if session:
tags.append({'namespace': 'aamas_proceedings:session', 'value': session['title']})
records.append({
'source': SOURCE, 'external_id': '2012/'+paper_id,
'url': url+'#'+paper_id, 'title': title, 'abstract': abstract,
'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
'year': 2012, 'track': track, 'status': 'accepted',
'observed_at': observed_at,
'raw_decision': 'listed in official AAMAS 2012 proceedings '+heading,
'metadata': {'publisher_paper_id': paper_id, 'paper_url': paper_url,
'section_url': url, 'section_heading': heading,
'publication_format': publication_format, 'session': session,
'authors_display': author_display, 'source_evidence': evidence,
'publication_evidence': 'official published proceedings entry; not an original decision note',
'track_scope_note': ('Full papers and extended abstracts include general and special submission tracks; '
'session topics do not establish an individual submission track.'
if track == 'proceedings_unspecified' else 'Explicit publisher section.'),
'decision_date_precision': 'not_supplied'},
'tags': tags})
if not records:
raise ValueError('No publisher paper entries found')
return records
def read_pages(cache=None, snapshots=None):
if bool(cache) == bool(snapshots):
raise ValueError('Choose exactly one HTTP cache or frozen snapshot directory')
urls = {'introduction': ROOT, **{key: ROOT+'resources/'+key+'.html' for key in SECTIONS}}
pages = {}
receipts = []
frozen = json.loads((Path(snapshots)/'receipts.json').read_text())['receipts'] if snapshots else []
for key, url in urls.items():
if snapshots:
receipt = one([r for r in frozen if r['url'] == url], 'frozen response receipt')
filename = Path(receipt['file'])
if filename.is_absolute() or '..' in filename.parts or receipt.get('status') != 200:
raise ValueError('Unsafe or unsuccessful frozen response')
data = (Path(snapshots)/filename).read_bytes()
if len(data) > 8*1024**2 or hashlib.sha256(data).hexdigest() != receipt['sha256'] or len(data) != receipt['bytes']:
raise ValueError('Frozen publisher response checksum or size mismatch')
observed = receipt['finished_at']
else:
data = fetch(url, cache, 8*1024**2, delay=1)
receipt = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_text())
observed = receipt['retrieved_at']
evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html', 'observed_at': observed,
'http_status': 200, 'response_receipt': receipt}
pages[key] = (data, evidence)
receipts.append(evidence)
return pages, receipts
def _collect(year, out, *, cache=None, snapshots=None):
if year == 2025:
from paper_atlas.collectors.aamas2025 import collect as collect_2025
return collect_2025(out, cache=cache, snapshots=snapshots)
if year != 2012:
raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported')
pages, receipts = read_pages(cache, snapshots)
intro = Document(pages['introduction'][0].decode('utf-8'))
if [clean(n.text()) for n in intro.root.walk() if n.tag == 'title'] != ['AAMAS 2012']:
raise ValueError('Unexpected proceedings introduction')
records = []
for section, (_, _, _, expected) in SECTIONS.items():
data, evidence = pages[section]
group = parse_section(data.decode('utf-8'), section, evidence['observed_at'], evidence)
if len(group) != expected:
raise ValueError('Publisher section count differs from reviewed inventory: '+section)
records.extend(group)
if len({r['external_id'] for r in records}) != len(records):
raise ValueError('Paper IDs collide across publisher sections')
grouped = defaultdict(list)
for record in records:
grouped[record['track']].append(record)
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
files = []
with tempfile.TemporaryDirectory(prefix='.aamas-', dir=out) as temporary:
stage = Path(temporary)
(stage/'evidence').mkdir()
for data, evidence in pages.values():
(stage/evidence['file']).write_bytes(data)
for track, group in sorted(grouped.items()):
path = stage/f'aamas-proceedings-2012-{track}.jsonl'
path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'AAMAS', 'year': 2012, 'track': track,
'count': len(group), 'file': str(out/path.name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'list_url': ROOT, 'status': 'partial', 'missing_abstract_count': sum(not r['abstract'] for r in group),
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
'note': 'Publisher sections verified; individual full/extended submission track, decision history and independent identities still need review.'}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
files.append(manifest)
result = {'source': SOURCE, 'year': year, 'finished_at': timestamp(), 'status': 'collected_pending_identity_review',
'records': len(records), 'files': files, 'source_receipts': receipts,
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in records)),
'database_written': False, 'pdfs_downloaded': False,
'note': 'Aggregate submission/withdrawal totals do not identify individual rejected or withdrawn papers.'}
(stage/'aamas-proceedings-2012.json').write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
# Validated files first; write the completed collection report last.
(out/'evidence').mkdir(exist_ok=True)
for path in (stage/'evidence').iterdir():
path.replace(out/'evidence'/path.name)
for path in stage.iterdir():
if path.is_file() and path.name != 'aamas-proceedings-2012.json':
path.replace(out/path.name)
(stage/'aamas-proceedings-2012.json').replace(out/'aamas-proceedings-2012.json')
return result
def collect(year, out, *, cache=None, snapshots=None):
if year not in (2012, 2025):
raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported')
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
with (out/'aamas.collect.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return _collect(year, out, cache=cache, snapshots=snapshots)
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--year', type=int, choices=[2012, 2025], required=True)
parser.add_argument('--out', required=True)
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument('--cache')
source.add_argument('--snapshots')
args = parser.parse_args(argv)
print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))