214 lines
12 KiB
Python
214 lines
12 KiB
Python
"""Official AAMAS proceedings metadata, with publication format separate from track."""
|
|
import argparse
|
|
from collections import Counter, defaultdict
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
from urllib.parse import urljoin, urlsplit
|
|
|
|
from paper_atlas.db.submissions import timestamp
|
|
from paper_atlas.io.html import Document, Node, clean
|
|
from paper_atlas.io.http import fetch
|
|
|
|
SOURCE = 'aamas_proceedings'
|
|
ROOT = 'https://www.ifaamas.org/Proceedings/aamas2012/'
|
|
# Counts independently checked against the frozen publisher sections. This
|
|
# adapter deliberately refuses other editions/layouts until they are reviewed.
|
|
SECTIONS = {
|
|
'fullpapers': ('Full Papers', 'full_paper', 'proceedings_unspecified', 136),
|
|
'extabstracts': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 146),
|
|
'demos': ('Demos', 'demo', 'demo', 28),
|
|
'doctorals': ('Doctorial Consortium Abstracts', 'doctoral_abstract', 'doctoral', 24),
|
|
}
|
|
|
|
|
|
def classes(node):
|
|
return set(node.attrs.get('class', '').split())
|
|
|
|
|
|
def text_without(node, omitted):
|
|
if node.tag in ('script', 'style') or classes(node) & omitted:
|
|
return ''
|
|
parts = [text_without(child, omitted) if isinstance(child, Node) else child
|
|
for child in node.children]
|
|
value = ''.join(parts)
|
|
return ' '+value+' ' if node.tag in ('p', 'div', 'section', 'li', 'br') else value
|
|
|
|
|
|
def one(nodes, description):
|
|
if len(nodes) != 1:
|
|
raise ValueError('Expected one '+description)
|
|
return nodes[0]
|
|
|
|
|
|
def parse_section(html, section, observed_at, evidence):
|
|
heading, publication_format, track, _ = SECTIONS[section]
|
|
document = Document(html)
|
|
titles = [clean(n.text()) for n in document.root.walk() if n.tag == 'title']
|
|
if titles != [heading+' - AAMAS 2012']:
|
|
raise ValueError('Unexpected proceedings edition or section title')
|
|
headings = [clean(n.text()) for n in document.root.walk() if n.tag == 'h1']
|
|
if headings != [heading]:
|
|
raise ValueError('Unexpected proceedings section heading')
|
|
url = ROOT+'resources/'+section+'.html'
|
|
records = []
|
|
seen = set()
|
|
session = None
|
|
for node in document.root.walk():
|
|
if node.tag == 'h2' and 'section' in classes(node):
|
|
session = {'id': node.attrs.get('id'), 'title': clean(node.text())}
|
|
if node.tag != 'div' or 'paper' not in classes(node):
|
|
continue
|
|
paper_id = node.attrs.get('id', '')
|
|
if not re.fullmatch(r'[A-Za-z0-9]+_[0-9]+', paper_id) or paper_id in seen:
|
|
raise ValueError('Missing or repeated publisher paper ID')
|
|
seen.add(paper_id)
|
|
title_node = one([n for n in node.walk() if n.tag == 'span' and 'title' in classes(n)], 'paper title')
|
|
link = one([n for n in title_node.children if isinstance(n, Node) and n.tag == 'a'], 'paper link')
|
|
paper_url = urljoin(url, link.attrs.get('href', ''))
|
|
if paper_url != ROOT+'papers/'+paper_id+'.pdf':
|
|
raise ValueError('Paper link differs from publisher ID or official host')
|
|
title = clean(text_without(link, {'abstract'}))
|
|
abstract_nodes = [n for n in link.walk() if n.tag == 'span' and 'abstract' in classes(n)]
|
|
if len(abstract_nodes) > 1:
|
|
raise ValueError('Multiple abstracts for one paper')
|
|
abstract = (clean(abstract_nodes[0].text()) or None) if abstract_nodes else None
|
|
author_node = one([n for n in node.walk() if n.tag == 'span' and 'authors' in classes(n)], 'author list')
|
|
author_display = clean(text_without(author_node, {'amount'}))
|
|
authors = [clean(name) for name in author_display.split(',')]
|
|
if not title or not author_display or any(not name for name in authors):
|
|
raise ValueError('Empty paper title or author')
|
|
tags = [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]
|
|
if session:
|
|
tags.append({'namespace': 'aamas_proceedings:session', 'value': session['title']})
|
|
records.append({
|
|
'source': SOURCE, 'external_id': '2012/'+paper_id,
|
|
'url': url+'#'+paper_id, 'title': title, 'abstract': abstract,
|
|
'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
|
|
'year': 2012, 'track': track, 'status': 'accepted',
|
|
'observed_at': observed_at,
|
|
'raw_decision': 'listed in official AAMAS 2012 proceedings '+heading,
|
|
'metadata': {'publisher_paper_id': paper_id, 'paper_url': paper_url,
|
|
'section_url': url, 'section_heading': heading,
|
|
'publication_format': publication_format, 'session': session,
|
|
'authors_display': author_display, 'source_evidence': evidence,
|
|
'publication_evidence': 'official published proceedings entry; not an original decision note',
|
|
'track_scope_note': ('Full papers and extended abstracts include general and special submission tracks; '
|
|
'session topics do not establish an individual submission track.'
|
|
if track == 'proceedings_unspecified' else 'Explicit publisher section.'),
|
|
'decision_date_precision': 'not_supplied'},
|
|
'tags': tags})
|
|
if not records:
|
|
raise ValueError('No publisher paper entries found')
|
|
return records
|
|
|
|
|
|
def read_pages(cache=None, snapshots=None):
|
|
if bool(cache) == bool(snapshots):
|
|
raise ValueError('Choose exactly one HTTP cache or frozen snapshot directory')
|
|
urls = {'introduction': ROOT, **{key: ROOT+'resources/'+key+'.html' for key in SECTIONS}}
|
|
pages = {}
|
|
receipts = []
|
|
frozen = json.loads((Path(snapshots)/'receipts.json').read_text())['receipts'] if snapshots else []
|
|
for key, url in urls.items():
|
|
if snapshots:
|
|
receipt = one([r for r in frozen if r['url'] == url], 'frozen response receipt')
|
|
filename = Path(receipt['file'])
|
|
if filename.is_absolute() or '..' in filename.parts or receipt.get('status') != 200:
|
|
raise ValueError('Unsafe or unsuccessful frozen response')
|
|
data = (Path(snapshots)/filename).read_bytes()
|
|
if len(data) > 8*1024**2 or hashlib.sha256(data).hexdigest() != receipt['sha256'] or len(data) != receipt['bytes']:
|
|
raise ValueError('Frozen publisher response checksum or size mismatch')
|
|
observed = receipt['finished_at']
|
|
else:
|
|
data = fetch(url, cache, 8*1024**2, delay=1)
|
|
receipt = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_text())
|
|
observed = receipt['retrieved_at']
|
|
evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
|
|
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html', 'observed_at': observed,
|
|
'http_status': 200, 'response_receipt': receipt}
|
|
pages[key] = (data, evidence)
|
|
receipts.append(evidence)
|
|
return pages, receipts
|
|
|
|
|
|
def _collect(year, out, *, cache=None, snapshots=None):
|
|
if year == 2025:
|
|
from paper_atlas.collectors.aamas2025 import collect as collect_2025
|
|
return collect_2025(out, cache=cache, snapshots=snapshots)
|
|
if year != 2012:
|
|
raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported')
|
|
pages, receipts = read_pages(cache, snapshots)
|
|
intro = Document(pages['introduction'][0].decode('utf-8'))
|
|
if [clean(n.text()) for n in intro.root.walk() if n.tag == 'title'] != ['AAMAS 2012']:
|
|
raise ValueError('Unexpected proceedings introduction')
|
|
records = []
|
|
for section, (_, _, _, expected) in SECTIONS.items():
|
|
data, evidence = pages[section]
|
|
group = parse_section(data.decode('utf-8'), section, evidence['observed_at'], evidence)
|
|
if len(group) != expected:
|
|
raise ValueError('Publisher section count differs from reviewed inventory: '+section)
|
|
records.extend(group)
|
|
if len({r['external_id'] for r in records}) != len(records):
|
|
raise ValueError('Paper IDs collide across publisher sections')
|
|
grouped = defaultdict(list)
|
|
for record in records:
|
|
grouped[record['track']].append(record)
|
|
out = Path(out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
files = []
|
|
with tempfile.TemporaryDirectory(prefix='.aamas-', dir=out) as temporary:
|
|
stage = Path(temporary)
|
|
(stage/'evidence').mkdir()
|
|
for data, evidence in pages.values():
|
|
(stage/evidence['file']).write_bytes(data)
|
|
for track, group in sorted(grouped.items()):
|
|
path = stage/f'aamas-proceedings-2012-{track}.jsonl'
|
|
path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
|
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'AAMAS', 'year': 2012, 'track': track,
|
|
'count': len(group), 'file': str(out/path.name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
'list_url': ROOT, 'status': 'partial', 'missing_abstract_count': sum(not r['abstract'] for r in group),
|
|
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
|
|
'note': 'Publisher sections verified; individual full/extended submission track, decision history and independent identities still need review.'}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
|
|
files.append(manifest)
|
|
result = {'source': SOURCE, 'year': year, 'finished_at': timestamp(), 'status': 'collected_pending_identity_review',
|
|
'records': len(records), 'files': files, 'source_receipts': receipts,
|
|
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in records)),
|
|
'database_written': False, 'pdfs_downloaded': False,
|
|
'note': 'Aggregate submission/withdrawal totals do not identify individual rejected or withdrawn papers.'}
|
|
(stage/'aamas-proceedings-2012.json').write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
|
# Validated files first; write the completed collection report last.
|
|
(out/'evidence').mkdir(exist_ok=True)
|
|
for path in (stage/'evidence').iterdir():
|
|
path.replace(out/'evidence'/path.name)
|
|
for path in stage.iterdir():
|
|
if path.is_file() and path.name != 'aamas-proceedings-2012.json':
|
|
path.replace(out/path.name)
|
|
(stage/'aamas-proceedings-2012.json').replace(out/'aamas-proceedings-2012.json')
|
|
return result
|
|
|
|
|
|
def collect(year, out, *, cache=None, snapshots=None):
|
|
if year not in (2012, 2025):
|
|
raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported')
|
|
out = Path(out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
with (out/'aamas.collect.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
return _collect(year, out, cache=cache, snapshots=snapshots)
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--year', type=int, choices=[2012, 2025], required=True)
|
|
parser.add_argument('--out', required=True)
|
|
source = parser.add_mutually_exclusive_group(required=True)
|
|
source.add_argument('--cache')
|
|
source.add_argument('--snapshots')
|
|
args = parser.parse_args(argv)
|
|
print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))
|