"""Collect authoritative ACL Anthology XML metadata, never PDFs.""" import argparse import json import re import xml.etree.ElementTree as ET from pathlib import Path from metadata_checkout import checkout_metadata VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'} def text(element): return ' '.join(''.join(element.itertext()).split()) if element is not None else '' def paper_id(collection,volume,number): if not str(number).isdigit() or not re.fullmatch(r'[A-Za-z0-9_-]+',str(volume)): raise ValueError('Invalid Anthology paper numbering') if re.fullmatch(r'\d{4}\.[a-z0-9]+',collection): return f'{collection}-{volume}.{int(number)}' if not re.fullmatch(r'[A-Z]\d{2}',collection) or not str(volume).isdigit(): raise ValueError('Unsupported historical identifier') small=collection.startswith('W') or collection=='C69' or (collection=='D19' and 50<=int(volume)<=66) width=2 if small else 1 digits=2 if small else 3 if len(str(int(volume)))>width or len(str(int(number)))>digits: raise ValueError('Historical identifier exceeds documented width') return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}' def volume_scope(meta,volume): codes=[text(n).lower() for n in meta.findall('venue')] book=text(meta.find('booktitle')) lower=book.lower() selected=sorted({VENUES[v] for v in codes if v in VENUES}) if 'findings' in codes: associated=volume.split('-')[0].lower() if associated in VENUES:selected=[VENUES[associated]] if not selected: return None joint_names=sorted({(VENUES|{'lrec':'LREC'})[v] for v in codes if v in VENUES or v=='lrec'}) if len(joint_names)>1:selected=joint_names track='main' if 'findings' in codes or 'findings of' in lower:track='findings' elif 'shared task' in lower:track='shared_task' elif 'workshop' in lower or 'student research' in lower:track='workshop' elif 'demonstration' in lower or 'system demonstrations' in lower:track='demo' elif 'tutorial' in lower:track='tutorial' elif 'industry' in lower:track='industry' return '+'.join(selected),selected,track def collect_xml(path,out,commit): root=ET.parse(path).getroot() collection=root.attrib['id'] results=[] for volume in root.findall('volume'): vid=volume.attrib['id'] meta=volume.find('meta') if meta is None:raise ValueError('Volume metadata missing') scope=volume_scope(meta,vid) key=f'{collection}-{vid}' if scope is None: results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))}) continue venue,aliases,track=scope year_text=text(meta.find('year')) if not re.fullmatch(r'\d{4}',year_text):raise ValueError('No explicit official volume year: '+key) year=int(year_text) records=[] frontmatter=0 for paper in volume.findall('paper'): pid=paper.attrib['id'] if int(pid)==0: frontmatter+=1 continue identifier=paper_id(collection,vid,pid) doi=text(paper.find('doi')) if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold(): raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier) title=text(paper.find('title')) if not title:raise ValueError('Missing paper title: '+identifier) authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')] records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}}) if not records: results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'}) continue target=Path(out)/f'acl-{key}.jsonl' target.parent.mkdir(parents=True,exist_ok=True) part=target.with_suffix('.part') with part.open('w') as output: for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n') part.replace(target) missing=sum(not row['abstract'] for row in records) manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'} target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)) results.append(manifest) print(json.dumps(manifest),flush=True) return results if __name__=='__main__': parser=argparse.ArgumentParser() parser.add_argument('--checkout',required=True) parser.add_argument('--out',required=True) parser.add_argument('--collection',action='append') args=parser.parse_args() commit=checkout_metadata('https://github.com/acl-org/acl-anthology.git',args.checkout,['/data/xml/']) out=Path(args.out);out.mkdir(parents=True,exist_ok=True) status_file=out/'acl-collection-status.json' catalogue=json.loads(status_file.read_text()) if status_file.exists() else {} paths=sorted((Path(args.checkout)/'data/xml').glob('*.xml')) if not paths:raise SystemExit('No official XML files found') if args.collection and set(args.collection)-{p.stem for p in paths}:parser.error('Requested collection is absent') errors=[] processed=0 for path in paths: if args.collection and path.stem not in args.collection:continue try:catalogue[path.stem]={'status':'processed','metadata_commit':commit,'volumes':collect_xml(path,args.out,commit)} except Exception as exc: catalogue[path.stem]={'status':'blocked','metadata_commit':commit,'error':str(exc)} errors.append(path.stem) print(json.dumps({'collection':path.stem,'error':str(exc)}),flush=True) processed+=1 if processed%25==0: part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file) print(json.dumps({'xml_collections_processed':processed}),flush=True) part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file) (out/'acl-last-run.json').write_text(json.dumps({'metadata_commit':commit,'collection_filter':args.collection,'blocked_collections':errors},indent=2)) if errors:raise SystemExit(1)