132 lines
7.3 KiB
Python
132 lines
7.3 KiB
Python
"""Collect authoritative ACL Anthology XML metadata, never PDFs."""
|
|
import argparse
|
|
import json
|
|
import re
|
|
import xml.etree.ElementTree as ET
|
|
from pathlib import Path
|
|
|
|
from metadata_checkout import checkout_metadata
|
|
|
|
VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'}
|
|
|
|
|
|
def text(element):
|
|
return ' '.join(''.join(element.itertext()).split()) if element is not None else ''
|
|
|
|
|
|
def paper_id(collection,volume,number):
|
|
if not str(number).isdigit() or not re.fullmatch(r'[A-Za-z0-9_-]+',str(volume)):
|
|
raise ValueError('Invalid Anthology paper numbering')
|
|
if re.fullmatch(r'\d{4}\.[a-z0-9]+',collection):
|
|
return f'{collection}-{volume}.{int(number)}'
|
|
if not re.fullmatch(r'[A-Z]\d{2}',collection) or not str(volume).isdigit():
|
|
raise ValueError('Unsupported historical identifier')
|
|
small=collection.startswith('W') or collection=='C69' or (collection=='D19' and 50<=int(volume)<=66)
|
|
width=2 if small else 1
|
|
digits=2 if small else 3
|
|
if len(str(int(volume)))>width or len(str(int(number)))>digits:
|
|
raise ValueError('Historical identifier exceeds documented width')
|
|
return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}'
|
|
|
|
|
|
def volume_scope(meta,volume):
|
|
codes=[text(n).lower() for n in meta.findall('venue')]
|
|
book=text(meta.find('booktitle'))
|
|
lower=book.lower()
|
|
selected=sorted({VENUES[v] for v in codes if v in VENUES})
|
|
if 'findings' in codes:
|
|
associated=volume.split('-')[0].lower()
|
|
if associated in VENUES:selected=[VENUES[associated]]
|
|
if not selected:
|
|
return None
|
|
joint_names=sorted({(VENUES|{'lrec':'LREC'})[v] for v in codes if v in VENUES or v=='lrec'})
|
|
if len(joint_names)>1:selected=joint_names
|
|
track='main'
|
|
if 'findings' in codes or 'findings of' in lower:track='findings'
|
|
elif 'shared task' in lower:track='shared_task'
|
|
elif 'workshop' in lower or 'student research' in lower:track='workshop'
|
|
elif 'demonstration' in lower or 'system demonstrations' in lower:track='demo'
|
|
elif 'tutorial' in lower:track='tutorial'
|
|
elif 'industry' in lower:track='industry'
|
|
return '+'.join(selected),selected,track
|
|
|
|
|
|
def collect_xml(path,out,commit):
|
|
root=ET.parse(path).getroot()
|
|
collection=root.attrib['id']
|
|
results=[]
|
|
for volume in root.findall('volume'):
|
|
vid=volume.attrib['id']
|
|
meta=volume.find('meta')
|
|
if meta is None:raise ValueError('Volume metadata missing')
|
|
scope=volume_scope(meta,vid)
|
|
key=f'{collection}-{vid}'
|
|
if scope is None:
|
|
results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))})
|
|
continue
|
|
venue,aliases,track=scope
|
|
year_text=text(meta.find('year'))
|
|
if not re.fullmatch(r'\d{4}',year_text):raise ValueError('No explicit official volume year: '+key)
|
|
year=int(year_text)
|
|
records=[]
|
|
frontmatter=0
|
|
for paper in volume.findall('paper'):
|
|
pid=paper.attrib['id']
|
|
if int(pid)==0:
|
|
frontmatter+=1
|
|
continue
|
|
identifier=paper_id(collection,vid,pid)
|
|
doi=text(paper.find('doi'))
|
|
if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold():
|
|
raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier)
|
|
title=text(paper.find('title'))
|
|
if not title:raise ValueError('Missing paper title: '+identifier)
|
|
authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')]
|
|
records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}})
|
|
if not records:
|
|
results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'})
|
|
continue
|
|
target=Path(out)/f'acl-{key}.jsonl'
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
part=target.with_suffix('.part')
|
|
with part.open('w') as output:
|
|
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
|
|
part.replace(target)
|
|
missing=sum(not row['abstract'] for row in records)
|
|
manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'}
|
|
target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2))
|
|
results.append(manifest)
|
|
print(json.dumps(manifest),flush=True)
|
|
return results
|
|
|
|
|
|
if __name__=='__main__':
|
|
parser=argparse.ArgumentParser()
|
|
parser.add_argument('--checkout',required=True)
|
|
parser.add_argument('--out',required=True)
|
|
parser.add_argument('--collection',action='append')
|
|
args=parser.parse_args()
|
|
commit=checkout_metadata('https://github.com/acl-org/acl-anthology.git',args.checkout,['/data/xml/'])
|
|
out=Path(args.out);out.mkdir(parents=True,exist_ok=True)
|
|
status_file=out/'acl-collection-status.json'
|
|
catalogue=json.loads(status_file.read_text()) if status_file.exists() else {}
|
|
paths=sorted((Path(args.checkout)/'data/xml').glob('*.xml'))
|
|
if not paths:raise SystemExit('No official XML files found')
|
|
if args.collection and set(args.collection)-{p.stem for p in paths}:parser.error('Requested collection is absent')
|
|
errors=[]
|
|
processed=0
|
|
for path in paths:
|
|
if args.collection and path.stem not in args.collection:continue
|
|
try:catalogue[path.stem]={'status':'processed','metadata_commit':commit,'volumes':collect_xml(path,args.out,commit)}
|
|
except Exception as exc:
|
|
catalogue[path.stem]={'status':'blocked','metadata_commit':commit,'error':str(exc)}
|
|
errors.append(path.stem)
|
|
print(json.dumps({'collection':path.stem,'error':str(exc)}),flush=True)
|
|
processed+=1
|
|
if processed%25==0:
|
|
part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file)
|
|
print(json.dumps({'xml_collections_processed':processed}),flush=True)
|
|
part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file)
|
|
(out/'acl-last-run.json').write_text(json.dumps({'metadata_commit':commit,'collection_filter':args.collection,'blocked_collections':errors},indent=2))
|
|
if errors:raise SystemExit(1)
|