82 lines
4.9 KiB
Python
82 lines
4.9 KiB
Python
"""Exact AMTCL volume provenance and native metadata/history preservation."""
|
|
import copy
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
import unittest
|
|
import xml.etree.ElementTree as ET
|
|
|
|
from paper_atlas.collectors.acl import collect_xml, volume_scope, text
|
|
from paper_atlas.collectors.acl_historical import event_scope
|
|
from paper_atlas.ingestion.records import ingest_file
|
|
|
|
FIXTURE = Path(__file__).parent/'fixtures/acl-1963-earlymt.xml'
|
|
COMMIT = 'a7be623d01c48c61405dec31c871121cdbed5a01'
|
|
|
|
|
|
class Acl1963Tests(unittest.TestCase):
|
|
def setUp(self):
|
|
self.volume = ET.parse(FIXTURE).getroot().find('volume')
|
|
self.meta = self.volume.find('meta')
|
|
|
|
def test_official_snapshot_and_exact_scope(self):
|
|
self.assertEqual(hashlib.sha256(FIXTURE.read_bytes()).hexdigest(), '4a4626d792ac507de0281cb358769f7246f4a0d6b6a9176ac02e61545c1635fc')
|
|
self.assertEqual(len(self.volume.findall('paper')), 34)
|
|
self.assertEqual(volume_scope(self.meta, '1', '1963.earlymt'), ('ACL', ['ACL', 'AMTCL'], 'proceedings_unspecified'))
|
|
scope = event_scope(self.meta, '1', '1963.earlymt')
|
|
self.assertEqual(scope['original_venue'], 'EarlyMT')
|
|
self.assertEqual(scope['publication_format'], 'published_abstract')
|
|
self.assertEqual(scope['event_dates'], ['1963-08-25', '1963-08-26'])
|
|
|
|
def test_other_earlymt_collections_and_volumes_stay_excluded(self):
|
|
for collection, volume in [('1962.earlymt', '1'), ('1971.earlymt', '1'), ('1963.earlymt', '2'), ('1963.other', '1')]:
|
|
with self.subTest(collection=collection, volume=volume):
|
|
self.assertIsNone(volume_scope(self.meta, volume, collection))
|
|
self.assertIsNone(volume_scope(self.meta, '1'))
|
|
|
|
def test_scope_metadata_drift_and_duplicates_are_rejected(self):
|
|
for field in ['venue', 'year', 'booktitle', 'address', 'month']:
|
|
for duplicate in [False, True]:
|
|
meta = copy.deepcopy(self.meta)
|
|
if duplicate:
|
|
ET.SubElement(meta, field).text = text(meta.find(field))
|
|
else:
|
|
meta.find(field).text = 'changed'
|
|
with self.subTest(field=field, duplicate=duplicate), self.assertRaises(ValueError):
|
|
volume_scope(meta, '1', '1963.earlymt')
|
|
|
|
def test_every_original_contribution_survives_collection(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
manifest, = collect_xml(FIXTURE, tmp, COMMIT)
|
|
rows = [json.loads(raw) for raw in Path(manifest['file']).read_text().splitlines()]
|
|
self.assertEqual(manifest['count'], 34)
|
|
self.assertEqual(manifest['missing_abstract_count'], 0)
|
|
self.assertEqual(manifest['excluded_materials'], [])
|
|
for paper, row in zip(self.volume.findall('paper'), rows):
|
|
self.assertEqual(row['external_id'], '1963.earlymt-1.'+paper.attrib['id'])
|
|
self.assertEqual(row['title'], text(paper.find('title')))
|
|
self.assertEqual(row['abstract'], text(paper.find('abstract')))
|
|
self.assertEqual(row['authors'], [{'given':text(a.find('first')), 'family':text(a.find('last'))} for a in paper.findall('author')])
|
|
self.assertEqual(row['metadata']['venue_codes'], ['earlymt'])
|
|
self.assertEqual(row['metadata']['publication_format'], 'published_abstract')
|
|
self.assertEqual(row['tags'], [{'namespace':'acl_anthology:publication_format','value':'published_abstract'}])
|
|
self.assertEqual((row['venue'], row['track'], row['status']), ('ACL', 'proceedings_unspecified', 'accepted'))
|
|
|
|
def test_native_ingestion_is_idempotent_and_preserves_abstracts_and_history(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root = Path(tmp)
|
|
manifest, = collect_xml(FIXTURE, root/'source', COMMIT)
|
|
db = root/'sample.sqlite'
|
|
ingest_file(db, manifest['file'])
|
|
ingest_file(db, manifest['file'])
|
|
with sqlite3.connect(db) as c:
|
|
for table in ['papers', 'source_records', 'submissions', 'decisions']:
|
|
self.assertEqual(c.execute('SELECT count(*) FROM '+table).fetchone()[0], 34)
|
|
self.assertEqual(c.execute("SELECT count(*) FROM papers WHERE abstract IS NOT NULL AND abstract!=''").fetchone()[0], 34)
|
|
self.assertEqual(c.execute("SELECT DISTINCT status FROM decisions").fetchall(), [('accepted',)])
|
|
self.assertEqual(c.execute("SELECT v.acronym,e.year,e.track FROM editions e JOIN venues v ON v.id=e.venue_id").fetchall(), [('ACL',1963,'proceedings_unspecified')])
|
|
self.assertIsNone(c.execute("SELECT 1 FROM venue_aliases WHERE alias='EarlyMT'").fetchone())
|
|
self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE t.namespace='acl_anthology:publication_format' AND t.value='published_abstract'").fetchone()[0], 34)
|