Preserve edition-specific ACL and TINLAP 1978 source evidence
Metadata integrity checks / metadata (push) Successful in 2m6s
Metadata integrity checks / metadata (push) Successful in 2m6s
This commit is contained in:
Vendored
+273
@@ -0,0 +1,273 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<collection id="T78">
|
||||
<volume id="1" type="proceedings">
|
||||
<!-- https://aclanthology.org/T78-1/ -->
|
||||
<meta>
|
||||
<booktitle>Theoretical Issues in Natural Language <fixed-case>P</fixed-case>rocessing-2</booktitle>
|
||||
<editor id="david-l-waltz"><first>David L.</first><last>Waltz</last></editor>
|
||||
<venue>tinlap</venue>
|
||||
<year>1978</year>
|
||||
</meta>
|
||||
<frontmatter>
|
||||
<!-- https://aclanthology.org/T78-1000/ -->
|
||||
<pdf hash="fb6164ad"/>
|
||||
<bibkey>tinlap-1978-theoretical</bibkey>
|
||||
</frontmatter>
|
||||
<paper id="1">
|
||||
<!-- https://aclanthology.org/T78-1001/ -->
|
||||
<title>Testing The Psychological Reality of a Representational Model</title>
|
||||
<author><first>Dedre</first><last>Gentner</last></author>
|
||||
<pdf hash="f723a956"/>
|
||||
<bibkey>gentner-1978-testing-psychological</bibkey>
|
||||
</paper>
|
||||
<paper id="2">
|
||||
<!-- https://aclanthology.org/T78-1002/ -->
|
||||
<title>What Makes Something “Ad Hoc”</title>
|
||||
<author id="roger-c-schank"><first>Roger C.</first><last>Schank</last></author>
|
||||
<pdf hash="636853f9"/>
|
||||
<bibkey>schank-1978-makes-something</bibkey>
|
||||
</paper>
|
||||
<paper id="3">
|
||||
<!-- https://aclanthology.org/T78-1003/ -->
|
||||
<title>The Relation of Grammar to Cognition–a Synopsis</title>
|
||||
<author><first>Leonard</first><last>Talmy</last></author>
|
||||
<pdf hash="101c67dc"/>
|
||||
<bibkey>talmy-1978-relation-grammar</bibkey>
|
||||
</paper>
|
||||
<paper id="4">
|
||||
<!-- https://aclanthology.org/T78-1004/ -->
|
||||
<title>On primitives, prototypes, and other semantic anomalies</title>
|
||||
<author><first>Terry</first><last>Winograd</last></author>
|
||||
<pdf hash="f8e47023"/>
|
||||
<bibkey>winograd-1978-primitives-prototypes</bibkey>
|
||||
</paper>
|
||||
<paper id="5">
|
||||
<!-- https://aclanthology.org/T78-1005/ -->
|
||||
<title>Taxonomic Lattice Structures for Situation Recognition</title>
|
||||
<author id="william-a-woods"><first>William A.</first><last>Woods</last></author>
|
||||
<pdf hash="92957ddf"/>
|
||||
<bibkey>woods-1978-taxonomic-lattice</bibkey>
|
||||
</paper>
|
||||
<paper id="6">
|
||||
<!-- https://aclanthology.org/T78-1006/ -->
|
||||
<title>Description Formation and Discourse Model Synthesis</title>
|
||||
<author id="bonnie-webber"><first>Bonnie Lynn</first><last>Webber</last></author>
|
||||
<pdf hash="ffded1d2"/>
|
||||
<bibkey>webber-1978-description-formation</bibkey>
|
||||
</paper>
|
||||
<paper id="7">
|
||||
<!-- https://aclanthology.org/T78-1007/ -->
|
||||
<title>The Processing of Referring Expressions within a Semantic Network</title>
|
||||
<author><first>John R.</first><last>Anderson</last></author>
|
||||
<pdf hash="8bcc083a"/>
|
||||
<bibkey>anderson-1978-processing-referring</bibkey>
|
||||
</paper>
|
||||
<paper id="8">
|
||||
<!-- https://aclanthology.org/T78-1008/ -->
|
||||
<title>Reference Diaries</title>
|
||||
<author><first>Herbert H.</first><last>Clark</last></author>
|
||||
<author><first>Catherine</first><last>Marshall</last></author>
|
||||
<pdf hash="d06b7ea2"/>
|
||||
<bibkey>clark-marshall-1978-reference-diaries</bibkey>
|
||||
</paper>
|
||||
<paper id="9">
|
||||
<!-- https://aclanthology.org/T78-1009/ -->
|
||||
<title>Subsequent Reference: Syntactic and Rhetorical Constraints</title>
|
||||
<author id="david-d-mcdonald"><first>David D.</first><last>McDonald</last></author>
|
||||
<pdf hash="2d424559"/>
|
||||
<bibkey>mcdonald-1978-subsequent-reference</bibkey>
|
||||
</paper>
|
||||
<paper id="10">
|
||||
<!-- https://aclanthology.org/T78-1010/ -->
|
||||
<title>Some Psycholinguistic Constraints on the Construction and Interpretation of Definite Descriptions</title>
|
||||
<author><first>Andrew</first><last>Ortony</last></author>
|
||||
<pdf hash="69807727"/>
|
||||
<bibkey>ortony-1978-psycholinguistic-constraints</bibkey>
|
||||
</paper>
|
||||
<paper id="11">
|
||||
<!-- https://aclanthology.org/T78-1011/ -->
|
||||
<title>Bound Variables and Other Anaphors</title>
|
||||
<author id="barbara-h-partee"><first>Barbara H.</first><last>Partee</last></author>
|
||||
<pdf hash="e8e6a510"/>
|
||||
<bibkey>partee-1978-bound-variables</bibkey>
|
||||
</paper>
|
||||
<paper id="12">
|
||||
<!-- https://aclanthology.org/T78-1012/ -->
|
||||
<title>The Use of Focus as a Tool for Disambiguation of Definite Noun Phrases</title>
|
||||
<author id="candace-l-sidner"><first>Candace L.</first><last>Sidner</last></author>
|
||||
<pdf hash="7b96d2b7"/>
|
||||
<bibkey>sidner-1978-use-focus</bibkey>
|
||||
</paper>
|
||||
<paper id="13">
|
||||
<!-- https://aclanthology.org/T78-1013/ -->
|
||||
<title>Focusing in Dialog</title>
|
||||
<author id="barbara-j-grosz"><first>Barbara J.</first><last>Grosz</last></author>
|
||||
<pdf hash="8c8d4c2e"/>
|
||||
<bibkey>grosz-1978-focusing-dialog</bibkey>
|
||||
</paper>
|
||||
<paper id="14">
|
||||
<!-- https://aclanthology.org/T78-1014/ -->
|
||||
<title>Topic Levels</title>
|
||||
<author><first>Joseph E.</first><last>Grimes</last></author>
|
||||
<pdf hash="f485b2ba"/>
|
||||
<bibkey>grimes-1978-topic-levels</bibkey>
|
||||
</paper>
|
||||
<paper id="15">
|
||||
<!-- https://aclanthology.org/T78-1015/ -->
|
||||
<title>Toward a Rational Model of Discourse Comprehension</title>
|
||||
<author><first>Jerry L.</first><last>Morgan</last></author>
|
||||
<pdf hash="5e12a597"/>
|
||||
<bibkey>morgan-1978-toward-rational</bibkey>
|
||||
</paper>
|
||||
<paper id="16">
|
||||
<!-- https://aclanthology.org/T78-1016/ -->
|
||||
<title>Assent and Compliance in Children’s Language Comprehension</title>
|
||||
<author><first>David R.</first><last>Olson</last></author>
|
||||
<pdf hash="708bc4a0"/>
|
||||
<bibkey>olson-1978-assent-compliance</bibkey>
|
||||
</paper>
|
||||
<paper id="17">
|
||||
<!-- https://aclanthology.org/T78-1017/ -->
|
||||
<title>Speech Acts as a Basis for Understanding Dialogue Coherence</title>
|
||||
<author id="c-raymond-perrault"><first>C. Raymond</first><last>Perrault</last></author>
|
||||
<author id="james-allen"><first>James F.</first><last>Allen</last></author>
|
||||
<pdf hash="d111402b"/>
|
||||
<bibkey>perrault-allen-1978-speech</bibkey>
|
||||
</paper>
|
||||
<paper id="18">
|
||||
<!-- https://aclanthology.org/T78-1018/ -->
|
||||
<title>A Framework for Comparing Language Experiences (with particular emphasis on: The Effect of Audience on Discourse Models)</title>
|
||||
<author><first>Andee</first><last>Rubin</last></author>
|
||||
<pdf hash="5d9a6150"/>
|
||||
<bibkey>rubin-1978-framework-comparing</bibkey>
|
||||
</paper>
|
||||
<paper id="19">
|
||||
<!-- https://aclanthology.org/T78-1019/ -->
|
||||
<title>Intentlonallty and Human Conversations</title>
|
||||
<author id="jaime-g-carbonell"><first>Jaime G.</first><last>Carbonell Jr</last></author>
|
||||
<pdf hash="c50fc619"/>
|
||||
<bibkey>carbonell-jr-1978-intentlonallty-human</bibkey>
|
||||
</paper>
|
||||
<paper id="20">
|
||||
<!-- https://aclanthology.org/T78-1020/ -->
|
||||
<title>On the Interdependence of Language and Perception</title>
|
||||
<author id="david-l-waltz"><first>David L.</first><last>Waltz</last></author>
|
||||
<pdf hash="3c708191"/>
|
||||
<bibkey>waltz-1978-interdependence-language</bibkey>
|
||||
</paper>
|
||||
<paper id="21">
|
||||
<!-- https://aclanthology.org/T78-1021/ -->
|
||||
<title>The Problem of Naming Shapes: Vision-Language Interface</title>
|
||||
<author id="ruzena-bajcsy"><first>R.</first><last>Bajcsy</last></author>
|
||||
<author id="aravind-joshi"><first>A. K.</first><last>Joshi</last></author>
|
||||
<pdf hash="50331b4e"/>
|
||||
<bibkey>bajcsy-joshi-1978-problem-naming</bibkey>
|
||||
</paper>
|
||||
<paper id="22">
|
||||
<!-- https://aclanthology.org/T78-1022/ -->
|
||||
<title>An Argument about the Composition of Conceptual Structure</title>
|
||||
<author><first>Ray</first><last>Jackendoff</last></author>
|
||||
<pdf hash="57d001f8"/>
|
||||
<bibkey>jackendoff-1978-argument-composition</bibkey>
|
||||
</paper>
|
||||
<paper id="23">
|
||||
<!-- https://aclanthology.org/T78-1023/ -->
|
||||
<title>On the Ontological Status of Visual Mental Images</title>
|
||||
<author><first>Stephen Michael</first><last>Kosslyn</last></author>
|
||||
<pdf hash="fd06e265"/>
|
||||
<bibkey>kosslyn-1978-ontological-status</bibkey>
|
||||
</paper>
|
||||
<paper id="24">
|
||||
<!-- https://aclanthology.org/T78-1024/ -->
|
||||
<title>What has language to do with perception? Some speculations on the <i>Lingua Mentis</i></title>
|
||||
<author><first>Zenon W.</first><last>Pylyshyn</last></author>
|
||||
<pdf hash="a2f3a044"/>
|
||||
<bibkey>pylyshyn-1978-language-perception</bibkey>
|
||||
</paper>
|
||||
<paper id="25">
|
||||
<!-- https://aclanthology.org/T78-1025/ -->
|
||||
<title>Semantic Primitives in Language and Vision</title>
|
||||
<author id="yorick-wilks"><first>Yorick</first><last>Wilks</last></author>
|
||||
<pdf hash="b4e13896"/>
|
||||
<bibkey>wilks-1978-semantic-primitives</bibkey>
|
||||
</paper>
|
||||
<paper id="26">
|
||||
<!-- https://aclanthology.org/T78-1026/ -->
|
||||
<title>A Note on Partial Match of Descriptions. Can One Simultaneously Question (Retrieve) and Inform (Update)?</title>
|
||||
<author id="aravind-joshi"><first>Aravind K.</first><last>Joshi</last></author>
|
||||
<pdf hash="74969243"/>
|
||||
<bibkey>joshi-1978-note-partial</bibkey>
|
||||
</paper>
|
||||
<paper id="27">
|
||||
<!-- https://aclanthology.org/T78-1027/ -->
|
||||
<title>With a Spoon in Hand This Must Be the Eating Frame</title>
|
||||
<author><first>Eugene</first><last>Charniak</last></author>
|
||||
<pdf hash="f1be68bb"/>
|
||||
<bibkey>charniak-1978-spoon-hand</bibkey>
|
||||
</paper>
|
||||
<paper id="28">
|
||||
<!-- https://aclanthology.org/T78-1028/ -->
|
||||
<title>Fragments of a Theory of Human Plausible Reasoning</title>
|
||||
<author><first>Allan</first><last>Collins</last></author>
|
||||
<pdf hash="67b4bca2"/>
|
||||
<bibkey>collins-1978-fragments-theory</bibkey>
|
||||
</paper>
|
||||
<paper id="29">
|
||||
<!-- https://aclanthology.org/T78-1029/ -->
|
||||
<title>Indirect Responses to Loaded Questions</title>
|
||||
<author><first>S. Jerrold</first><last>Kaplan</last></author>
|
||||
<pdf hash="135037f9"/>
|
||||
<bibkey>kaplan-1978-indirect-responses</bibkey>
|
||||
</paper>
|
||||
<paper id="30">
|
||||
<!-- https://aclanthology.org/T78-1030/ -->
|
||||
<title>On Reasoning by Default</title>
|
||||
<author><first>Raymond</first><last>Reiter</last></author>
|
||||
<pdf hash="0de53d33"/>
|
||||
<bibkey>reiter-1978-reasoning-default</bibkey>
|
||||
</paper>
|
||||
<paper id="31">
|
||||
<!-- https://aclanthology.org/T78-1031/ -->
|
||||
<title>Path-Based and Node-Based Inference in Semantic Networks</title>
|
||||
<author id="stuart-c-shapiro"><first>Stuart C.</first><last>Shapiro</last></author>
|
||||
<pdf hash="2ddc5535"/>
|
||||
<bibkey>shapiro-1978-path-based</bibkey>
|
||||
</paper>
|
||||
<paper id="32">
|
||||
<!-- https://aclanthology.org/T78-1032/ -->
|
||||
<title>The Representation of Derivable Information in Memory: When What Might Have Been Left Unsaid Is Said</title>
|
||||
<author><first>Rand J.</first><last>Spiro</last></author>
|
||||
<author><first>Joseph</first><last>Esposito</last></author>
|
||||
<author><first>Richard J.</first><last>Vondruska</last></author>
|
||||
<pdf hash="7758fb45"/>
|
||||
<bibkey>spiro-etal-1978-representation-derivable</bibkey>
|
||||
</paper>
|
||||
<paper id="33">
|
||||
<!-- https://aclanthology.org/T78-1033/ -->
|
||||
<title>A Heuristic for Paradigms</title>
|
||||
<author><first>Joseph E.</first><last>Grimes</last></author>
|
||||
<pdf hash="53b42c16"/>
|
||||
<bibkey>grimes-1978-heuristic-paradigms</bibkey>
|
||||
</paper>
|
||||
<paper id="34">
|
||||
<!-- https://aclanthology.org/T78-1034/ -->
|
||||
<title>A Computational Account of Some Constraints on Language</title>
|
||||
<author id="mitch-marcus"><first>Mitchell</first><last>Marcus</last></author>
|
||||
<pdf hash="c9724c43"/>
|
||||
<bibkey>marcus-1978-computational-account</bibkey>
|
||||
</paper>
|
||||
<paper id="35">
|
||||
<!-- https://aclanthology.org/T78-1035/ -->
|
||||
<title>Remarks on Processing, Constraints, and the Lexicon</title>
|
||||
<author id="thomas-wasow"><first>Thomas</first><last>Wasow</last></author>
|
||||
<pdf hash="8beed275"/>
|
||||
<bibkey>wasow-1978-remarks-processing</bibkey>
|
||||
</paper>
|
||||
<paper id="36">
|
||||
<!-- https://aclanthology.org/T78-1036/ -->
|
||||
<title>List of questions suggested for consideration in each session</title>
|
||||
<pdf hash="1950b908"/>
|
||||
<bibkey>nn-1978-list-questions</bibkey>
|
||||
</paper>
|
||||
</volume>
|
||||
</collection>
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Pinned T78 evidence, edition-local aliases and an explicitly identified material."""
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from paper_atlas.collectors.acl import collect_xml, volume_scope
|
||||
from paper_atlas.collectors.acl_historical import event_scope, material_reason
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
|
||||
FIXTURE = Path(__file__).parent/'fixtures/acl-t78.xml'
|
||||
COMMIT = 'a7be623d01c48c61405dec31c871121cdbed5a01'
|
||||
|
||||
|
||||
class AclHistoricalTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.root = ET.parse(FIXTURE).getroot()
|
||||
self.volume = self.root.find('volume')
|
||||
self.meta = self.volume.find('meta')
|
||||
|
||||
def test_frozen_volume_matches_official_snapshot(self):
|
||||
self.assertEqual(hashlib.sha256(FIXTURE.read_bytes()).hexdigest(), 'f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84')
|
||||
self.assertEqual(len(self.volume.findall('paper')), 36)
|
||||
self.assertEqual(volume_scope(self.meta, '1', 'T78'), ('ACL+TINLAP', ['ACL', 'TINLAP'], 'proceedings_unspecified'))
|
||||
|
||||
def test_unknown_year_collection_or_volume_is_not_globally_aliased(self):
|
||||
for collection, volume in [('T75', '1'), ('T87', '1'), ('T78', '2'), ('C65', '1')]:
|
||||
with self.subTest(collection=collection, volume=volume):
|
||||
self.assertIsNone(event_scope(self.meta, volume, collection))
|
||||
self.assertIsNone(volume_scope(self.meta, volume, collection))
|
||||
self.assertIsNone(volume_scope(self.meta, '1'))
|
||||
|
||||
def test_known_volume_scope_rejects_critical_metadata_drift(self):
|
||||
for field, replacement in [('venue', 'acl'), ('year', '1979'), ('booktitle', 'Another TINLAP'), ('editor/first', 'Another')]:
|
||||
changed = copy.deepcopy(self.meta)
|
||||
changed.find(field).text = replacement
|
||||
with self.subTest(field=field), self.assertRaises(ValueError):
|
||||
event_scope(changed, '1', 'T78')
|
||||
changed = copy.deepcopy(self.meta)
|
||||
ET.SubElement(changed, 'year').text = '1978'
|
||||
with self.assertRaises(ValueError):
|
||||
event_scope(changed, '1', 'T78')
|
||||
|
||||
def test_evidence_retains_original_series_dates_and_role_boundary(self):
|
||||
scope = event_scope(self.meta, '1', 'T78')
|
||||
self.assertEqual(scope['original_venue'], 'TINLAP')
|
||||
self.assertEqual(scope['event_dates'], ['1978-07-25', '1978-07-27'])
|
||||
self.assertEqual([e['pdf_pages'] for e in scope['evidence']], [[9, 10], [3]])
|
||||
self.assertIn('not identified', scope['track_scope_note'])
|
||||
self.assertIn('pending', scope['evidence'][0]['scope'])
|
||||
|
||||
def test_only_exact_session_questions_material_is_excluded(self):
|
||||
paper = copy.deepcopy(self.volume.findall('paper')[-1])
|
||||
self.assertIn('Session question list', material_reason('T78', '1', paper))
|
||||
self.assertIsNone(material_reason('C65', '1', paper))
|
||||
self.assertIsNone(material_reason('T78', '2', paper))
|
||||
paper.set('id', '37')
|
||||
self.assertIsNone(material_reason('T78', '1', paper))
|
||||
paper.set('id', '36')
|
||||
ET.SubElement(paper, 'author')
|
||||
with self.assertRaises(ValueError):
|
||||
material_reason('T78', '1', paper)
|
||||
|
||||
def test_all_authored_source_fields_and_material_receipt_survive_collection(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
manifest, = collect_xml(FIXTURE, temporary, COMMIT)
|
||||
rows = [json.loads(raw) for raw in Path(manifest['file']).read_text().splitlines()]
|
||||
self.assertEqual(manifest['count'], 35)
|
||||
self.assertEqual(manifest['xml_paper_elements'], 36)
|
||||
self.assertEqual(manifest['missing_abstract_count'], 35)
|
||||
self.assertEqual(manifest['excluded_materials'][0]['external_id'], 'T78-1036')
|
||||
for number, row in enumerate(rows, 1):
|
||||
paper = self.volume.findall('paper')[number-1]
|
||||
self.assertEqual(row['external_id'], f'T78-1{number:03d}')
|
||||
self.assertEqual(row['title'], ' '.join(''.join(paper.find('title').itertext()).split()))
|
||||
self.assertEqual(row['metadata']['venue_codes'], ['tinlap'])
|
||||
self.assertEqual(row['metadata']['historical_event_scope']['original_venue'], 'TINLAP')
|
||||
self.assertEqual(row['metadata']['metadata_commit'], COMMIT)
|
||||
self.assertEqual((row['status'], row['track'], row['abstract']), ('accepted', 'proceedings_unspecified', None))
|
||||
|
||||
def test_old_coling_scope_is_unchanged_by_calendar_replacement_evidence(self):
|
||||
meta = ET.fromstring('<meta><venue>coling</venue><year>1965</year><booktitle>COLING 1965</booktitle></meta>')
|
||||
self.assertEqual(volume_scope(meta, '1', 'C65'), ('COLING', ['COLING'], 'main'))
|
||||
self.assertIsNone(event_scope(meta, '1', 'C65'))
|
||||
|
||||
def test_native_ingestion_does_not_alias_other_tinlap_years_or_infer_negative_status(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
manifest, = collect_xml(FIXTURE, root/'source', COMMIT)
|
||||
db = root/'sample.sqlite'
|
||||
store = Store(db)
|
||||
pid, rid = store.record('test_control', 'tinlap1975', 'https://example.org/tinlap1975',
|
||||
'An earlier TINLAP contribution', None, ['Jane Doe'])
|
||||
store.decision(pid, rid, 'TINLAP', 1975, 'main', 'tinlap1975', 'https://example.org/tinlap1975', 'accepted', 'test control')
|
||||
store.conn.commit()
|
||||
store.conn.close()
|
||||
ingest_file(db, manifest['file'])
|
||||
ingest_file(db, manifest['file'])
|
||||
with sqlite3.connect(db) as c:
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 36)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 36)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0], 36)
|
||||
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status!='accepted'").fetchone()[0], 0)
|
||||
self.assertIsNone(c.execute("SELECT 1 FROM source_records WHERE external_id='T78-1036'").fetchone())
|
||||
aliases = c.execute('SELECT v.acronym,a.alias FROM venue_aliases a JOIN venues v ON v.id=a.venue_id ORDER BY a.alias').fetchall()
|
||||
self.assertEqual(aliases, [('ACL+TINLAP', 'ACL'), ('ACL+TINLAP', 'TINLAP')])
|
||||
metadata = json.loads(c.execute("SELECT metadata_json FROM source_records WHERE external_id='T78-1001'").fetchone()[0])
|
||||
self.assertEqual(metadata['historical_event_scope']['event_year'], 1978)
|
||||
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
|
||||
Reference in New Issue
Block a user