253 lines
14 KiB
Python
253 lines
14 KiB
Python
"""Read-only, evidence-preserving identity candidates for normalized source files.
|
|
|
|
A candidate is never permission to import or merge. Absence of an exact match is
|
|
never proof of a new work. Run again against a fresh snapshot before curation.
|
|
"""
|
|
import argparse
|
|
from collections import Counter
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from paper_atlas.db.store import author_keys, normalized
|
|
from paper_atlas.db.submissions import timestamp
|
|
from paper_atlas.postgres.client import ReadConnection
|
|
|
|
STATUSES = {'accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown'}
|
|
|
|
|
|
def digest(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def authors_multiset(authors):
|
|
# author_keys intentionally removes duplicates; identity review must retain them.
|
|
return Counter(key for author in authors for key in author_keys([author]))
|
|
|
|
|
|
def load_inputs(files):
|
|
records, inputs, seen = [], [], set()
|
|
for filename in files:
|
|
path = Path(filename).resolve()
|
|
manifest_path = path.with_suffix('.manifest.json')
|
|
raw_manifest = manifest_path.read_bytes()
|
|
manifest = json.loads(raw_manifest)
|
|
hasher, count = hashlib.sha256(), 0
|
|
with path.open('rb') as stream:
|
|
for line_number, line in enumerate(stream, 1):
|
|
hasher.update(line)
|
|
row = json.loads(line)
|
|
for key in ('source', 'external_id', 'title', 'venue', 'track'):
|
|
if not isinstance(row.get(key), str) or not row[key].strip():
|
|
raise ValueError(f'Invalid {key}: {path.name}:{line_number}')
|
|
if type(row.get('year')) is not int or row.get('status') not in STATUSES:
|
|
raise ValueError('Invalid year or explicit source status')
|
|
for key in ('source', 'venue', 'year', 'track'):
|
|
if row[key] != manifest.get(key):
|
|
raise ValueError(f'Manifest scope mismatch: {key}')
|
|
authors = row.get('authors')
|
|
if not isinstance(authors, list) or any(not isinstance(a, (str, list, dict)) for a in authors):
|
|
raise ValueError('Invalid authors')
|
|
identifiers = row.get('identifiers', {})
|
|
if not isinstance(identifiers, dict) or any(
|
|
not isinstance(k, str) or not k.strip() or not isinstance(v, str) or not v.strip()
|
|
for k, v in identifiers.items()):
|
|
raise ValueError('Identifiers must contain singular nonempty strings')
|
|
key = (row['source'], row['external_id'])
|
|
if key in seen:
|
|
raise ValueError('Duplicate source/external_id across inputs')
|
|
seen.add(key)
|
|
record = {k: row[k] for k in ('source', 'external_id', 'title', 'authors',
|
|
'venue', 'year', 'track', 'status')}
|
|
record.update(identifiers=identifiers, file=str(path), line=line_number,
|
|
raw_line_sha256=digest(line), source_record_hits=[],
|
|
identifier_hits=[], title_candidate_ids=[])
|
|
records.append(record)
|
|
count += 1
|
|
if not count or count != manifest.get('count') or hasher.hexdigest() != manifest.get('file_sha256'):
|
|
raise ValueError('Input count/hash mismatch or empty input')
|
|
inputs.append({'file': str(path), 'file_sha256': hasher.hexdigest(), 'count': count,
|
|
'manifest_sha256': digest(raw_manifest), 'manifest': manifest})
|
|
if not records:
|
|
raise ValueError('At least one nonempty input is required')
|
|
return records, inputs
|
|
|
|
|
|
def _lookup(connection, values, columns, lateral):
|
|
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
|
width = len(columns)
|
|
for start in range(0, len(values), 100):
|
|
batch = values[start:start + 100]
|
|
sql = 'WITH keys(' + ','.join(columns) + ') AS (VALUES ' + ','.join(
|
|
'(' + ','.join(['?'] * width) + ')' for _ in batch) + ') SELECT keys.n,hit.* FROM keys JOIN LATERAL (' + lateral + ') hit ON true'
|
|
for row in connection.execute(sql, tuple(value for item in batch for value in item)):
|
|
yield dict(row)
|
|
|
|
|
|
def find_candidates(connection, records):
|
|
source_keys = [(i, row['source'], row['external_id']) for i, row in enumerate(records)]
|
|
for hit in _lookup(connection, source_keys, ('n', 'source', 'external_id'), '''
|
|
SELECT r.id,r.paper_id FROM sources s JOIN source_records r ON r.source_id=s.id
|
|
WHERE s.name=keys.source AND r.external_id=keys.external_id'''):
|
|
records[hit.pop('n')]['source_record_hits'].append(hit)
|
|
identifiers = []
|
|
for i, row in enumerate(records):
|
|
pairs = {(row['source'], row['external_id']), *row['identifiers'].items()}
|
|
identifiers.extend((i, scheme, value) for scheme, value in sorted(pairs))
|
|
for hit in _lookup(connection, identifiers, ('n', 'scheme', 'value'), '''
|
|
SELECT scheme,value,paper_id FROM identifiers
|
|
WHERE scheme=keys.scheme AND value=keys.value'''):
|
|
records[hit.pop('n')]['identifier_hits'].append(hit)
|
|
title_keys = [(i, normalized(row['title'])) for i, row in enumerate(records)]
|
|
for hit in _lookup(connection, title_keys, ('n', 'title_key'), '''
|
|
SELECT id AS paper_id FROM papers WHERE title_key=keys.title_key ORDER BY id LIMIT 21'''):
|
|
records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
|
|
|
|
|
|
def entity_packet(connection, paper_id):
|
|
paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone()
|
|
if paper is None:
|
|
raise RuntimeError('Candidate disappeared from read-only snapshot')
|
|
sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source
|
|
FROM source_records r JOIN sources s ON s.id=r.source_id
|
|
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))]
|
|
attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
|
|
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
|
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))]
|
|
packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute(
|
|
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))],
|
|
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
|
|
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
|
|
for attempt in packet['attempts']:
|
|
observations = [dict(r) for r in connection.execute(
|
|
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))]
|
|
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
|
|
provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source
|
|
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
|
|
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
|
|
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))]
|
|
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
|
|
for decision in attempt['decisions']:
|
|
provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source
|
|
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
|
|
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
|
|
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))]
|
|
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
|
|
return packet
|
|
|
|
|
|
def classify(record, entities):
|
|
anchors = {r['paper_id'] for r in record['source_record_hits'] + record['identifier_hits']}
|
|
titles = record['title_candidate_ids']
|
|
flags = []
|
|
own_authors = authors_multiset(record['authors'])
|
|
if not own_authors:
|
|
flags.append('missing_source_authors')
|
|
if any(n > 1 for n in own_authors.values()):
|
|
flags.append('duplicate_normalized_author_names')
|
|
if len(normalized(record['title'])) < 25:
|
|
flags.append('short_title')
|
|
comparisons = []
|
|
for pid in sorted(anchors | set(titles)):
|
|
entity = entities[pid]
|
|
other = authors_multiset(entity['authors'])
|
|
comparisons.append({'paper_id': pid,
|
|
'normalized_title_equal': normalized(record['title']) == normalized(entity['title']),
|
|
'nonempty_author_multiset_equal': bool(own_authors) and own_authors == other,
|
|
'author_name_overlap': sorted(own_authors.keys() & other.keys())})
|
|
if len(anchors) > 1:
|
|
kind = 'conflicting_existing_bindings_hold'
|
|
elif record['source_record_hits']:
|
|
kind = 'existing_source_record_reconcile_no_replay'
|
|
elif anchors:
|
|
kind = 'existing_identifier_binding_requires_review'
|
|
elif len(titles) >= 21:
|
|
kind = 'title_candidate_limit_reached_hold'
|
|
elif not titles:
|
|
kind = 'no_exact_match_not_proof_of_novelty'
|
|
elif len(titles) > 1:
|
|
kind = 'multiple_exact_title_entities_hold'
|
|
elif comparisons[0]['nonempty_author_multiset_equal']:
|
|
kind = 'one_exact_title_author_multiset_requires_review'
|
|
elif comparisons[0]['author_name_overlap']:
|
|
kind = 'partial_author_overlap_requires_review'
|
|
else:
|
|
kind = 'no_author_agreement_hold'
|
|
return {'kind': kind, 'flags': flags, 'comparisons': comparisons,
|
|
'title_candidates_truncated': len(titles) >= 21,
|
|
'decision': 'not_approved_for_import'}
|
|
|
|
|
|
def _atomic_json(path, value):
|
|
temporary = path.with_suffix(path.suffix + '.part')
|
|
with temporary.open('w', encoding='utf-8') as stream:
|
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
|
stream.write('\n')
|
|
temporary.replace(path)
|
|
|
|
|
|
def audit(files, out, *, pg_service, pg_schema):
|
|
# Reject input corruption before opening a database connection or reserving output.
|
|
records, inputs = load_inputs(files)
|
|
out = Path(out)
|
|
out.mkdir(parents=True, exist_ok=False)
|
|
(out / 'audit.lock').write_text(timestamp() + '\n')
|
|
connection = None
|
|
try:
|
|
connection = ReadConnection(pg_service, pg_schema)
|
|
connection.execute("SET LOCAL statement_timeout='60s'")
|
|
if connection.execute('SHOW transaction_read_only').fetchone()[0] != 'on':
|
|
raise RuntimeError('A read-only transaction is required')
|
|
snapshot = dict(connection.execute('''SELECT current_database() AS database,current_schema() AS schema,
|
|
transaction_timestamp()::text AS transaction_at,pg_current_snapshot()::text AS snapshot''').fetchone())
|
|
find_candidates(connection, records)
|
|
ids = sorted({hit['paper_id'] for row in records for hit in row['source_record_hits'] + row['identifier_hits']} |
|
|
{pid for row in records for pid in row['title_candidate_ids']})
|
|
entities, index = {}, {}
|
|
for number, pid in enumerate(ids, 1):
|
|
packet = entity_packet(connection, pid)
|
|
relative = Path('entities') / str(pid // 10000) / f'{pid}.json.gz'
|
|
target = out / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
raw = gzip.compress(json.dumps(packet, ensure_ascii=False).encode(), mtime=0)
|
|
target.write_bytes(raw)
|
|
index[str(pid)] = {'file': relative.as_posix(), 'sha256': digest(raw)}
|
|
entities[pid] = {'title': packet['paper']['title'],
|
|
'authors': json.loads(packet['paper']['authors_json'])}
|
|
if number % 250 == 0 or number == len(ids):
|
|
print(json.dumps({'phase': 'entity_evidence', 'completed': number, 'total': len(ids)}), flush=True)
|
|
kinds = Counter()
|
|
for row in records:
|
|
row.update(classify(row, entities))
|
|
kinds[row['kind']] += 1
|
|
report = {'status': 'completed_read_only_candidates_not_import_approval',
|
|
'started_snapshot': snapshot, 'completed_at': timestamp(), 'transaction_read_only': True,
|
|
'database_written': False, 'approved_for_import': 0, 'records': len(records),
|
|
'kind_counts': dict(kinds), 'inputs': inputs, 'entities': index, 'entries': records,
|
|
'limitations': ['Exact keys only; no match does not establish a new work.',
|
|
'Name agreement does not prove person or scientific-work identity.',
|
|
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
|
'Source/history limits are explicit in compressed entity packets.',
|
|
'Not a final whole-database deduplication or coverage audit.']}
|
|
_atomic_json(out / 'postgresql-preflight.json', report)
|
|
return report
|
|
except Exception as error:
|
|
_atomic_json(out / 'failed.json', {'status': 'failed_no_import_approval',
|
|
'error_type': type(error).__name__, 'database_written': False})
|
|
raise
|
|
finally:
|
|
if connection is not None:
|
|
connection.close()
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
|
parser.add_argument('--files', nargs='+', required=True, type=Path)
|
|
parser.add_argument('--pg-service', required=True)
|
|
parser.add_argument('--pg-schema', required=True)
|
|
parser.add_argument('--out', required=True, type=Path)
|
|
args = parser.parse_args(argv)
|
|
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
|
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|