"""Read-only, evidence-preserving identity candidates for normalized source files. A candidate is never permission to import or merge. Absence of an exact match is never proof of a new work. Run again against a fresh snapshot before curation. """ import argparse from collections import Counter import gzip import hashlib import json from pathlib import Path from paper_atlas.db.store import author_keys, normalized from paper_atlas.db.submissions import timestamp from paper_atlas.postgres.client import ReadConnection STATUSES = {'accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown'} def digest(data): return hashlib.sha256(data).hexdigest() def authors_multiset(authors): # author_keys intentionally removes duplicates; identity review must retain them. return Counter(key for author in authors for key in author_keys([author])) def load_inputs(files): records, inputs, seen = [], [], set() for filename in files: path = Path(filename).resolve() manifest_path = path.with_suffix('.manifest.json') raw_manifest = manifest_path.read_bytes() manifest = json.loads(raw_manifest) hasher, count = hashlib.sha256(), 0 with path.open('rb') as stream: for line_number, line in enumerate(stream, 1): hasher.update(line) row = json.loads(line) for key in ('source', 'external_id', 'title', 'venue', 'track'): if not isinstance(row.get(key), str) or not row[key].strip(): raise ValueError(f'Invalid {key}: {path.name}:{line_number}') if type(row.get('year')) is not int or row.get('status') not in STATUSES: raise ValueError('Invalid year or explicit source status') for key in ('source', 'venue', 'year', 'track'): if row[key] != manifest.get(key): raise ValueError(f'Manifest scope mismatch: {key}') authors = row.get('authors') if not isinstance(authors, list) or any(not isinstance(a, (str, list, dict)) for a in authors): raise ValueError('Invalid authors') identifiers = row.get('identifiers', {}) if not isinstance(identifiers, dict) or any( not isinstance(k, str) or not k.strip() or not isinstance(v, str) or not v.strip() for k, v in identifiers.items()): raise ValueError('Identifiers must contain singular nonempty strings') key = (row['source'], row['external_id']) if key in seen: raise ValueError('Duplicate source/external_id across inputs') seen.add(key) record = {k: row[k] for k in ('source', 'external_id', 'title', 'authors', 'venue', 'year', 'track', 'status')} record.update(identifiers=identifiers, file=str(path), line=line_number, raw_line_sha256=digest(line), source_record_hits=[], identifier_hits=[], title_candidate_ids=[]) records.append(record) count += 1 if not count or count != manifest.get('count') or hasher.hexdigest() != manifest.get('file_sha256'): raise ValueError('Input count/hash mismatch or empty input') inputs.append({'file': str(path), 'file_sha256': hasher.hexdigest(), 'count': count, 'manifest_sha256': digest(raw_manifest), 'manifest': manifest}) if not records: raise ValueError('At least one nonempty input is required') return records, inputs def load_mm_program_inputs(report_path): """Read every MM observation without turning a program into import records. Reconcile the complete output against its frozen raw source before any DB connection. The synthetic unknown status is an audit placeholder only. """ from paper_atlas.collectors import mm2026 report_path = Path(report_path).absolute() raw_report = mm2026.safe_read(report_path, 256*1024) report = json.loads(raw_report) if (report.get('source') != 'mm_program' or report.get('records') != 2143 or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28 or report.get('source_body_sha256') != mm2026.BODY_SHA or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA or report.get('database_written') is not False or report.get('approved_for_import') != 0): raise ValueError('Unreviewed MM observation report') root = report_path.parent raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2) raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024) if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA: raise ValueError('MM raw evidence changed') expected_rows, _ = mm2026.parse_program(raw_source) expected = {r['paper_id']:r for r in expected_rows} records, inputs, seen, tracks = [], [], set(), set() for item in report['files']: track = item.get('track_literal') if (track not in mm2026.TRACK_COUNTS or track in tracks or item.get('format') != 'mm_program_observation_v1_not_ingestion_record' or item.get('approved_for_import') != 0): raise ValueError('Duplicate or unreviewed MM track format') tracks.add(track) # Stored absolute paths are provenance, not instructions to read elsewhere. path = root/Path(item['file']).name if path.suffix != '.jsonl': raise ValueError('Invalid MM observation filename') raw = mm2026.safe_read(path, 16*1024**2) lines = raw.splitlines(keepends=True) if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]: raise ValueError('MM observation file count/hash mismatch') for number,line in enumerate(lines, 1): row = json.loads(line) pid = row.get('paper_id') if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track: raise ValueError('MM observation differs from frozen source or repeats an ID') seen.add(pid) records.append({'source':'mm_program','external_id':'2026/'+pid, 'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track, 'status':'unknown','identifiers':{},'file':str(path),'line':number, 'raw_line_sha256':digest(line),'source_record_hits':[], 'identifier_hits':[],'title_candidate_ids':[], 'input_format':'mm_program_observation_v1_not_ingestion_record', 'program_id':pid,'program_contexts':row['contexts'], 'source_observed_status':row['observed_status'], 'missing_source_fields':row['missing_fields'], 'publication_identity_verified':False,'final_decision_verified':False, 'contribution_type_verified':False}) inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines), 'manifest_sha256':digest(raw_report),'manifest':{ 'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown', 'format':'mm_program_observation_v1_not_ingestion_record', 'scope':'Read-only lookup input only; unknown is not a source decision.', 'source_report':str(report_path),'source_body_sha256':digest(raw_source)}}) if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys(): raise ValueError('Incomplete MM observation input; all28tracks required') return records, inputs def _lookup(connection, values, columns, lateral): """Bound scalar VALUES allow the existing text codec to encode each key.""" width = len(columns) for start in range(0, len(values), 100): batch = values[start:start + 100] sql = 'WITH keys(' + ','.join(columns) + ') AS (VALUES ' + ','.join( '(' + ','.join(['?'] * width) + ')' for _ in batch) + ') SELECT keys.n,hit.* FROM keys JOIN LATERAL (' + lateral + ') hit ON true' for row in connection.execute(sql, tuple(value for item in batch for value in item)): yield dict(row) def find_candidates(connection, records): source_keys = [(i, row['source'], row['external_id']) for i, row in enumerate(records)] for hit in _lookup(connection, source_keys, ('n', 'source', 'external_id'), ''' SELECT r.id,r.paper_id FROM sources s JOIN source_records r ON r.source_id=s.id WHERE s.name=keys.source AND r.external_id=keys.external_id'''): records[hit.pop('n')]['source_record_hits'].append(hit) identifiers = [] for i, row in enumerate(records): pairs = {(row['source'], row['external_id']), *row['identifiers'].items()} identifiers.extend((i, scheme, value) for scheme, value in sorted(pairs)) for hit in _lookup(connection, identifiers, ('n', 'scheme', 'value'), ''' SELECT scheme,value,paper_id FROM identifiers WHERE scheme=keys.scheme AND value=keys.value'''): records[hit.pop('n')]['identifier_hits'].append(hit) title_keys = [(i, normalized(row['title'])) for i, row in enumerate(records)] for hit in _lookup(connection, title_keys, ('n', 'title_key'), ''' SELECT id AS paper_id FROM papers WHERE title_key=keys.title_key ORDER BY id LIMIT 21'''): records[hit['n']]['title_candidate_ids'].append(hit['paper_id']) def entity_packet(connection, paper_id): paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone() if paper is None: raise RuntimeError('Candidate disappeared from read-only snapshot') sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source FROM source_records r JOIN sources s ON s.id=r.source_id WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))] attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))] packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute( 'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))], 'source_records': sources[:100], 'source_records_truncated': len(sources) > 100, 'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200} for attempt in packet['attempts']: observations = [dict(r) for r in connection.execute( 'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))] attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200 provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=? ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))] attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100 for decision in attempt['decisions']: provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=? ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))] decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100 return packet def classify(record, entities): anchors = {r['paper_id'] for r in record['source_record_hits'] + record['identifier_hits']} titles = record['title_candidate_ids'] flags = [] own_authors = authors_multiset(record['authors']) if not own_authors: flags.append('missing_source_authors') if any(n > 1 for n in own_authors.values()): flags.append('duplicate_normalized_author_names') if len(normalized(record['title'])) < 25: flags.append('short_title') comparisons = [] for pid in sorted(anchors | set(titles)): entity = entities[pid] other = authors_multiset(entity['authors']) comparisons.append({'paper_id': pid, 'normalized_title_equal': normalized(record['title']) == normalized(entity['title']), 'nonempty_author_multiset_equal': bool(own_authors) and own_authors == other, 'author_name_overlap': sorted(own_authors.keys() & other.keys())}) if len(anchors) > 1: kind = 'conflicting_existing_bindings_hold' elif record['source_record_hits']: kind = 'existing_source_record_reconcile_no_replay' elif anchors: kind = 'existing_identifier_binding_requires_review' elif len(titles) >= 21: kind = 'title_candidate_limit_reached_hold' elif not titles: kind = 'no_exact_match_not_proof_of_novelty' elif len(titles) > 1: kind = 'multiple_exact_title_entities_hold' elif comparisons[0]['nonempty_author_multiset_equal']: kind = 'one_exact_title_author_multiset_requires_review' elif comparisons[0]['author_name_overlap']: kind = 'partial_author_overlap_requires_review' else: kind = 'no_author_agreement_hold' return {'kind': kind, 'flags': flags, 'comparisons': comparisons, 'title_candidates_truncated': len(titles) >= 21, 'decision': 'not_approved_for_import'} def _atomic_json(path, value): temporary = path.with_suffix(path.suffix + '.part') with temporary.open('w', encoding='utf-8') as stream: json.dump(value, stream, ensure_ascii=False, indent=2) stream.write('\n') temporary.replace(path) def audit(files, out, *, pg_service, pg_schema, mm_program_report=None): # Reject input corruption before opening a database connection or reserving output. if bool(files) == bool(mm_program_report): raise ValueError('Choose source files or one MM program report') records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files) out = Path(out) out.mkdir(parents=True, exist_ok=False) (out / 'audit.lock').write_text(timestamp() + '\n') connection = None try: connection = ReadConnection(pg_service, pg_schema) connection.execute("SET LOCAL statement_timeout='60s'") if connection.execute('SHOW transaction_read_only').fetchone()[0] != 'on': raise RuntimeError('A read-only transaction is required') snapshot = dict(connection.execute('''SELECT current_database() AS database,current_schema() AS schema, transaction_timestamp()::text AS transaction_at,pg_current_snapshot()::text AS snapshot''').fetchone()) find_candidates(connection, records) ids = sorted({hit['paper_id'] for row in records for hit in row['source_record_hits'] + row['identifier_hits']} | {pid for row in records for pid in row['title_candidate_ids']}) entities, index = {}, {} for number, pid in enumerate(ids, 1): packet = entity_packet(connection, pid) relative = Path('entities') / str(pid // 10000) / f'{pid}.json.gz' target = out / relative target.parent.mkdir(parents=True, exist_ok=True) raw = gzip.compress(json.dumps(packet, ensure_ascii=False).encode(), mtime=0) target.write_bytes(raw) index[str(pid)] = {'file': relative.as_posix(), 'sha256': digest(raw)} entities[pid] = {'title': packet['paper']['title'], 'authors': json.loads(packet['paper']['authors_json'])} if number % 250 == 0 or number == len(ids): print(json.dumps({'phase': 'entity_evidence', 'completed': number, 'total': len(ids)}), flush=True) kinds = Counter() for row in records: row.update(classify(row, entities)) kinds[row['kind']] += 1 report = {'status': 'completed_read_only_candidates_not_import_approval', 'started_snapshot': snapshot, 'completed_at': timestamp(), 'transaction_read_only': True, 'database_written': False, 'approved_for_import': 0, 'records': len(records), 'kind_counts': dict(kinds), 'inputs': inputs, 'entities': index, 'entries': records, 'limitations': ['Exact keys only; no match does not establish a new work.', 'Name agreement does not prove person or scientific-work identity.', 'Bound identifiers can be incorrect; review titles, authors and original evidence.', 'Source/history limits are explicit in compressed entity packets.', 'Program observations and unknown audit status do not establish contribution type, publication or a final decision.', 'Not a final whole-database deduplication or coverage audit.']} _atomic_json(out / 'postgresql-preflight.json', report) return report except Exception as error: _atomic_json(out / 'failed.json', {'status': 'failed_no_import_approval', 'error_type': type(error).__name__, 'database_written': False}) raise finally: if connection is not None: connection.close() def main(argv=None, *, prog=None): parser = argparse.ArgumentParser(prog=prog, description=__doc__) inputs = parser.add_mutually_exclusive_group(required=True) inputs.add_argument('--files', nargs='+', type=Path) inputs.add_argument('--mm-program-report', type=Path, help='Read all frozen MM program observations directly; never generate an import queue') parser.add_argument('--pg-service', required=True) parser.add_argument('--pg-schema', required=True) parser.add_argument('--out', required=True, type=Path) args = parser.parse_args(argv) report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema, mm_program_report=args.mm_program_report) print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))