diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 9980a17..3a210b0 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -96,6 +96,7 @@ jobs: subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True) smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()" diff --git a/README.md b/README.md index 0d700b3..b4c42f2 100644 --- a/README.md +++ b/README.md @@ -116,3 +116,5 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目 整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。 已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。 + +入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。 diff --git a/docs/source-identity-audit.md b/docs/source-identity-audit.md new file mode 100644 index 0000000..8f41ad0 --- /dev/null +++ b/docs/source-identity-audit.md @@ -0,0 +1,21 @@ +# 来源身份只读预检 + +`audit source-identities` 在 PostgreSQL 的同一 `REPEATABLE READ READ ONLY` 快照中,检查规范化 JSONL 的来源绑定、稳定标识及精确标题候选,并导出候选论文的原始来源和投稿决定历史。用于入库前评审,不执行导入、合并或标识修改。 + +```sh +PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas audit source-identities \ + --pg-service foundry-research --pg-schema research \ + --files /data/paper-index/collected/example/main.jsonl \ + /data/paper-index/collected/example/position.jsonl \ + --out /data/paper-index/collected/example-identity-preflight-v1 +``` + +每个 JSONL 必须有同名 `.manifest.json`,包含 `source`、`venue`、`year`、`track`、`count` 和 `file_sha256`。逐行校验届次、显式决定状态、标识类型和跨文件来源主键唯一性;文件哈希和数量必须相符。只读取调用者指定的文件,不跟随 manifest 中的绝对 `file` 路径;原始行哈希包含换行符。原始 JSONL 和 manifest 应与报告一起保留。 + +输出目录必须不存在,防止复用旧报告或覆盖证据。失败会保留 `failed.json` 及已完成分片,不能把部分输出当成功;排查后使用新的输出目录重新核查。成功报告 `postgresql-preflight.json` 最后原子写入,记录输入哈希、数据库快照和每条候选。实体详情存为 `entities/<分片>/.json.gz`,报告中列出压缩文件的 SHA-256;包含完整 paper、标识和限量原始来源、投稿、决定及出处。每篇最多保留 100 条来源、200 次投稿,每次投稿最多 200 条决定,每组决定或投稿出处最多 100 条,所有截断均显式标记。精确标题最多取 21 个候选;达到上限即标记,不能据此推算总数。这里没有计算全表精确数量。 + +候选分类用于安排审核:既有来源记录需要核对并避免重放,多个绑定冲突必须暂缓;单个 DOI、arXiv 或 OpenReview 绑定也可能错误,必须核对原文、标题、作者和历史。作者比较保留完整规范化姓名的重复次数,但姓名相同不能证明是同一个人。未找到精确标题或标识不能证明是新作品,仍需近似标题、版本和跨会历史核查。 + +所有条目固定为 `not_approved_for_import`,批准数量为 0。不要直接把此报告当成 curation 计划或导入队列。后续获批导入前应重新检查实时来源绑定;来源摘要缺失不得覆盖已有摘要,旧拒稿与新录用应作为不同投稿经历保留。本命令不能代替所有导入结束后的全库去重、覆盖和约束验收。 + +构建与测试仅在已确认的 `ces-control:host` 远程 runner 执行。集成测试使用已有 PostgreSQL 服务做真实只读候选查询,不往生产库添加测试记录。 diff --git a/paper_atlas/audits/source_identity.py b/paper_atlas/audits/source_identity.py new file mode 100644 index 0000000..24be057 --- /dev/null +++ b/paper_atlas/audits/source_identity.py @@ -0,0 +1,252 @@ +"""Read-only, evidence-preserving identity candidates for normalized source files. + +A candidate is never permission to import or merge. Absence of an exact match is +never proof of a new work. Run again against a fresh snapshot before curation. +""" +import argparse +from collections import Counter +import gzip +import hashlib +import json +from pathlib import Path + +from paper_atlas.db.store import author_keys, normalized +from paper_atlas.db.submissions import timestamp +from paper_atlas.postgres.client import ReadConnection + +STATUSES = {'accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown'} + + +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def authors_multiset(authors): + # author_keys intentionally removes duplicates; identity review must retain them. + return Counter(key for author in authors for key in author_keys([author])) + + +def load_inputs(files): + records, inputs, seen = [], [], set() + for filename in files: + path = Path(filename).resolve() + manifest_path = path.with_suffix('.manifest.json') + raw_manifest = manifest_path.read_bytes() + manifest = json.loads(raw_manifest) + hasher, count = hashlib.sha256(), 0 + with path.open('rb') as stream: + for line_number, line in enumerate(stream, 1): + hasher.update(line) + row = json.loads(line) + for key in ('source', 'external_id', 'title', 'venue', 'track'): + if not isinstance(row.get(key), str) or not row[key].strip(): + raise ValueError(f'Invalid {key}: {path.name}:{line_number}') + if type(row.get('year')) is not int or row.get('status') not in STATUSES: + raise ValueError('Invalid year or explicit source status') + for key in ('source', 'venue', 'year', 'track'): + if row[key] != manifest.get(key): + raise ValueError(f'Manifest scope mismatch: {key}') + authors = row.get('authors') + if not isinstance(authors, list) or any(not isinstance(a, (str, list, dict)) for a in authors): + raise ValueError('Invalid authors') + identifiers = row.get('identifiers', {}) + if not isinstance(identifiers, dict) or any( + not isinstance(k, str) or not k.strip() or not isinstance(v, str) or not v.strip() + for k, v in identifiers.items()): + raise ValueError('Identifiers must contain singular nonempty strings') + key = (row['source'], row['external_id']) + if key in seen: + raise ValueError('Duplicate source/external_id across inputs') + seen.add(key) + record = {k: row[k] for k in ('source', 'external_id', 'title', 'authors', + 'venue', 'year', 'track', 'status')} + record.update(identifiers=identifiers, file=str(path), line=line_number, + raw_line_sha256=digest(line), source_record_hits=[], + identifier_hits=[], title_candidate_ids=[]) + records.append(record) + count += 1 + if not count or count != manifest.get('count') or hasher.hexdigest() != manifest.get('file_sha256'): + raise ValueError('Input count/hash mismatch or empty input') + inputs.append({'file': str(path), 'file_sha256': hasher.hexdigest(), 'count': count, + 'manifest_sha256': digest(raw_manifest), 'manifest': manifest}) + if not records: + raise ValueError('At least one nonempty input is required') + return records, inputs + + +def _lookup(connection, values, columns, lateral): + """Bound scalar VALUES allow the existing text codec to encode each key.""" + width = len(columns) + for start in range(0, len(values), 100): + batch = values[start:start + 100] + sql = 'WITH keys(' + ','.join(columns) + ') AS (VALUES ' + ','.join( + '(' + ','.join(['?'] * width) + ')' for _ in batch) + ') SELECT keys.n,hit.* FROM keys JOIN LATERAL (' + lateral + ') hit ON true' + for row in connection.execute(sql, tuple(value for item in batch for value in item)): + yield dict(row) + + +def find_candidates(connection, records): + source_keys = [(i, row['source'], row['external_id']) for i, row in enumerate(records)] + for hit in _lookup(connection, source_keys, ('n', 'source', 'external_id'), ''' + SELECT r.id,r.paper_id FROM sources s JOIN source_records r ON r.source_id=s.id + WHERE s.name=keys.source AND r.external_id=keys.external_id'''): + records[hit.pop('n')]['source_record_hits'].append(hit) + identifiers = [] + for i, row in enumerate(records): + pairs = {(row['source'], row['external_id']), *row['identifiers'].items()} + identifiers.extend((i, scheme, value) for scheme, value in sorted(pairs)) + for hit in _lookup(connection, identifiers, ('n', 'scheme', 'value'), ''' + SELECT scheme,value,paper_id FROM identifiers + WHERE scheme=keys.scheme AND value=keys.value'''): + records[hit.pop('n')]['identifier_hits'].append(hit) + title_keys = [(i, normalized(row['title'])) for i, row in enumerate(records)] + for hit in _lookup(connection, title_keys, ('n', 'title_key'), ''' + SELECT id AS paper_id FROM papers WHERE title_key=keys.title_key ORDER BY id LIMIT 21'''): + records[hit['n']]['title_candidate_ids'].append(hit['paper_id']) + + +def entity_packet(connection, paper_id): + paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone() + if paper is None: + raise RuntimeError('Candidate disappeared from read-only snapshot') + sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source + FROM source_records r JOIN sources s ON s.id=r.source_id + WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))] + attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key + FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id + WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))] + packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute( + 'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))], + 'source_records': sources[:100], 'source_records_truncated': len(sources) > 100, + 'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200} + for attempt in packet['attempts']: + observations = [dict(r) for r in connection.execute( + 'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))] + attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200 + provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source + FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id + JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=? + ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))] + attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100 + for decision in attempt['decisions']: + provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source + FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id + JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=? + ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))] + decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100 + return packet + + +def classify(record, entities): + anchors = {r['paper_id'] for r in record['source_record_hits'] + record['identifier_hits']} + titles = record['title_candidate_ids'] + flags = [] + own_authors = authors_multiset(record['authors']) + if not own_authors: + flags.append('missing_source_authors') + if any(n > 1 for n in own_authors.values()): + flags.append('duplicate_normalized_author_names') + if len(normalized(record['title'])) < 25: + flags.append('short_title') + comparisons = [] + for pid in sorted(anchors | set(titles)): + entity = entities[pid] + other = authors_multiset(entity['authors']) + comparisons.append({'paper_id': pid, + 'normalized_title_equal': normalized(record['title']) == normalized(entity['title']), + 'nonempty_author_multiset_equal': bool(own_authors) and own_authors == other, + 'author_name_overlap': sorted(own_authors.keys() & other.keys())}) + if len(anchors) > 1: + kind = 'conflicting_existing_bindings_hold' + elif record['source_record_hits']: + kind = 'existing_source_record_reconcile_no_replay' + elif anchors: + kind = 'existing_identifier_binding_requires_review' + elif len(titles) >= 21: + kind = 'title_candidate_limit_reached_hold' + elif not titles: + kind = 'no_exact_match_not_proof_of_novelty' + elif len(titles) > 1: + kind = 'multiple_exact_title_entities_hold' + elif comparisons[0]['nonempty_author_multiset_equal']: + kind = 'one_exact_title_author_multiset_requires_review' + elif comparisons[0]['author_name_overlap']: + kind = 'partial_author_overlap_requires_review' + else: + kind = 'no_author_agreement_hold' + return {'kind': kind, 'flags': flags, 'comparisons': comparisons, + 'title_candidates_truncated': len(titles) >= 21, + 'decision': 'not_approved_for_import'} + + +def _atomic_json(path, value): + temporary = path.with_suffix(path.suffix + '.part') + with temporary.open('w', encoding='utf-8') as stream: + json.dump(value, stream, ensure_ascii=False, indent=2) + stream.write('\n') + temporary.replace(path) + + +def audit(files, out, *, pg_service, pg_schema): + # Reject input corruption before opening a database connection or reserving output. + records, inputs = load_inputs(files) + out = Path(out) + out.mkdir(parents=True, exist_ok=False) + (out / 'audit.lock').write_text(timestamp() + '\n') + connection = None + try: + connection = ReadConnection(pg_service, pg_schema) + connection.execute("SET LOCAL statement_timeout='60s'") + if connection.execute('SHOW transaction_read_only').fetchone()[0] != 'on': + raise RuntimeError('A read-only transaction is required') + snapshot = dict(connection.execute('''SELECT current_database() AS database,current_schema() AS schema, + transaction_timestamp()::text AS transaction_at,pg_current_snapshot()::text AS snapshot''').fetchone()) + find_candidates(connection, records) + ids = sorted({hit['paper_id'] for row in records for hit in row['source_record_hits'] + row['identifier_hits']} | + {pid for row in records for pid in row['title_candidate_ids']}) + entities, index = {}, {} + for number, pid in enumerate(ids, 1): + packet = entity_packet(connection, pid) + relative = Path('entities') / str(pid // 10000) / f'{pid}.json.gz' + target = out / relative + target.parent.mkdir(parents=True, exist_ok=True) + raw = gzip.compress(json.dumps(packet, ensure_ascii=False).encode(), mtime=0) + target.write_bytes(raw) + index[str(pid)] = {'file': relative.as_posix(), 'sha256': digest(raw)} + entities[pid] = {'title': packet['paper']['title'], + 'authors': json.loads(packet['paper']['authors_json'])} + if number % 250 == 0 or number == len(ids): + print(json.dumps({'phase': 'entity_evidence', 'completed': number, 'total': len(ids)}), flush=True) + kinds = Counter() + for row in records: + row.update(classify(row, entities)) + kinds[row['kind']] += 1 + report = {'status': 'completed_read_only_candidates_not_import_approval', + 'started_snapshot': snapshot, 'completed_at': timestamp(), 'transaction_read_only': True, + 'database_written': False, 'approved_for_import': 0, 'records': len(records), + 'kind_counts': dict(kinds), 'inputs': inputs, 'entities': index, 'entries': records, + 'limitations': ['Exact keys only; no match does not establish a new work.', + 'Name agreement does not prove person or scientific-work identity.', + 'Bound identifiers can be incorrect; review titles, authors and original evidence.', + 'Source/history limits are explicit in compressed entity packets.', + 'Not a final whole-database deduplication or coverage audit.']} + _atomic_json(out / 'postgresql-preflight.json', report) + return report + except Exception as error: + _atomic_json(out / 'failed.json', {'status': 'failed_no_import_approval', + 'error_type': type(error).__name__, 'database_written': False}) + raise + finally: + if connection is not None: + connection.close() + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog, description=__doc__) + parser.add_argument('--files', nargs='+', required=True, type=Path) + parser.add_argument('--pg-service', required=True) + parser.add_argument('--pg-schema', required=True) + parser.add_argument('--out', required=True, type=Path) + args = parser.parse_args(argv) + report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema) + print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False)) diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index f2601d9..1e17980 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -46,6 +46,7 @@ COMMANDS = { ('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'), ('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'), ('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'), + ('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'), ('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'), } diff --git a/tests/test_source_identity.py b/tests/test_source_identity.py new file mode 100644 index 0000000..09c4e6c --- /dev/null +++ b/tests/test_source_identity.py @@ -0,0 +1,148 @@ +import gzip +import hashlib +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from paper_atlas.audits.source_identity import audit, classify, load_inputs +from paper_atlas.postgres.client import ReadConnection + + +def record(**updates): + row = dict(source='reviewed_program', external_id='2026/forum-a', + title='A sufficiently specific scientific paper title', authors=['Jane Doe', 'John Smith'], + venue='NeurIPS', year=2026, track='main', status='accepted', identifiers={}, + source_record_hits=[], identifier_hits=[], title_candidate_ids=[]) + row.update(updates) + return row + + +def write_input(root, rows): + path = root / 'source.jsonl' + raw = b''.join((json.dumps(row) + '\n').encode() for row in rows) + path.write_bytes(raw) + manifest = {k: rows[0][k] for k in ('source', 'venue', 'year', 'track')} + manifest.update(count=len(rows), file_sha256=hashlib.sha256(raw).hexdigest(), + file='/ignored/untrusted/location.jsonl', status='partial') + path.with_suffix('.manifest.json').write_text(json.dumps(manifest)) + return path + + +class CandidateTests(unittest.TestCase): + def test_duplicate_names_do_not_equal_distinct_author_multiset(self): + row = record(authors=['Jane Doe', 'Jane Doe'], title_candidate_ids=[7]) + result = classify(row, {7: {'title': row['title'], 'authors': ['Jane Doe']}}) + self.assertEqual(result['kind'], 'partial_author_overlap_requires_review') + self.assertIn('duplicate_normalized_author_names', result['flags']) + self.assertFalse(result['comparisons'][0]['nonempty_author_multiset_equal']) + + def test_arxiv_author_representation_and_order_match_but_never_approve(self): + row = record(title_candidate_ids=[7]) + result = classify(row, {7: {'title': row['title'], 'authors': [['Smith', 'John', ''], ['Doe', 'Jane', '']]}}) + self.assertEqual(result['kind'], 'one_exact_title_author_multiset_requires_review') + self.assertEqual(result['decision'], 'not_approved_for_import') + + def test_binding_conflict_overrides_existing_source(self): + row = record(source_record_hits=[{'paper_id': 7}], identifier_hits=[{'paper_id': 8}]) + entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in (7, 8)} + self.assertEqual(classify(row, entities)['kind'], 'conflicting_existing_bindings_hold') + + def test_wrong_doi_binding_retains_author_title_disagreement(self): + row = record(identifier_hits=[{'paper_id': 7}]) + result = classify(row, {7: {'title': 'Completely different work', 'authors': ['Somebody Else']}}) + self.assertEqual(result['kind'], 'existing_identifier_binding_requires_review') + self.assertFalse(result['comparisons'][0]['normalized_title_equal']) + self.assertEqual(result['comparisons'][0]['author_name_overlap'], []) + + def test_no_hit_is_not_novelty_and_missing_authors_never_agree(self): + self.assertEqual(classify(record(), {})['kind'], 'no_exact_match_not_proof_of_novelty') + row = record(authors=[], title_candidate_ids=[7]) + result = classify(row, {7: {'title': row['title'], 'authors': []}}) + self.assertEqual(result['kind'], 'no_author_agreement_hold') + + def test_title_candidate_cap_is_explicit(self): + row = record(title_candidate_ids=list(range(21))) + entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in range(21)} + self.assertTrue(classify(row, entities)['title_candidates_truncated']) + self.assertEqual(classify(row, entities)['kind'], 'title_candidate_limit_reached_hold') + + +class InputTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def test_raw_line_hash_and_manifest_path_not_followed(self): + path = write_input(self.root, [record()]) + rows, inputs = load_inputs([path]) + self.assertEqual(rows[0]['raw_line_sha256'], hashlib.sha256(path.read_bytes()).hexdigest()) + self.assertEqual(inputs[0]['count'], 1) + + def test_hash_tamper_rejected_before_database_connection(self): + path = write_input(self.root, [record()]) + path.write_bytes(path.read_bytes().replace(b'Jane Doe', b'Jane Roe')) + with patch('paper_atlas.audits.source_identity.ReadConnection') as connection: + with self.assertRaisesRegex(ValueError, 'hash mismatch'): + audit([path], self.root / 'out', pg_service='test', pg_schema='research') + connection.assert_not_called() + + def test_manifest_scope_and_count_tamper_and_duplicate_sources(self): + path = write_input(self.root, [record()]) + manifest_path = path.with_suffix('.manifest.json') + manifest = json.loads(manifest_path.read_text()) + for key, value in [('year', 2025), ('count', 2)]: + altered = dict(manifest, **{key: value}) + manifest_path.write_text(json.dumps(altered)) + with self.assertRaises(ValueError): + load_inputs([path]) + manifest_path.write_text(json.dumps(manifest)) + with self.assertRaisesRegex(ValueError, 'Duplicate source'): + load_inputs([path, path]) + + def test_existing_output_never_replayed_and_failure_not_completion(self): + path = write_input(self.root, [record()]) + out = self.root / 'out' + with patch('paper_atlas.audits.source_identity.ReadConnection', side_effect=RuntimeError('unavailable')): + with self.assertRaises(RuntimeError): + audit([path], out, pg_service='test', pg_schema='research') + self.assertTrue((out / 'failed.json').exists()) + self.assertFalse((out / 'postgresql-preflight.json').exists()) + with self.assertRaises(FileExistsError): + audit([path], out, pg_service='test', pg_schema='research') + + +@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the authorized remote PostgreSQL service') +class ReadOnlyIntegrationTests(unittest.TestCase): + def test_existing_binding_full_evidence_and_no_import_approval(self): + service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE'] + connection = ReadConnection(service, 'research') + try: + self.assertEqual(connection.execute('SHOW transaction_read_only').fetchone()[0], 'on') + hit = connection.execute("SELECT value,paper_id FROM identifiers WHERE scheme='arxiv' ORDER BY value LIMIT 1").fetchone() + self.assertIsNotNone(hit) + paper = connection.execute('SELECT title,authors_json FROM papers WHERE id=?', (hit['paper_id'],)).fetchone() + finally: + connection.close() + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + path = write_input(root, [record(title=paper['title'], authors=json.loads(paper['authors_json']), + identifiers={'arxiv': hit['value']})]) + report = audit([path], root / 'out', pg_service=service, pg_schema='research') + self.assertTrue(report['transaction_read_only']) + self.assertFalse(report['database_written']) + self.assertEqual(report['approved_for_import'], 0) + row, = report['entries'] + self.assertIn(hit['paper_id'], [r['paper_id'] for r in row['identifier_hits']]) + shard = report['entities'][str(hit['paper_id'])] + raw = (root / 'out' / shard['file']).read_bytes() + self.assertEqual(hashlib.sha256(raw).hexdigest(), shard['sha256']) + packet = json.loads(gzip.decompress(raw)) + self.assertEqual(packet['paper']['id'], hit['paper_id']) + self.assertEqual(packet['paper']['title'], paper['title']) + self.assertEqual(packet['paper']['authors_json'], paper['authors_json']) + self.assertIn('attempts', packet) + self.assertTrue((root / 'out/postgresql-preflight.json').is_file())