"""Add reviewed stable identifiers without rewriting bibliographic source fields. Hashes establish file integrity, not the truth of an identity assertion. Identity review and a fresh database preflight remain separate requirements. """ import argparse from datetime import datetime import hashlib import json from pathlib import Path import re import tempfile from urllib.parse import urlsplit SCOPE = 'input_and_evidence_integrity_only' MAX_METADATA_BYTES = 16 * 1024 * 1024 def digest(data): return hashlib.sha256(data).hexdigest() def object_keys(value, required, optional=()): if not isinstance(value, dict) or not set(required) <= value.keys() or value.keys() - set(required) - set(optional): raise ValueError('Missing or unsupported plan fields') def json_bytes(value): return (json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n').encode('utf-8') def decode(raw): def unique(pairs): result = {} for key, value in pairs: if key in result: raise ValueError('Duplicate JSON key: ' + key) result[key] = value return result return json.loads(raw, object_pairs_hook=unique, parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Non-finite JSON value'))) def metadata(path, expected=None): with Path(path).open('rb') as stream: raw = stream.read(MAX_METADATA_BYTES + 1) if len(raw) > MAX_METADATA_BYTES: raise ValueError('Metadata exceeds size limit') if expected is not None: check_hash(raw, expected) return raw, decode(raw) def check_hash(raw, expected): if not isinstance(expected, str) or not re.fullmatch('[0-9a-f]{64}', expected) or digest(raw) != expected: raise ValueError('SHA256 mismatch') def identifier(scheme, value): if not isinstance(value, str) or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value): raise ValueError('Invalid identifier') if scheme == 'arxiv': pattern = r'(?:\d{2}(?:0[1-9]|1[0-2])\.\d{4,5}|[a-zA-Z][a-zA-Z.-]*/\d{2}(?:0[1-9]|1[0-2])\d{3})' valid = re.fullmatch(pattern, value) elif scheme == 'doi': valid = re.fullmatch(r'10\.\d{4,9}/\S+', value) and value == value.lower() else: valid = False if not valid: raise ValueError('Unsupported or noncanonical identifier: ' + scheme) def prepare(plan_path, output): plan_raw, plan = metadata(plan_path) object_keys(plan, ('format_version', 'inputs', 'assertions')) if type(plan['format_version']) is not int or plan['format_version'] != 1: raise ValueError('Unsupported format version') if not isinstance(plan['inputs'], list) or not plan['inputs'] or not isinstance(plan['assertions'], list) or not plan['assertions']: raise ValueError('Inputs and assertions must be nonempty lists') output = Path(output).absolute() if output.exists(): raise ValueError('Output directory must not already exist') if not output.parent.is_dir(): raise ValueError('Output parent directory must exist') assertions = {} evidence_files = {} checked_evidence = {} for assertion in plan['assertions']: object_keys(assertion, ('source', 'external_id', 'record_sha256', 'identifiers', 'evidence')) if any(not isinstance(assertion[key], str) or not assertion[key] for key in ('source', 'external_id')): raise ValueError('Invalid source identity') key = (assertion['source'], assertion['external_id']) if key in assertions: raise ValueError('Duplicate identity assertion') ids = assertion['identifiers'] if not isinstance(ids, dict) or not ids: raise ValueError('Identifiers must be a nonempty object') for scheme, value in ids.items(): identifier(scheme, value) if not isinstance(assertion['evidence'], list) or not assertion['evidence']: raise ValueError('Evidence is required') references = [] for evidence in assertion['evidence']: object_keys(evidence, ('file', 'sha256', 'url', 'checked_at', 'note')) if not all(isinstance(value, str) and value for value in evidence.values()): raise ValueError('Invalid evidence fields') url = urlsplit(evidence['url']) if url.scheme != 'https' or not url.hostname or url.username or url.password: raise ValueError('Evidence requires an HTTPS URL without credentials') if datetime.fromisoformat(evidence['checked_at'].replace('Z', '+00:00')).utcoffset() is None: raise ValueError('Evidence timestamp requires a timezone') evidence_key = (str(Path(evidence['file']).absolute()), evidence['sha256']) if evidence_key not in checked_evidence: checked_evidence[evidence_key] = metadata(evidence['file'], evidence['sha256'])[0] raw = checked_evidence[evidence_key] evidence_files[evidence['sha256']] = raw references.append({**evidence, 'file': 'evidence/' + evidence['sha256'] + '.json'}) assertions[key] = (assertion, references) seen = set() applied = set() names = set() report = {'format_version': 1, 'plan_sha256': digest(plan_raw), 'verification_scope': SCOPE, 'semantic_identity_verified': False, 'database_written': False, 'files': [], 'assertions_applied': 0, 'identifiers_added': 0} with tempfile.TemporaryDirectory(prefix='.identifier-curation-', dir=output.parent) as temporary: staging = Path(temporary) / 'prepared' staging.mkdir() (staging / 'evidence').mkdir() (staging / 'plan.json').write_bytes(plan_raw) for sha, raw in evidence_files.items(): (staging / 'evidence' / (sha + '.json')).write_bytes(raw) for entry in plan['inputs']: object_keys(entry, ('file', 'sha256', 'manifest_sha256')) path = Path(entry['file']) if path.suffix != '.jsonl' or path.name in names: raise ValueError('Inputs require distinct JSONL basenames') names.add(path.name) _, manifest = metadata(path.with_suffix('.manifest.json'), entry['manifest_sha256']) if not isinstance(manifest, dict) or not {'source','venue','year','track','count','list_url','file_sha256'} <= manifest.keys(): raise ValueError('Complete source manifest required') if manifest['file_sha256'] != entry['sha256']: raise ValueError('Manifest and plan hashes differ') before = hashlib.sha256() after = hashlib.sha256() count = 0 with path.open('rb') as source, (staging / path.name).open('wb') as target: for raw in source: before.update(raw) row = decode(raw) if any(row[field] != manifest[field] for field in ('source', 'venue', 'year', 'track')): raise ValueError('Record differs from source manifest') key = (row['source'], row['external_id']) if key in seen: raise ValueError('Duplicate source identity in input batch') seen.add(key) if key in assertions: assertion, references = assertions[key] check_hash(raw, assertion['record_sha256']) existing = row.get('identifiers', {}) if existing is None: existing = {} if not isinstance(existing, dict): raise ValueError('Invalid existing identifiers') for scheme, value in assertion['identifiers'].items(): if scheme in existing and existing[scheme] != value: raise ValueError('Existing identifier conflict') details = row.setdefault('metadata', {}) if not isinstance(details, dict): raise ValueError('Invalid record metadata') history = details.setdefault('identifier_curation', []) if not isinstance(history, list): raise ValueError('Invalid curation history') history.append({'plan_sha256': digest(plan_raw), 'input_file_sha256': entry['sha256'], 'record_sha256': digest(raw), 'previous_identifiers': dict(existing), 'identifiers_asserted': assertion['identifiers'], 'evidence': references, 'verification_scope': SCOPE}) report['identifiers_added'] += len(assertion['identifiers'].keys() - existing.keys()) row['identifiers'] = {**existing, **assertion['identifiers']} raw = json_bytes(row) applied.add(key) target.write(raw) after.update(raw) count += 1 if before.hexdigest() != entry['sha256'] or not re.fullmatch('[0-9a-f]{64}', entry['sha256']): raise ValueError('Input SHA256 mismatch') if not count or type(manifest['count']) is not int or count != manifest['count']: raise ValueError('Manifest count mismatch') provenance = {'plan_sha256': digest(plan_raw), 'input_file_sha256': before.hexdigest(), 'input_manifest_sha256': entry['manifest_sha256'], 'verification_scope': SCOPE} prepared = {**manifest, 'file': str(output / path.name), 'file_sha256': after.hexdigest(), 'identifier_curation': provenance} (staging / path.with_suffix('.manifest.json').name).write_bytes(json_bytes(prepared)) report['files'].append({'file': str(output / path.name), 'count': count, 'input_sha256': before.hexdigest(), 'sha256': after.hexdigest()}) if applied != assertions.keys(): raise ValueError('Assertion target absent from input batch') report['assertions_applied'] = len(applied) (staging / 'curation-report.json').write_bytes(json_bytes(report)) # All files are on the same filesystem. No database or input mutation. if output.exists(): raise ValueError('Output directory appeared during preparation') staging.rename(output) return report def main(argv=None, *, prog=None): parser = argparse.ArgumentParser(prog=prog, description=__doc__) parser.add_argument('--plan', required=True, type=Path) parser.add_argument('--output', required=True, type=Path) args = parser.parse_args(argv) print(json.dumps(prepare(args.plan, args.output), ensure_ascii=False))