213 lines
11 KiB
Python
213 lines
11 KiB
Python
"""Add reviewed stable identifiers without rewriting bibliographic source fields.
|
|
|
|
Hashes establish file integrity, not the truth of an identity assertion. Identity
|
|
review and a fresh database preflight remain separate requirements.
|
|
"""
|
|
import argparse
|
|
from datetime import datetime
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
from urllib.parse import urlsplit
|
|
|
|
|
|
SCOPE = 'input_and_evidence_integrity_only'
|
|
MAX_METADATA_BYTES = 16 * 1024 * 1024
|
|
|
|
|
|
def digest(data):
|
|
return hashlib.sha256(data).hexdigest()
|
|
|
|
|
|
def object_keys(value, required, optional=()):
|
|
if not isinstance(value, dict) or not set(required) <= value.keys() or value.keys() - set(required) - set(optional):
|
|
raise ValueError('Missing or unsupported plan fields')
|
|
|
|
|
|
def json_bytes(value):
|
|
return (json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n').encode('utf-8')
|
|
|
|
|
|
def decode(raw):
|
|
def unique(pairs):
|
|
result = {}
|
|
for key, value in pairs:
|
|
if key in result:
|
|
raise ValueError('Duplicate JSON key: ' + key)
|
|
result[key] = value
|
|
return result
|
|
return json.loads(raw, object_pairs_hook=unique,
|
|
parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Non-finite JSON value')))
|
|
|
|
|
|
def metadata(path, expected=None):
|
|
with Path(path).open('rb') as stream:
|
|
raw = stream.read(MAX_METADATA_BYTES + 1)
|
|
if len(raw) > MAX_METADATA_BYTES:
|
|
raise ValueError('Metadata exceeds size limit')
|
|
if expected is not None:
|
|
check_hash(raw, expected)
|
|
return raw, decode(raw)
|
|
|
|
|
|
def check_hash(raw, expected):
|
|
if not isinstance(expected, str) or not re.fullmatch('[0-9a-f]{64}', expected) or digest(raw) != expected:
|
|
raise ValueError('SHA256 mismatch')
|
|
|
|
|
|
def identifier(scheme, value):
|
|
if not isinstance(value, str) or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value):
|
|
raise ValueError('Invalid identifier')
|
|
if scheme == 'arxiv':
|
|
pattern = r'(?:\d{2}(?:0[1-9]|1[0-2])\.\d{4,5}|[a-zA-Z][a-zA-Z.-]*/\d{2}(?:0[1-9]|1[0-2])\d{3})'
|
|
valid = re.fullmatch(pattern, value)
|
|
elif scheme == 'doi':
|
|
valid = re.fullmatch(r'10\.\d{4,9}/\S+', value) and value == value.lower()
|
|
else:
|
|
valid = False
|
|
if not valid:
|
|
raise ValueError('Unsupported or noncanonical identifier: ' + scheme)
|
|
|
|
|
|
def prepare(plan_path, output):
|
|
plan_raw, plan = metadata(plan_path)
|
|
object_keys(plan, ('format_version', 'inputs', 'assertions'))
|
|
if type(plan['format_version']) is not int or plan['format_version'] != 1:
|
|
raise ValueError('Unsupported format version')
|
|
if not isinstance(plan['inputs'], list) or not plan['inputs'] or not isinstance(plan['assertions'], list) or not plan['assertions']:
|
|
raise ValueError('Inputs and assertions must be nonempty lists')
|
|
output = Path(output).absolute()
|
|
if output.exists():
|
|
raise ValueError('Output directory must not already exist')
|
|
if not output.parent.is_dir():
|
|
raise ValueError('Output parent directory must exist')
|
|
assertions = {}
|
|
evidence_files = {}
|
|
checked_evidence = {}
|
|
for assertion in plan['assertions']:
|
|
object_keys(assertion, ('source', 'external_id', 'record_sha256', 'identifiers', 'evidence'))
|
|
if any(not isinstance(assertion[key], str) or not assertion[key] for key in ('source', 'external_id')):
|
|
raise ValueError('Invalid source identity')
|
|
key = (assertion['source'], assertion['external_id'])
|
|
if key in assertions:
|
|
raise ValueError('Duplicate identity assertion')
|
|
ids = assertion['identifiers']
|
|
if not isinstance(ids, dict) or not ids:
|
|
raise ValueError('Identifiers must be a nonempty object')
|
|
for scheme, value in ids.items():
|
|
identifier(scheme, value)
|
|
if not isinstance(assertion['evidence'], list) or not assertion['evidence']:
|
|
raise ValueError('Evidence is required')
|
|
references = []
|
|
for evidence in assertion['evidence']:
|
|
object_keys(evidence, ('file', 'sha256', 'url', 'checked_at', 'note'))
|
|
if not all(isinstance(value, str) and value for value in evidence.values()):
|
|
raise ValueError('Invalid evidence fields')
|
|
url = urlsplit(evidence['url'])
|
|
if url.scheme != 'https' or not url.hostname or url.username or url.password:
|
|
raise ValueError('Evidence requires an HTTPS URL without credentials')
|
|
if datetime.fromisoformat(evidence['checked_at'].replace('Z', '+00:00')).utcoffset() is None:
|
|
raise ValueError('Evidence timestamp requires a timezone')
|
|
evidence_key = (str(Path(evidence['file']).absolute()), evidence['sha256'])
|
|
if evidence_key not in checked_evidence:
|
|
checked_evidence[evidence_key] = metadata(evidence['file'], evidence['sha256'])[0]
|
|
raw = checked_evidence[evidence_key]
|
|
evidence_files[evidence['sha256']] = raw
|
|
references.append({**evidence, 'file': 'evidence/' + evidence['sha256'] + '.json'})
|
|
assertions[key] = (assertion, references)
|
|
seen = set()
|
|
applied = set()
|
|
names = set()
|
|
report = {'format_version': 1, 'plan_sha256': digest(plan_raw),
|
|
'verification_scope': SCOPE, 'semantic_identity_verified': False,
|
|
'database_written': False, 'files': [], 'assertions_applied': 0, 'identifiers_added': 0}
|
|
with tempfile.TemporaryDirectory(prefix='.identifier-curation-', dir=output.parent) as temporary:
|
|
staging = Path(temporary) / 'prepared'
|
|
staging.mkdir()
|
|
(staging / 'evidence').mkdir()
|
|
(staging / 'plan.json').write_bytes(plan_raw)
|
|
for sha, raw in evidence_files.items():
|
|
(staging / 'evidence' / (sha + '.json')).write_bytes(raw)
|
|
for entry in plan['inputs']:
|
|
object_keys(entry, ('file', 'sha256', 'manifest_sha256'))
|
|
path = Path(entry['file'])
|
|
if path.suffix != '.jsonl' or path.name in names:
|
|
raise ValueError('Inputs require distinct JSONL basenames')
|
|
names.add(path.name)
|
|
_, manifest = metadata(path.with_suffix('.manifest.json'), entry['manifest_sha256'])
|
|
if not isinstance(manifest, dict) or not {'source','venue','year','track','count','list_url','file_sha256'} <= manifest.keys():
|
|
raise ValueError('Complete source manifest required')
|
|
if manifest['file_sha256'] != entry['sha256']:
|
|
raise ValueError('Manifest and plan hashes differ')
|
|
before = hashlib.sha256()
|
|
after = hashlib.sha256()
|
|
count = 0
|
|
with path.open('rb') as source, (staging / path.name).open('wb') as target:
|
|
for raw in source:
|
|
before.update(raw)
|
|
row = decode(raw)
|
|
if any(row[field] != manifest[field] for field in ('source', 'venue', 'year', 'track')):
|
|
raise ValueError('Record differs from source manifest')
|
|
key = (row['source'], row['external_id'])
|
|
if key in seen:
|
|
raise ValueError('Duplicate source identity in input batch')
|
|
seen.add(key)
|
|
if key in assertions:
|
|
assertion, references = assertions[key]
|
|
check_hash(raw, assertion['record_sha256'])
|
|
existing = row.get('identifiers', {})
|
|
if existing is None:
|
|
existing = {}
|
|
if not isinstance(existing, dict):
|
|
raise ValueError('Invalid existing identifiers')
|
|
for scheme, value in assertion['identifiers'].items():
|
|
if scheme in existing and existing[scheme] != value:
|
|
raise ValueError('Existing identifier conflict')
|
|
details = row.setdefault('metadata', {})
|
|
if not isinstance(details, dict):
|
|
raise ValueError('Invalid record metadata')
|
|
history = details.setdefault('identifier_curation', [])
|
|
if not isinstance(history, list):
|
|
raise ValueError('Invalid curation history')
|
|
history.append({'plan_sha256': digest(plan_raw), 'input_file_sha256': entry['sha256'],
|
|
'record_sha256': digest(raw), 'previous_identifiers': dict(existing),
|
|
'identifiers_asserted': assertion['identifiers'], 'evidence': references,
|
|
'verification_scope': SCOPE})
|
|
report['identifiers_added'] += len(assertion['identifiers'].keys() - existing.keys())
|
|
row['identifiers'] = {**existing, **assertion['identifiers']}
|
|
raw = json_bytes(row)
|
|
applied.add(key)
|
|
target.write(raw)
|
|
after.update(raw)
|
|
count += 1
|
|
if before.hexdigest() != entry['sha256'] or not re.fullmatch('[0-9a-f]{64}', entry['sha256']):
|
|
raise ValueError('Input SHA256 mismatch')
|
|
if not count or type(manifest['count']) is not int or count != manifest['count']:
|
|
raise ValueError('Manifest count mismatch')
|
|
provenance = {'plan_sha256': digest(plan_raw), 'input_file_sha256': before.hexdigest(),
|
|
'input_manifest_sha256': entry['manifest_sha256'], 'verification_scope': SCOPE}
|
|
prepared = {**manifest, 'file': str(output / path.name), 'file_sha256': after.hexdigest(),
|
|
'identifier_curation': provenance}
|
|
(staging / path.with_suffix('.manifest.json').name).write_bytes(json_bytes(prepared))
|
|
report['files'].append({'file': str(output / path.name), 'count': count,
|
|
'input_sha256': before.hexdigest(), 'sha256': after.hexdigest()})
|
|
if applied != assertions.keys():
|
|
raise ValueError('Assertion target absent from input batch')
|
|
report['assertions_applied'] = len(applied)
|
|
(staging / 'curation-report.json').write_bytes(json_bytes(report))
|
|
# All files are on the same filesystem. No database or input mutation.
|
|
if output.exists():
|
|
raise ValueError('Output directory appeared during preparation')
|
|
staging.rename(output)
|
|
return report
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
|
parser.add_argument('--plan', required=True, type=Path)
|
|
parser.add_argument('--output', required=True, type=Path)
|
|
args = parser.parse_args(argv)
|
|
print(json.dumps(prepare(args.plan, args.output), ensure_ascii=False))
|