feat: add scoped document API and resumable batch uploader

scripts/docs-admin.sh

@@ -3,7 +3,7 @@
 set -eu
 cd /opt/endorphin-tabularium-docs/current
 case "${1:-}" in
-  list|get|save|history|export|backup|import)
+  list|get|save|history|export|backup|import|key-create|key-list|key-revoke)
     docker compose -p endorphin-tabularium-docs --env-file /etc/endorphin-tabularium-docs/production.env --env-file release.env -f compose.yml exec -T docs node apps/docs/docs-cli.cjs "$@" ;;
-  *) echo 'Usage: docs list | get <slug> | save < JSON | history <sourceSlug> | export <private-file> | backup <private-file> | import < JSON' >&2; exit 1 ;;
+  *) echo 'Usage: docs key-create <name> <private-file> [read|read-write] [days] | key-list | key-revoke <id> | list | get <slug> | save < JSON | history <sourceSlug> | export <private-file> | backup <private-file> | import < JSON' >&2; exit 1 ;;
 esac

scripts/documents-client.py

@@ -0,0 +1,167 @@
+"""Upload documents through the database API; credentials stay in a private file."""
+import argparse
+import json
+import os
+from pathlib import Path
+import stat
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import uuid
+
+
+class DocumentClientError(ValueError):
+    pass
+
+
+class NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        # Never forward a bearer credential to a redirected host or protocol.
+        return None
+
+
+def server_url(value):
+    parsed = urllib.parse.urlsplit(value)
+    if (parsed.username or parsed.password or parsed.query or parsed.fragment
+            or parsed.path not in ('', '/') or not parsed.hostname):
+        raise ValueError('Provide a server origin without credentials, path, query or fragment')
+    if parsed.scheme != 'https' and not (
+            parsed.scheme == 'http' and parsed.hostname in ('localhost', '127.0.0.1', '::1')):
+        raise ValueError('Use HTTPS or an HTTP loopback connection through a trusted SSH tunnel')
+    return value.rstrip('/')
+
+
+def read_token(filename):
+    path = Path(filename)
+    if stat.S_IMODE(path.stat().st_mode) & 0o077:
+        raise ValueError('API key file must be private (chmod 600)')
+    value = path.read_text().strip()
+    if value.startswith('{'):
+        value = json.loads(value)['token']
+    if not isinstance(value, str) or not value.startswith('tdoc_') or len(value) != 48:
+        raise ValueError('Invalid document API key file')
+    return value
+
+
+def request(base, token, method, path, payload=None, request_id=None):
+    data = None if payload is None else json.dumps(payload, ensure_ascii=False).encode()
+    if data is not None and len(data) > 8 * 1024 * 1024:
+        raise ValueError('Request exceeds 8 MiB; split into smaller atomic batches')
+    headers = {'Authorization': 'Bearer ' + token, 'Accept': 'application/json'}
+    if data is not None:
+        headers['Content-Type'] = 'application/json'
+        headers['Idempotency-Key'] = request_id
+    opener = urllib.request.build_opener(NoRedirect())
+    for attempt in range(2):
+        req = urllib.request.Request(base + path, data=data, method=method, headers=headers)
+        try:
+            with opener.open(req, timeout=30) as response:
+                return json.load(response)
+        except urllib.error.HTTPError as error:
+            if error.code in (502, 503, 504) and attempt == 0:
+                error.close()
+                time.sleep(1)
+                continue
+            status = error.code
+            error.close()
+            explanations = {400: 'Invalid document or pagination', 401: 'Key missing, expired or revoked',
+                            403: 'Read-only key', 404: 'Document or revision not found',
+                            409: 'Revision or request-id conflict; read current state before retrying',
+                            413: 'Request too large', 503: 'Database unavailable'}
+            raise DocumentClientError(f'HTTP {status}: ' + explanations.get(status, 'Request failed')) from None
+        except (urllib.error.URLError, TimeoutError):
+            if attempt == 0:
+                time.sleep(1)
+                continue
+            raise DocumentClientError('Connection failed; retry using the same request id and unchanged input') from None
+
+
+def document_batches(documents, max_bytes=8 * 1024 * 1024):
+    if not isinstance(documents, list) or not documents:
+        raise DocumentClientError('Provide a nonempty documents array')
+    batches, chunk, size, seen = [], [], 20, set()
+    for document in documents:
+        if not isinstance(document, dict) or not isinstance(document.get('sourceSlug'), str):
+            raise DocumentClientError('Every document needs a sourceSlug')
+        if document['sourceSlug'] in seen:
+            raise DocumentClientError('Duplicate sourceSlug in upload')
+        seen.add(document['sourceSlug'])
+        encoded = len(json.dumps(document, ensure_ascii=False).encode()) + 2
+        if encoded + 20 > max_bytes:
+            raise DocumentClientError('A document exceeds the request limit')
+        if chunk and (len(chunk) == 100 or size + encoded > max_bytes):
+            batches.append(chunk)
+            chunk, size = [], 20
+        chunk.append(document)
+        size += encoded
+    batches.append(chunk)
+    return batches
+
+
+def main():
+    os.umask(0o077)
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--url', default=os.environ.get('TABULARIUM_DOCS_URL'))
+    parser.add_argument('--key-file', default=os.environ.get('TABULARIUM_DOCS_KEY_FILE'))
+    sub = parser.add_subparsers(dest='command', required=True)
+    listing = sub.add_parser('list')
+    listing.add_argument('--limit', type=int, default=50)
+    listing.add_argument('--offset', type=int, default=0)
+    reading = sub.add_parser('get')
+    reading.add_argument('slug')
+    mode = reading.add_mutually_exclusive_group()
+    mode.add_argument('--history', action='store_true')
+    mode.add_argument('--revision', type=int)
+    reading.add_argument('--limit', type=int, default=50)
+    reading.add_argument('--offset', type=int, default=0)
+    saving = sub.add_parser('save')
+    saving.add_argument('file', help='Private JSON file or - for stdin; one document or {documents: [...]}')
+    saving.add_argument('--request-id', help='Reuse this id only with the same input when retrying')
+    args = parser.parse_args()
+    if not args.url or not args.key_file:
+        parser.error('Set --url and --key-file or TABULARIUM_DOCS_URL and TABULARIUM_DOCS_KEY_FILE')
+    try:
+        base = server_url(args.url)
+        token = read_token(args.key_file)
+        if args.command == 'list':
+            result = request(base, token, 'GET', '/api/documents?' + urllib.parse.urlencode(
+                {'limit': args.limit, 'offset': args.offset}))
+        elif args.command == 'get':
+            path = '/api/documents/item/' + urllib.parse.quote(args.slug, safe='/')
+            if args.history:
+                path += '?' + urllib.parse.urlencode({'history': 1, 'limit': args.limit, 'offset': args.offset})
+            elif args.revision is not None:
+                path += '?' + urllib.parse.urlencode({'revision': args.revision})
+            result = request(base, token, 'GET', path)
+        else:
+            payload = json.load(sys.stdin) if args.file == '-' else json.loads(Path(args.file).read_text())
+            request_id = args.request_id or str(uuid.uuid4())
+            if not request_id or len(request_id) > 100 or any(c not in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._:-' for c in request_id):
+                raise ValueError('Invalid request id')
+            print('Request id: ' + request_id, file=sys.stderr)
+            if isinstance(payload, dict) and 'documents' in payload:
+                if set(payload) != {'documents'}:
+                    raise DocumentClientError('Batch payload must contain only documents')
+                chunks = document_batches(payload['documents'])
+                receipts, replayed = [], 0
+                for index, chunk in enumerate(chunks, 1):
+                    batch_id = f'{request_id}:{index}'
+                    saved = request(base, token, 'POST', '/api/documents/batch', {'documents': chunk}, batch_id)
+                    receipts.extend(saved['documents'])
+                    replayed += int(saved['replayed'])
+                    print(f'Batch {index}/{len(chunks)} committed; {len(receipts)} documents total; request id: {batch_id}', file=sys.stderr)
+                result = {'documents': receipts, 'batches': len(chunks), 'replayedBatches': replayed, 'requestId': request_id}
+            else:
+                result = request(base, token, 'POST', '/api/documents', payload, request_id)
+        print(json.dumps(result, ensure_ascii=False, indent=2))
+    except DocumentClientError as error:
+        raise SystemExit(str(error)) from None
+    except (ValueError, OSError, KeyError, TypeError):
+        # Never print exception objects that could contain credentials or document data.
+        raise SystemExit('Document request failed. Check the URL, private key file, JSON schema and server response; credentials omitted.') from None
+
+
+if __name__ == '__main__':
+    main()

scripts/test_documents_client.py

@@ -0,0 +1,85 @@
+"""Lightweight client tests: no HTTP server, compilation or real credentials."""
+import importlib.util
+import io
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest.mock import patch
+import urllib.error
+import urllib.request
+
+spec = importlib.util.spec_from_file_location('documents_client', Path(__file__).with_name('documents-client.py'))
+client = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(client)
+
+
+class ClientTests(unittest.TestCase):
+    def test_only_tls_or_loopback_origins_are_accepted(self):
+        self.assertEqual(client.server_url('https://docs.example/'), 'https://docs.example')
+        self.assertEqual(client.server_url('http://127.0.0.1:18135'), 'http://127.0.0.1:18135')
+        for url in ['http://docs.example', 'https://user:secret@docs.example', 'file:///tmp/data',
+                    'https://docs.example/api', 'https://docs.example/?token=secret']:
+            with self.assertRaises(ValueError):
+                client.server_url(url)
+
+    def test_key_file_permissions_and_formats(self):
+        with tempfile.TemporaryDirectory() as directory:
+            path = Path(directory) / 'key.json'
+            token = 'tdoc_' + 'x' * 43
+            path.write_text(json.dumps({'token': token}))
+            path.chmod(0o600)
+            self.assertEqual(client.read_token(path), token)
+            path.chmod(0o644)
+            with self.assertRaises(ValueError):
+                client.read_token(path)
+
+    def test_redirects_never_forward_bearer_credentials(self):
+        request = urllib.request.Request('https://docs.example', headers={'Authorization': 'Bearer test-only'})
+        self.assertIsNone(client.NoRedirect().redirect_request(request, None, 302, '', {}, 'https://other.example'))
+
+    def test_network_retry_preserves_idempotency_and_body(self):
+        calls = []
+
+        class Opener:
+            def open(self, request, timeout):
+                calls.append(request)
+                if len(calls) == 1:
+                    raise urllib.error.URLError('temporary network failure')
+                return io.BytesIO(b'{"replayed":true}')
+
+        with patch.object(client.urllib.request, 'build_opener', return_value=Opener()), patch.object(client.time, 'sleep'):
+            result = client.request('https://docs.example', 'test-token', 'POST', '/api/documents', {'revision': 0}, 'request-1')
+        self.assertTrue(result['replayed'])
+        self.assertEqual(calls[0].data, calls[1].data)
+        self.assertEqual(calls[0].get_header('Idempotency-key'), 'request-1')
+        self.assertEqual(calls[1].get_header('Idempotency-key'), 'request-1')
+
+    def test_large_uploads_are_split_by_count_and_bytes(self):
+        documents = [{'sourceSlug': str(i), 'content': 'a' * 20} for i in range(201)]
+        batches = client.document_batches(documents)
+        self.assertEqual([len(batch) for batch in batches], [100, 100, 1])
+        self.assertEqual([doc for batch in batches for doc in batch], documents)
+        bounded = client.document_batches(documents[:5], max_bytes=150)
+        self.assertTrue(all(len(json.dumps({'documents': batch}, ensure_ascii=False).encode()) <= 150 for batch in bounded))
+
+    def test_duplicate_identifiers_and_oversized_items_fail_before_upload(self):
+        with self.assertRaises(ValueError):
+            client.document_batches([{'sourceSlug': 'same'}, {'sourceSlug': 'same'}])
+        with self.assertRaises(ValueError):
+            client.document_batches([{'sourceSlug': 'one', 'content': 'a' * 500}], max_bytes=100)
+
+    def test_conflict_is_reported_without_printing_server_content_or_secret(self):
+        class Opener:
+            def open(self, request, timeout):
+                raise urllib.error.HTTPError(request.full_url, 409, 'Conflict', {}, io.BytesIO(b'{"error":"private-content"}'))
+
+        with patch.object(client.urllib.request, 'build_opener', return_value=Opener()):
+            with self.assertRaisesRegex(client.DocumentClientError, 'HTTP 409') as result:
+                client.request('https://docs.example', 'private-token', 'POST', '/api/documents', {}, 'request-1')
+        self.assertNotIn('private-token', str(result.exception))
+        self.assertNotIn('private-content', str(result.exception))
+
+
+if __name__ == '__main__':
+    unittest.main()