diff --git a/deploy/research/query-indexes.sql b/deploy/research/query-indexes.sql new file mode 100644 index 0000000..314cf06 --- /dev/null +++ b/deploy/research/query-indexes.sql @@ -0,0 +1,14 @@ +-- Manual operator step against the published Paper Atlas schema, outside a +-- transaction. Core's SELECT-only runtime role must never execute this file. +-- These indexes support the actual author search and exact category filter; +-- the migration already owns the title/abstract English GIN index. +SET lock_timeout = '2s'; +SET statement_timeout = '20min'; +SET maintenance_work_mem = '32MB'; +SET max_parallel_maintenance_workers = 0; +CREATE INDEX CONCURRENTLY IF NOT EXISTS paper_tags_tag_paper + ON research.paper_tags(tag_id,paper_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS papers_authors_search_gin + ON research.papers USING gin(to_tsvector('simple'::regconfig,authors_json)); +-- After an interrupted concurrent build, inspect pg_index.indisvalid before +-- reuse. IF NOT EXISTS alone does not repair an invalid index. diff --git a/docs/research-api.md b/docs/research-api.md index d15ee56..bc825bb 100644 --- a/docs/research-api.md +++ b/docs/research-api.md @@ -9,12 +9,12 @@ collections remain project scoped. `internal/research` implements the HTTP contract and a PostgreSQL repository based on the fixed Paper Atlas `a3b677b` schema (TEXT/BIGINT columns, native English title/abstract search index). `cmd/foundry` enables it only when -`FOUNDRY_RESEARCH_DATABASE_URL` is configured. No live research connection has -been configured during the ongoing migration. -The existing SQLite implementation was withdrawn. Wait for the separate Paper -Atlas PostgreSQL migration to finish, inspect the published schema and indexes, -then verify the queries, indexes and full-corpus performance with a -least-privilege read-only connection before enabling it in the existing server. +`FOUNDRY_RESEARCH_DATABASE_URL` is configured. The Paper Atlas migration is now published: 21 tables passed its independent +count/content-hash acceptance. A dedicated `foundry_research_reader` role has +SELECT on the ten tables consumed here, schema USAGE, and read-only defaults. +Its connection is stored only in the protected server file below. +The existing SQLite implementation was withdrawn. Full-corpus API performance +and frontend integration still require verification before enabling production. Do not add `FOUNDRY_RESEARCH_DATABASE`. The server-side deployment may load a protected @@ -23,7 +23,10 @@ The server-side deployment may load a protected Keep the connection string out of Git, Actions variables, browser configuration and command arguments. Use a dedicated role with schema USAGE and table SELECT only. Core uses a separate two-connection pool, read-only connection defaults and -read-only transactions. Startup checks the published schema/SELECT permissions; +read-only transactions. Each research transaction disables JIT locally: the +measured count plan spent about 2.85 seconds compiling JIT functions, and its +end-to-end execution exceeded eight seconds. This does not alter shared +PostgreSQL settings or replace exact counts with estimates. Startup checks the published schema/SELECT permissions; an explicitly enabled but invalid connection fails candidate readiness. Deployment migration defaults now follow the shared `foundry-postgres` @@ -47,16 +50,25 @@ raw SQL or user-supplied full-text syntax. Title/abstract search uses the migration's English vector with `plainto_tsquery`; author search uses simple-language terms over the stored -author JSON. Both treat input as terms, never full-text operators. Author search -and topic filtering still need full-corpus execution-plan/performance checks -after publication; the migration's title/abstract index does not index authors. +author JSON. Both treat input as terms, never full-text operators. Initial full-corpus measurements showed eight-second timeouts for text, +author and topic counts; the unfiltered count took 7.25 seconds. Execution plans +confirmed sequential scans for authors and category joins. +`deploy/research/query-indexes.sql` supplies operator-run concurrent indexes for +those two missing access paths. It is separate from Core startup and auth +migrations; do not grant DDL to the runtime reader. Both indexes completed on the confirmed ECS (category 35.4 seconds, author +436.47 seconds); end-to-end timings are still under verification. List responses contain `items`, `total`, `topics`, `page`, and `pageSize`. Each paper preserves original title/abstract, authors, source dates, topics and source links. Publication decisions need source evidence; absent decisions stay unknown. No invented repositories, metrics or generated research conclusions. -The repository must count and read each list in one read-only snapshot. The +The repository counts and reads each list in one read-only snapshot. +Paging first materializes matching IDs, then joins only that page to the wide +paper rows. Filtered searches first materialize matching IDs without an +ORDER BY/LIMIT, then sort/page those IDs, preventing sparse terms from choosing +a backward heap scan with repeated text-vector computation. Activity sampling showed that selecting full rows before LIMIT +caused sparse category queries to read large portions of the paper heap. The HTTP layer permits two in-flight queries with an eight-second context budget. Busy requests return 429 with `Retry-After: 1`; deadlines return 504; storage errors return a generic 503 without database details. Missing papers return 404. @@ -71,11 +83,21 @@ query performance or production availability. The PostgreSQL repository batches source/tag/identifier/submission queries in the same read-only repeatable-read transaction. Author tuples are displayed as -given name, family name and suffix; null abstracts/dates remain null. Publication +given name, family name and suffix; an optional fourth affiliation field +is excluded from the display name (observed in real arXiv records); null abstracts/dates remain null. Publication status comes from the latest observed decision, using decision date and ID as tie-breakers, and stays unknown when no decision exists. Unit checks cover author shapes, bound user values and schema qualification; real PostgreSQL -corpus checks remain pending the migration. +the real Core detail and homepage HTTP routes have returned 200. +The frontend has also rendered the actual 3,526,999-paper corpus through a +loopback SSH tunnel: eight rows, matching detail title/source, desktop and +mobile layout, no page exceptions or horizontal overflow in that check. +Author search returned 146 records in 0.336 seconds. After paging/author fixes, category `cs.AR` returned 9,457 records in 4.906 +seconds, `CartoonGAN` returned three in 0.332 seconds, and the multi-term search +`attention is all you need` returned 8,653 in 6.325 seconds. These were sequential +manual HTTP checks, not a load test. Very broad text searches can still exceed +the eight-second budget and need further performance work. This is not a +production deployment or an authentication acceptance result. The existing `deploy/check-schema.sh` also loads the synthetic fixture in `internal/research/testdata/corpus.sql` inside its disposable PostgreSQL @@ -85,3 +107,21 @@ integration checks execute real SQL for filtering, stable paging, nullable content, source/tag/identifier joins, latest decision tie-breakers, permission denial and SQL-backed HTTP routes. They never connect to the shared migration instance and do not establish full-corpus performance or live-data correctness. + +## Text storage compatibility (repair 4e62710) + +The Paper Atlas repair uses `nul-soh-v1`: NUL is stored as SOH/SOH, and a +literal SOH as SOH/STX, preserving binary text ordering. Core reads the schema +comment `paper-atlas:text-codec=nul-soh-v1` through `pg_catalog` at startup, +restores SQL text before JSON parsing/API responses, and encodes bound search +terms. An absent/unrelated schema comment keeps legacy text unchanged; an +unsupported marked codec or malformed encoded text fails explicitly. + +Schema comments are public metadata, so consumers need no privileges on the +private migration controller. The research role still needs only schema USAGE +and table SELECT. The codec adaptation remains local without CI or branch pushes; +the published schema marker has been verified with the runtime reader. +All research tests passed manually against disposable PostgreSQL on the +confirmed ECS, including the encoded SQL fixture for schema detection, raw TEXT restoration, +JSON author/metadata decoding, search and the HTTP round trip. Passing isolated +fixtures does not establish full-corpus API performance. diff --git a/internal/research/http.go b/internal/research/http.go index 3b339b6..9f86ac1 100644 --- a/internal/research/http.go +++ b/internal/research/http.go @@ -198,10 +198,11 @@ func (s *service) serve(w http.ResponseWriter, r *http.Request) { reply(w, 200, result) return } + var state interface{ SQLState() string } switch { case errors.Is(err, sql.ErrNoRows): reject(w, 404, "not_found") - case errors.Is(ctx.Err(), context.DeadlineExceeded), errors.Is(err, context.DeadlineExceeded): + case errors.Is(ctx.Err(), context.DeadlineExceeded), errors.Is(err, context.DeadlineExceeded), errors.As(err, &state) && state.SQLState() == "57014": reject(w, 504, "research_timeout") default: reject(w, 503, "research_unavailable") diff --git a/internal/research/http_test.go b/internal/research/http_test.go index 47573cd..b9267fc 100644 --- a/internal/research/http_test.go +++ b/internal/research/http_test.go @@ -100,6 +100,11 @@ func TestQueryDefaultsAndUnicode(t *testing.T) { } } +type databaseFailure string + +func (failure databaseFailure) Error() string { return "private database error" } +func (failure databaseFailure) SQLState() string { return string(failure) } + func TestStorageErrorsAreRedacted(t *testing.T) { for _, test := range []struct { err error @@ -108,6 +113,8 @@ func TestStorageErrorsAreRedacted(t *testing.T) { }{ {sql.ErrNoRows, 404, "not_found"}, {context.DeadlineExceeded, 504, "research_timeout"}, + {errors.Join(databaseFailure("57014"), errors.New("private statement")), 504, "research_timeout"}, + {databaseFailure("42501"), 503, "research_unavailable"}, {errors.New("postgres://private-user:private-password@db/corpus"), 503, "research_unavailable"}, } { mux := http.NewServeMux() diff --git a/internal/research/postgres.go b/internal/research/postgres.go index 72e6ed7..7198dc2 100644 --- a/internal/research/postgres.go +++ b/internal/research/postgres.go @@ -15,8 +15,10 @@ import ( // and BIGINT and creates the English title/abstract GIN expression below. // No schema creation, indexing or other writes belong to this repository. type PostgresRepository struct { - db *sql.DB - prefix string + db *sql.DB + prefix string + schema string + encoded bool } var schemaName = regexp.MustCompile(`^[a-z][a-z0-9_]{0,35}$`) @@ -25,7 +27,7 @@ func NewPostgresRepository(db *sql.DB, schema string) (*PostgresRepository, erro if db == nil || !schemaName.MatchString(schema) { return nil, errors.New("invalid research database configuration") } - return &PostgresRepository{db: db, prefix: `"` + schema + `".`}, nil + return &PostgresRepository{db: db, prefix: `"` + schema + `".`, schema: schema}, nil } func (r *PostgresRepository) query(value string) string { @@ -39,7 +41,7 @@ func (r *PostgresRepository) begin(ctx context.Context) (*sql.Tx, error) { } _, err = tx.ExecContext(ctx, `SELECT set_config('statement_timeout','8000',true), set_config('lock_timeout','1000',true), set_config('work_mem','2MB',true), - set_config('max_parallel_workers_per_gather','0',true)`) + set_config('max_parallel_workers_per_gather','0',true), set_config('jit','off',true)`) if err != nil { _ = tx.Rollback() return nil, err @@ -55,6 +57,16 @@ func (r *PostgresRepository) Check(ctx context.Context) error { return err } defer tx.Rollback() + var annotation sql.NullString + err = tx.QueryRowContext(ctx, `SELECT pg_catalog.obj_description(oid,'pg_namespace') + FROM pg_catalog.pg_namespace WHERE nspname=$1`, r.schema).Scan(&annotation) + if err != nil { + return err + } + encoded, err := encodedSchema(annotation) + if err != nil { + return err + } rows, err := tx.QueryContext(ctx, r.query(`SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published,p.title_key, r.external_id,r.url,r.metadata_json,so.name,i.scheme,i.value,t.namespace,t.value, s.id,e.year,e.track,v.acronym,d.status,d.evidence_url,d.observed_at,d.decision_date @@ -69,7 +81,11 @@ func (r *PostgresRepository) Check(ctx context.Context) error { if err = rows.Close(); err != nil { return err } - return tx.Commit() + if err = tx.Commit(); err != nil { + return err + } + r.encoded = encoded + return nil } func whereClause(filters Filters) (string, []any) { @@ -98,12 +114,12 @@ func whereClause(filters Filters) (string, []any) { type scanner interface{ Scan(...any) error } -func scanPaper(row scanner) (Paper, error) { +func (r *PostgresRepository) scanPaper(row scanner) (Paper, error) { p := Paper{Authors: []string{}, Topics: []string{}, Sources: []Source{}} var id int64 var abstract, published sql.NullString var authors string - if err := row.Scan(&id, &p.Title, &abstract, &authors, &published); err != nil { + if err := row.Scan(&id, r.text(&p.Title), r.text(&abstract), r.text(&authors), r.text(&published)); err != nil { return p, err } p.ID = strconv.FormatInt(id, 10) @@ -133,9 +149,14 @@ func decodeAuthors(raw string) ([]string, error) { continue } var tuple []string - if json.Unmarshal(entry, &tuple) != nil || len(tuple) == 0 || len(tuple) > 3 { + if json.Unmarshal(entry, &tuple) != nil || len(tuple) == 0 || len(tuple) > 4 { return nil, errors.New("unsupported stored author") } + // arXiv may append an affiliation after family, given and suffix. + // It is source metadata, not part of the display name. + if len(tuple) == 4 { + tuple = tuple[:3] + } if len(tuple) >= 2 { tuple[0], tuple[1] = tuple[1], tuple[0] } @@ -159,6 +180,10 @@ func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, e return result, err } defer tx.Rollback() + if r.encoded { + filters.Query = encodeText(filters.Query) + filters.Topic = encodeText(filters.Topic) + } where, args := whereClause(filters) if err = tx.QueryRowContext(ctx, r.query(`SELECT count(*) FROM {schema}papers p`+where), args...).Scan(&result.Total); err != nil { return result, err @@ -168,14 +193,33 @@ func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, e order = "p.title_key,p.id" } args = append(args, filters.PageSize, (filters.Page-1)*filters.PageSize) - query := fmt.Sprintf(`SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published - FROM {schema}papers p%s ORDER BY %s LIMIT $%d OFFSET $%d`, where, order, len(args)-1, len(args)) + // Select IDs before loading potentially TOASTed titles/abstracts. With + // sparse categories, selecting full rows here can make the planner walk + // hundreds of thousands of wide heap rows before finding eight matches. + candidateQuery := `SELECT p.id FROM {schema}papers p` + candidateOrder := order + prefix := "" + if where != "" { + columns := "p.id" + if filters.Sort == "title" { + columns += ",p.title_key" + } + // Apply search before LIMIT/ORDER BY so a sparse term cannot turn + // into a backward heap scan that recomputes vectors for millions. + prefix = fmt.Sprintf(`matches AS MATERIALIZED (SELECT %s FROM {schema}papers p%s), `, columns, where) + candidateQuery = "SELECT id FROM matches" + candidateOrder = strings.ReplaceAll(order, "p.", "") + } + query := fmt.Sprintf(`WITH %spage_ids AS MATERIALIZED ( + %s ORDER BY %s LIMIT $%d OFFSET $%d + ) SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published + FROM page_ids JOIN {schema}papers p ON p.id=page_ids.id ORDER BY %s`, prefix, candidateQuery, candidateOrder, len(args)-1, len(args), order) rows, err := tx.QueryContext(ctx, r.query(query), args...) if err != nil { return result, err } for rows.Next() { - p, err := scanPaper(rows) + p, err := r.scanPaper(rows) if err != nil { _ = rows.Close() return result, err @@ -196,7 +240,7 @@ func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, e } for rows.Next() { var topic string - if err = rows.Scan(&topic); err != nil { + if err = rows.Scan(r.text(&topic)); err != nil { _ = rows.Close() return result, err } @@ -216,7 +260,7 @@ func (r *PostgresRepository) Get(ctx context.Context, id int64) (Paper, error) { return Paper{}, err } defer tx.Rollback() - p, err := scanPaper(tx.QueryRowContext(ctx, r.query(`SELECT id,title,abstract,authors_json,first_published FROM {schema}papers WHERE id=$1`), id)) + p, err := r.scanPaper(tx.QueryRowContext(ctx, r.query(`SELECT id,title,abstract,authors_json,first_published FROM {schema}papers WHERE id=$1`), id)) if err != nil { return p, err } @@ -252,7 +296,7 @@ func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Pa for rows.Next() { var id int64 var topic string - if err = rows.Scan(&id, &topic); err != nil { + if err = rows.Scan(&id, r.text(&topic)); err != nil { _ = rows.Close() return err } @@ -273,7 +317,7 @@ func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Pa var id int64 var source Source var metadata string - if err = rows.Scan(&id, &source.Label, &source.ExternalID, &source.URL, &metadata); err != nil { + if err = rows.Scan(&id, r.text(&source.Label), r.text(&source.ExternalID), r.text(&source.URL), r.text(&metadata)); err != nil { _ = rows.Close() return err } @@ -309,7 +353,7 @@ func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Pa for rows.Next() { var id int64 var value string - if err = rows.Scan(&id, &value); err != nil { + if err = rows.Scan(&id, r.text(&value)); err != nil { _ = rows.Close() return err } @@ -333,7 +377,7 @@ func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Pa for rows.Next() { var id int64 var publication Publication - if err = rows.Scan(&id, &publication.Venue, &publication.Year, &publication.Track, &publication.Status, &publication.EvidenceURL); err != nil { + if err = rows.Scan(&id, r.text(&publication.Venue), &publication.Year, r.text(&publication.Track), r.text(&publication.Status), r.text(&publication.EvidenceURL)); err != nil { _ = rows.Close() return err } diff --git a/internal/research/postgres_integration_test.go b/internal/research/postgres_integration_test.go index e93c097..65c999b 100644 --- a/internal/research/postgres_integration_test.go +++ b/internal/research/postgres_integration_test.go @@ -102,7 +102,7 @@ func TestPostgresMetadataPreservesEvidence(t *testing.T) { if err != nil { t.Fatal(err) } - if len(p.Sources) != 2 || p.DOI == nil || *p.DOI != "10.fixture/other" || len(p.Publications) != 1 || p.Publications[0].Status != "unknown" || p.Publications[0].EvidenceURL != "" { + if !reflect.DeepEqual(p.Authors, []string{"Carla Network"}) || len(p.Sources) != 2 || p.DOI == nil || *p.DOI != "10.fixture/other" || len(p.Publications) != 1 || p.Publications[0].Status != "unknown" || p.Publications[0].EvidenceURL != "" { t.Fatalf("unknown decision/source mapping: %+v", p) } p, err = r.Get(ctx, 2) @@ -120,16 +120,19 @@ func TestPostgresCorpusIsReadOnlyAndHTTPWorks(t *testing.T) { if err != nil { t.Fatal(err) } - var readOnly, isolation string + var readOnly, isolation, jit string if err = tx.QueryRowContext(ctx, "SHOW transaction_read_only").Scan(&readOnly); err != nil { t.Fatal(err) } if err = tx.QueryRowContext(ctx, "SHOW transaction_isolation").Scan(&isolation); err != nil { t.Fatal(err) } + if err = tx.QueryRowContext(ctx, "SHOW jit").Scan(&jit); err != nil { + t.Fatal(err) + } _ = tx.Rollback() - if readOnly != "on" || isolation != "repeatable read" { - t.Fatalf("unsafe read transaction: %s %s", readOnly, isolation) + if readOnly != "on" || isolation != "repeatable read" || jit != "off" { + t.Fatalf("unexpected read transaction settings: %s %s jit=%s", readOnly, isolation, jit) } _, err = r.db.ExecContext(ctx, `UPDATE research_fixture.papers SET title='unexpected write' WHERE id=1`) var state interface{ SQLState() string } @@ -152,3 +155,32 @@ func TestPostgresCorpusIsReadOnlyAndHTTPWorks(t *testing.T) { t.Fatalf("missing HTTP status %d", w.Code) } } + +func TestPostgresPublishedTextCodec(t *testing.T) { + legacy, ctx := integrationRepository(t) + r, err := NewPostgresRepository(legacy.db, "research_encoded_fixture") + if err != nil { + t.Fatal(err) + } + if err = r.Check(ctx); err != nil { + t.Fatal(err) + } + p, err := r.Get(ctx, 1) + if err != nil { + t.Fatal(err) + } + if p.Title != "Graph\x00original\x01" || p.Abstract == nil || *p.Abstract != "Text\x00tail" || !reflect.DeepEqual(p.Authors, []string{"Ada\x00Smith", "Group\x01Name"}) || p.Comments == nil || *p.Comments != "Original\x00comment" { + t.Fatalf("published text was not restored: %+v", p) + } + mux := http.NewServeMux() + Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"}) + w := request(mux, "GET", "/projects/thesisly/research/papers/1") + var decoded Paper + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &decoded) != nil || decoded.Title != p.Title || decoded.Abstract == nil || *decoded.Abstract != *p.Abstract { + t.Fatalf("HTTP lost original control characters: %d %s", w.Code, w.Body.String()) + } + page, err := r.List(ctx, Filters{Page: 1, PageSize: 8, Query: "graph"}) + if err != nil || page.Total != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title { + t.Fatalf("encoded corpus search: %+v %v", page, err) + } +} diff --git a/internal/research/postgres_test.go b/internal/research/postgres_test.go index 48287bd..e1bffb8 100644 --- a/internal/research/postgres_test.go +++ b/internal/research/postgres_test.go @@ -14,6 +14,7 @@ func TestStoredAuthorShapes(t *testing.T) { want []string }{ {`[]`, []string{}}, + {`[["Buhrman","Harry","","CWI"],["Family","Given","Jr.","University"]]`, []string{"Harry Buhrman", "Given Family Jr."}}, {`["The BABAR Collaboration","B. Aubert"]`, []string{"The BABAR Collaboration", "B. Aubert"}}, {`[["Balázs","C.",""],["Berger","E. L.",""],["Smith","Ada","Jr."]]`, []string{"C. Balázs", "E. L. Berger", "Ada Smith Jr."}}, {`[" 张三 ",["Only family"],["","Only given",""],""]`, []string{"张三", "Only family", "Only given"}}, @@ -27,7 +28,7 @@ func TestStoredAuthorShapes(t *testing.T) { t.Fatal("missing authors must remain an array") } } - for _, raw := range []string{`broken`, `{"unexpected":"object"}`, `[42]`, `[["a","b","c","d"]]`} { + for _, raw := range []string{`broken`, `{"unexpected":"object"}`, `[42]`, `[["a","b","c","d","e"]]`} { if _, err := decodeAuthors(raw); err == nil { t.Fatalf("accepted unsupported authors %s", raw) } diff --git a/internal/research/testdata/corpus.sql b/internal/research/testdata/corpus.sql index 48323aa..3bbd5ac 100644 --- a/internal/research/testdata/corpus.sql +++ b/internal/research/testdata/corpus.sql @@ -33,7 +33,7 @@ CREATE INDEX decisions_submission_order ON research_fixture.decisions(submission INSERT INTO research_fixture.papers VALUES (1,'Graph & learning ','Neural graph models preserve original abstracts.','graph learning','[["Smith","Ada",""],"The Research Group"]','2024-01-02'), (2,'Alpha topology',NULL,'alpha topology','["Bruno Park"]',NULL), - (3,'Graph systems','Graph systems and data.','graph systems','["Carla Network"]','2023-01-01'), + (3,'Graph systems','Graph systems and data.','graph systems','[["Network","Carla","","Fixture University"]]','2023-01-01'), (4,'Zebra methods','Evidence and uncertainty.','zebra methods','[]','2022-01-01'), (5,'Alpha Topology','Another independent record.','alpha topology','["Dana Other"]','2025-01-01'); INSERT INTO research_fixture.sources VALUES(10,'arxiv'),(20,'pmlr'); @@ -57,3 +57,25 @@ CREATE ROLE foundry_research_fixture LOGIN; REVOKE ALL ON SCHEMA research_fixture FROM PUBLIC; GRANT USAGE ON SCHEMA research_fixture TO foundry_research_fixture; GRANT SELECT ON ALL TABLES IN SCHEMA research_fixture TO foundry_research_fixture; + +-- Separate encoded schema mirrors the published Paper Atlas nul-soh-v1 format. +CREATE SCHEMA research_encoded_fixture; +COMMENT ON SCHEMA research_encoded_fixture IS 'paper-atlas:text-codec=nul-soh-v1'; +DO $$ +DECLARE item text; +BEGIN + FOREACH item IN ARRAY ARRAY['papers','sources','source_records','identifiers','tags','paper_tags','venues','editions','submissions','decisions'] LOOP + EXECUTE format('CREATE TABLE research_encoded_fixture.%I (LIKE research_fixture.%I INCLUDING ALL)',item,item); + EXECUTE format('INSERT INTO research_encoded_fixture.%I SELECT * FROM research_fixture.%I',item,item); + END LOOP; +END $$; +UPDATE research_encoded_fixture.papers +SET title='Graph'||chr(1)||chr(1)||'original'||chr(1)||chr(2), + abstract='Text'||chr(1)||chr(1)||'tail', + authors_json='["Ada\u0000Smith","Group\u0001Name"]' +WHERE id=1; +UPDATE research_encoded_fixture.source_records +SET metadata_json='{"comments":"Original\u0000comment"}' WHERE id=101; +REVOKE ALL ON SCHEMA research_encoded_fixture FROM PUBLIC; +GRANT USAGE ON SCHEMA research_encoded_fixture TO foundry_research_fixture; +GRANT SELECT ON ALL TABLES IN SCHEMA research_encoded_fixture TO foundry_research_fixture; diff --git a/internal/research/text.go b/internal/research/text.go new file mode 100644 index 0000000..abebbbb --- /dev/null +++ b/internal/research/text.go @@ -0,0 +1,88 @@ +package research + +import ( + "database/sql" + "errors" + "strings" +) + +// Matches Paper Atlas repair 4e62710's nul-soh-v1 protocol: NUL -> SOH/SOH and literal +// SOH -> SOH/STX. All other bytes retain their original binary ordering. +const textCodec = "nul-soh-v1" + +func encodedSchema(annotation sql.NullString) (bool, error) { + const prefix = "paper-atlas:text-codec=" + if !annotation.Valid || !strings.HasPrefix(annotation.String, prefix) { + return false, nil + } + if strings.TrimPrefix(annotation.String, prefix) != textCodec { + return false, errors.New("unsupported research text codec") + } + return true, nil +} + +func encodeText(value string) string { + return strings.ReplaceAll(strings.ReplaceAll(value, "\x01", "\x01\x02"), "\x00", "\x01\x01") +} + +func decodeText(value string) (string, error) { + if !strings.ContainsRune(value, '\x01') { + return value, nil + } + var result strings.Builder + for { + position := strings.IndexByte(value, 1) + if position < 0 { + result.WriteString(value) + return result.String(), nil + } + result.WriteString(value[:position]) + if position+1 >= len(value) { + return "", errors.New("invalid research text escape") + } + switch value[position+1] { + case 1: + result.WriteByte(0) + case 2: + result.WriteByte(1) + default: + return "", errors.New("invalid research text escape") + } + value = value[position+2:] + } +} + +type textColumn struct { + encoded bool + target any +} + +func (column textColumn) Scan(source any) error { + var value sql.NullString + if err := value.Scan(source); err != nil { + return err + } + if value.Valid && column.encoded { + var err error + value.String, err = decodeText(value.String) + if err != nil { + return err + } + } + switch target := column.target.(type) { + case *string: + if !value.Valid { + return errors.New("unexpected null research text") + } + *target = value.String + case *sql.NullString: + *target = value + default: + return errors.New("unsupported research text destination") + } + return nil +} + +func (r *PostgresRepository) text(target any) sql.Scanner { + return textColumn{encoded: r.encoded, target: target} +} diff --git a/internal/research/text_test.go b/internal/research/text_test.go new file mode 100644 index 0000000..a972ef2 --- /dev/null +++ b/internal/research/text_test.go @@ -0,0 +1,69 @@ +package research + +import ( + "database/sql" + "strings" + "testing" +) + +func TestCorpusTextCodecRoundTripAndOrder(t *testing.T) { + values := []string{"", "\x00", "\x00\x01", "\x01", "\x01\x02", "\x02", "A", "A\x00", "A\x01", "Ada 张三 Ω", "Z\x00\x01\x02尾"} + for _, value := range values { + encoded := encodeText(value) + decoded, err := decodeText(encoded) + if err != nil || decoded != value || strings.ContainsRune(encoded, 0) { + t.Fatalf("round trip failed for %q", value) + } + for _, other := range values { + if strings.Compare(value, other) != strings.Compare(encoded, encodeText(other)) { + t.Fatalf("binary ordering changed for %q %q", value, other) + } + } + } + for _, invalid := range []string{"\x01", "prefix\x01x", "\x01\x00"} { + if _, err := decodeText(invalid); err == nil { + t.Fatalf("accepted malformed escape %q", invalid) + } + } +} + +func TestCorpusTextColumnsRespectCodecAndNulls(t *testing.T) { + var required string + if err := (textColumn{target: &required}).Scan("\x01\x01"); err != nil || required != "\x01\x01" { + t.Fatal("legacy text was decoded") + } + if err := (textColumn{encoded: true, target: &required}).Scan([]byte("before\x01\x01after\x01\x02")); err != nil || required != "before\x00after\x01" { + t.Fatalf("encoded text did not decode: %q %v", required, err) + } + var nullable sql.NullString + if err := (textColumn{encoded: true, target: &nullable}).Scan(nil); err != nil || nullable.Valid { + t.Fatal("null changed into empty text") + } + if err := (textColumn{encoded: true, target: &nullable}).Scan("\x01\x01"); err != nil || !nullable.Valid || nullable.String != "\x00" { + t.Fatal("nullable text was not restored") + } + if err := (textColumn{encoded: true, target: &required}).Scan(nil); err == nil { + t.Fatal("accepted null for required text") + } + if err := (textColumn{encoded: true, target: &required}).Scan("\x01x"); err == nil { + t.Fatal("accepted corrupt encoded text") + } +} + +func TestCorpusSchemaCodecMarker(t *testing.T) { + for _, annotation := range []sql.NullString{{}, {String: "unrelated schema description", Valid: true}} { + encoded, err := encodedSchema(annotation) + if encoded || err != nil { + t.Fatal("legacy schema changed encoding") + } + } + encoded, err := encodedSchema(sql.NullString{String: "paper-atlas:text-codec=nul-soh-v1", Valid: true}) + if !encoded || err != nil { + t.Fatal("published codec was not recognized") + } + for _, name := range []string{"unknown", "", "legacy"} { + if _, err := encodedSchema(sql.NullString{String: "paper-atlas:text-codec=" + name, Valid: true}); err == nil { + t.Fatal("unknown codec was accepted") + } + } +}