diff --git a/.env.example b/.env.example index c6447fe..f79d8e6 100644 --- a/.env.example +++ b/.env.example @@ -16,3 +16,10 @@ SMTP_FROM=noreply@notice.entroia.com # Optional Sign in with Slack. SLACK_CLIENT_ID=replace-me SLACK_CLIENT_SECRET=replace-me + +# Optional shared paper corpus; enable only after PostgreSQL publication. +# Store the real read-only connection string in the protected server research.env. +FOUNDRY_RESEARCH_DATABASE_URL= +FOUNDRY_RESEARCH_SCHEMA=research +# Deployment migration container; independent of database/schema names. +FOUNDRY_POSTGRES_CONTAINER=foundry-postgres diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index f3ad8e9..c666327 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -74,6 +74,7 @@ jobs: env: PUBLIC_ORIGIN: https://api.example.com DATABASE_URL: postgres://foundry:fixture@127.0.0.1:5432/foundry + FOUNDRY_RESEARCH_DATABASE_URL: '' GOOGLE_CLIENT_ID: fixture GOOGLE_CLIENT_SECRET: fixture FOUNDRY_PROJECTS: '{"notes":"http://localhost:3000/auth/callback"}' diff --git a/README.md b/README.md index 1ea1502..2253a25 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,10 @@ 4. 由 Gitea Actions 在 `ces-control` 上使用 Go 1.25 构建并检查。部署工作流仅处理 `main` 上影响后端或部署的改动,将二进制启动在服务器 Docker 私网,由 Traefik 提供 HTTPS。通过 `GET /healthz` 检查进程,通过 `GET /readyz` 检查认证库。 5. 代理避免记录 Google 回调的查询参数及认证请求头。设置入口限流,并定时删除三张临时表(`oauth_attempts`、`login_grants`、`project_sessions`)中 `expires_at <= now()` 的记录。 +## 共享论文库(Thesisly) + +可选只读接入见[共享论文 API](docs/research-api.md)。Thesisly 使用同一后端的 `/projects/{project}/research/papers` 列表、搜索和详情;只有配置服务器上的 `FOUNDRY_RESEARCH_DATABASE_URL` 时才启用。连接串保存在受保护的 `research.env`,不放入前端或 Actions Variables。当前仍等待 Paper Atlas 全量 PostgreSQL 迁移完成及真实数据验收,尚未启用线上论文接口。 + ## Actions 部署配置 在仓库的 Actions Secrets 配置 `GOOGLE_CLIENT_ID` 和 `GOOGLE_CLIENT_SECRET`,在 Actions Variables 配置 `FOUNDRY_PROJECTS`。变量值为 JSON 对象,例如 `{"prompt":"http://localhost:17321/auth/callback"}`。部署读取服务器上预先配置的 `/opt/endoleap-infra/foundry/database.env`,OAuth 值由 Actions 在启动容器时注入。 diff --git a/cmd/foundry/main.go b/cmd/foundry/main.go index 5b3e2a2..32d4196 100644 --- a/cmd/foundry/main.go +++ b/cmd/foundry/main.go @@ -13,6 +13,7 @@ import ( "gitea.entroia.com/Zayden-Jung/foundry-core/internal/auth" "gitea.entroia.com/Zayden-Jung/foundry-core/internal/codex" "gitea.entroia.com/Zayden-Jung/foundry-core/internal/platform" + "gitea.entroia.com/Zayden-Jung/foundry-core/internal/research" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/stdlib" ) @@ -61,6 +62,33 @@ func main() { log.Fatal(err) } mux := http.NewServeMux() + var papers research.Repository + if researchDSN := os.Getenv("FOUNDRY_RESEARCH_DATABASE_URL"); researchDSN != "" { + researchConfig, err := pgx.ParseConfig(researchDSN) + if err != nil { + log.Fatal("Invalid research database configuration") + } + researchConfig.RuntimeParams["default_transaction_read_only"] = "on" + researchConfig.RuntimeParams["application_name"] = "foundry-research" + researchDB := stdlib.OpenDB(*researchConfig) + defer researchDB.Close() + researchDB.SetMaxOpenConns(2) + researchDB.SetMaxIdleConns(2) + researchDB.SetConnMaxIdleTime(5 * time.Minute) + researchDB.SetConnMaxLifetime(30 * time.Minute) + repository, err := research.NewPostgresRepository(researchDB, env("FOUNDRY_RESEARCH_SCHEMA", "research")) + if err != nil { + log.Fatal("Invalid research database configuration") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err = repository.Check(ctx) + cancel() + if err != nil { + log.Fatal("Research database is unavailable or not published") + } + papers = repository + } + research.Register(mux, papers, callbacks) if err := codex.Register(mux, callbacks, os.Getenv("FOUNDRY_CODEX_SOCKET"), os.Getenv("FOUNDRY_CODEX_GRANTS")); err != nil { log.Fatal(err) } diff --git a/deploy/check-schema.sh b/deploy/check-schema.sh index ab87d48..941baa7 100644 --- a/deploy/check-schema.sh +++ b/deploy/check-schema.sh @@ -67,12 +67,14 @@ END $$; ROLLBACK; SQL printf 'Migration preserved existing logins and enforced project isolation.\n' -for package in platform auth; do +psql < internal/research/testdata/corpus.sql +for package in platform auth research; do CGO_ENABLED=0 go test -mod=readonly -c -o "bin/$package-tests" "./internal/$package" docker run --rm --network "container:$container" --user 1000:1000 \ --read-only --cap-drop ALL --security-opt no-new-privileges \ --tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m,mode=1777 \ --mount "type=bind,source=$(pwd)/bin/$package-tests,target=/opt/integration-tests,readonly" \ -e PLATFORM_TEST_DATABASE_URL=postgres://foundry_fixture@127.0.0.1:5432/postgres?sslmode=disable \ + -e RESEARCH_TEST_DATABASE_URL=postgres://foundry_research_fixture@127.0.0.1:5432/postgres?sslmode=disable \ --entrypoint /opt/integration-tests postgres:17-alpine -test.v done diff --git a/deploy/migrate.py b/deploy/migrate.py index 7cc46ff..d1ff616 100644 --- a/deploy/migrate.py +++ b/deploy/migrate.py @@ -2,12 +2,13 @@ """Apply checked migrations atomically from the trusted deployment runner.""" import argparse import hashlib +import os from pathlib import Path import re import subprocess parser = argparse.ArgumentParser() -parser.add_argument("--container", default="endoleap-dev-postgres-1") +parser.add_argument("--container", default=os.environ.get("FOUNDRY_POSTGRES_CONTAINER", "foundry-postgres")) parser.add_argument("--database", default="foundry_auth") parser.add_argument("--role", default="foundry") args = parser.parse_args() diff --git a/deploy/release.sh b/deploy/release.sh index af98e99..b75658c 100644 --- a/deploy/release.sh +++ b/deploy/release.sh @@ -53,6 +53,10 @@ mail_env=() if [[ -s "$foundry/mail.env" ]]; then mail_env=(--env-file "$foundry/mail.env") fi +research_env=() +if [[ -s "$foundry/research.env" ]]; then + research_env=(--env-file "$foundry/research.env") +fi slack_env=() if [[ -n "${SLACK_CLIENT_ID:-}" || -n "${SLACK_CLIENT_SECRET:-}" ]]; then for name in SLACK_CLIENT_ID SLACK_CLIENT_SECRET; do @@ -118,7 +122,7 @@ if [[ -d "$codex_dir" ]]; then ) fi -docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack_env[@]}" --network "$network" \ +docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack_env[@]}" "${research_env[@]}" --network "$network" \ --env-file "$foundry/database.env" -e HOST -e PORT -e PUBLIC_ORIGIN \ -e GOOGLE_CLIENT_ID -e GOOGLE_CLIENT_SECRET -e FOUNDRY_PROJECTS -e FOUNDRY_ONBOARDING \ --user 1000:1000 --cap-drop ALL --security-opt no-new-privileges \ diff --git a/deploy/research/query-indexes.sql b/deploy/research/query-indexes.sql new file mode 100644 index 0000000..314cf06 --- /dev/null +++ b/deploy/research/query-indexes.sql @@ -0,0 +1,14 @@ +-- Manual operator step against the published Paper Atlas schema, outside a +-- transaction. Core's SELECT-only runtime role must never execute this file. +-- These indexes support the actual author search and exact category filter; +-- the migration already owns the title/abstract English GIN index. +SET lock_timeout = '2s'; +SET statement_timeout = '20min'; +SET maintenance_work_mem = '32MB'; +SET max_parallel_maintenance_workers = 0; +CREATE INDEX CONCURRENTLY IF NOT EXISTS paper_tags_tag_paper + ON research.paper_tags(tag_id,paper_id); +CREATE INDEX CONCURRENTLY IF NOT EXISTS papers_authors_search_gin + ON research.papers USING gin(to_tsvector('simple'::regconfig,authors_json)); +-- After an interrupted concurrent build, inspect pg_index.indisvalid before +-- reuse. IF NOT EXISTS alone does not repair an invalid index. diff --git a/docs/research-api.md b/docs/research-api.md new file mode 100644 index 0000000..08217ed --- /dev/null +++ b/docs/research-api.md @@ -0,0 +1,145 @@ +# Shared research API + +This work belongs to Thesisly 研迹 and uses Foundry Core rather than a separate +application backend. The corpus is public; account sessions and future private +collections remain project scoped. + +## Current implementation boundary + +`internal/research` implements the HTTP contract and a PostgreSQL repository +based on the fixed Paper Atlas `a3b677b` schema (TEXT/BIGINT columns, native +English title/abstract search index). `cmd/foundry` enables it only when +`FOUNDRY_RESEARCH_DATABASE_URL` is configured. The Paper Atlas migration is now published: 21 tables passed its independent +count/content-hash acceptance. A dedicated `foundry_research_reader` role has +SELECT on the ten tables consumed here, schema USAGE, and read-only defaults. +Its connection is stored only in the protected server file below. +The existing SQLite implementation was withdrawn. Real-corpus integration has +been checked through a loopback Core and the local frontend. Formal production +release still requires the user to merge and choose to deploy; this branch is +not deployed by these checks. Very broad queries remain subject to the budget +and performance limitation documented below. +Do not add `FOUNDRY_RESEARCH_DATABASE`. + +The server-side deployment may load a protected +`/opt/endoleap-infra/foundry/research.env` containing +`FOUNDRY_RESEARCH_DATABASE_URL` and `FOUNDRY_RESEARCH_SCHEMA` (default `research`). +Keep the connection string out of Git, Actions variables, browser configuration +and command arguments. Use a dedicated role with schema USAGE and table SELECT +only. Core uses a separate two-connection pool, read-only connection defaults and +read-only transactions. Each research transaction disables JIT locally: the +measured count plan spent about 2.85 seconds compiling JIT functions, and its +end-to-end execution exceeded eight seconds. This does not alter shared +PostgreSQL settings or replace exact counts with estimates. Startup checks the published schema/SELECT permissions; +an explicitly enabled but invalid connection fails candidate readiness. + +Deployment migration defaults now follow the shared `foundry-postgres` +container name. `FOUNDRY_POSTGRES_CONTAINER` or the existing `--container` option +can select a different container without changing database/table names. + +## Routes + +- `GET /projects/{project}/research/papers` +- `GET /projects/{project}/research/papers/{id}` + +Only configured Foundry projects are accepted. Reading the public corpus does +not require a session; these routes do not expose private data or accept writes. + +List parameters are `q` (200 Unicode characters, 20 words), `searchIn` +(`text` or `author`), `topic` (an exact arXiv category), `sort` (`recent` or +`title`), `page` (1–1000), and `pageSize` (1–50, default 8). Unknown, malformed +or repeated parameters return 400. `recent` means descending stable paper ID, +not publication date. Search terms must become bound query parameters, never +raw SQL or user-supplied full-text syntax. + +Title/abstract search uses the migration's English vector with +`plainto_tsquery`; author search uses simple-language terms over the stored +author JSON. Both treat input as terms, never full-text operators. Initial full-corpus measurements showed eight-second timeouts for text, +author and topic counts; the unfiltered count took 7.25 seconds. Execution plans +confirmed sequential scans for authors and category joins. +`deploy/research/query-indexes.sql` supplies operator-run concurrent indexes for +those two missing access paths. It is separate from Core startup and auth +migrations; do not grant DDL to the runtime reader. Both indexes completed on the confirmed ECS (category 35.4 seconds, author +436.47 seconds); measured end-to-end checks are documented below. + +List responses contain `items`, `total`, `topics`, `page`, and `pageSize`. +Each paper preserves original title/abstract, authors, source dates, topics and +source links. Publication decisions need source evidence; absent decisions stay +unknown. No invented repositories, metrics or generated research conclusions. + +The repository counts and reads each list in one read-only snapshot. +Paging first materializes matching IDs, then joins only that page to the wide +paper rows. Filtered searches first materialize matching IDs without an +ORDER BY/LIMIT, then sort/page those IDs, preventing sparse terms from choosing +a backward heap scan with repeated text-vector computation. Activity sampling showed that selecting full rows before LIMIT +caused sparse category queries to read large portions of the paper heap. The +HTTP layer permits two in-flight queries with an eight-second context budget. +Busy requests return 429 with `Retry-After: 1`; deadlines return 504; storage +errors return a generic 503 without database details. Missing papers return 404. + +## Validation + +`go test -race -count=1 ./internal/research` covers routing, query limits, +Unicode, anonymous reads, rejected mutations, project lookup, errors, saturation +and timeout slot release. Run compilation/tests only on the user-confirmed ECS +runner. These transport tests do not prove a database connection, full-corpus +query performance or production availability. + +The PostgreSQL repository batches source/tag/identifier/submission queries in +the same read-only repeatable-read transaction. Author tuples are displayed as +given name, family name and suffix; an optional fourth affiliation field +is excluded from the display name (observed in real arXiv records); null abstracts/dates remain null. Publication +status comes from the latest observed decision, using decision date and ID as +tie-breakers, and stays unknown when no decision exists. Unit checks cover +author shapes, bound user values and schema qualification. The real Core detail +and homepage HTTP routes have returned 200. +The frontend has also rendered the actual 3,526,999-paper corpus through a +loopback SSH tunnel: eight rows, matching detail title/source, desktop and +mobile layout, no page exceptions or horizontal overflow in that check. +Author search returned 146 records in 0.336 seconds. After paging/author fixes, category `cs.AR` returned 9,457 records in 4.906 +seconds, `CartoonGAN` returned three in 0.332 seconds, and the multi-term search +`attention is all you need` returned 8,653 in 6.325 seconds. These were sequential +manual HTTP checks, not a load test. Very broad text searches can still exceed +the eight-second budget; clients retain search controls so users can narrow +the query or retry. This known limit is not an estimated count or sample +fallback. This is not a +production deployment or an authentication acceptance result. + +The existing `deploy/check-schema.sh` also loads the synthetic fixture in +`internal/research/testdata/corpus.sql` inside its disposable PostgreSQL +container (`--network none`, no host port). The research test process receives a +separate SELECT-only fixture role via `RESEARCH_TEST_DATABASE_URL`. These +integration checks execute real SQL for filtering, stable paging, nullable +content, source/tag/identifier joins, latest decision tie-breakers, permission +denial and SQL-backed HTTP routes. They never connect to the shared migration +instance and do not establish full-corpus performance or live-data correctness. + +## Text storage compatibility (repair 4e62710) + +The Paper Atlas repair uses `nul-soh-v1`: NUL is stored as SOH/SOH, and a +literal SOH as SOH/STX, preserving binary text ordering. Core reads the schema +comment `paper-atlas:text-codec=nul-soh-v1` through `pg_catalog` at startup, +restores SQL text before JSON parsing/API responses, and encodes bound search +terms. An absent/unrelated schema comment keeps legacy text unchanged; an +unsupported marked codec or malformed encoded text fails explicitly. + +Schema comments are public metadata, so consumers need no privileges on the +private migration controller. The research role still needs only schema USAGE +and table SELECT. The codec adaptation is on the review branch, pushed with [skip ci] under the +user's no-CI instruction; the published schema marker has been verified with +the runtime reader. +All research tests passed manually against disposable PostgreSQL on the +confirmed ECS, including the encoded SQL fixture for schema detection, raw TEXT restoration, +JSON author/metadata decoding, search and the HTTP round trip. Passing isolated +fixtures does not establish full-corpus API performance. + + +## Identity handoff + +The existing production Core received a configuration-only reload using its +unchanged release binary: the original projects were retained and thesisly was +registered. Both Google and Slack identity records were observed in this +project. The user independently confirmed each provider's real authorization, +reload/new-tab persistence and logout behavior. Browser control was unavailable, +so these browser results are attributed to user acceptance rather than an +automated real-account test. This does not deploy the research work branch; +mail delivery was not exercised. diff --git a/internal/research/http.go b/internal/research/http.go new file mode 100644 index 0000000..9f86ac1 --- /dev/null +++ b/internal/research/http.go @@ -0,0 +1,210 @@ +// Package research serves the shared, public paper corpus. Account data and +// private collections do not belong to these anonymous read-only endpoints. +package research + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type Source struct { + Label string `json:"label"` + URL string `json:"url"` + ExternalID string `json:"externalId"` +} + +type Publication struct { + Venue string `json:"venue"` + Year int `json:"year"` + Track string `json:"track"` + Status string `json:"status"` + EvidenceURL string `json:"evidenceUrl,omitempty"` +} + +type Paper struct { + ID string `json:"id"` + Title string `json:"title"` + Abstract *string `json:"abstract"` + Authors []string `json:"authors"` + PublishedAt *string `json:"publishedAt"` + Topics []string `json:"topics"` + Sources []Source `json:"sources"` + Publications []Publication `json:"publications,omitempty"` + DOI *string `json:"doi,omitempty"` + Journal *string `json:"journal,omitempty"` + Comments *string `json:"comments,omitempty"` +} + +type Filters struct { + Query string + SearchIn string + Topic string + Sort string + Page int + PageSize int +} + +type Page struct { + Items []Paper `json:"items"` + Total int64 `json:"total"` + Topics []string `json:"topics"` + Page int `json:"page"` + PageSize int `json:"pageSize"` +} + +// Repository implementations must use a read-only transaction for each List, +// so its count, topics and page describe the same database snapshot. +type Repository interface { + List(context.Context, Filters) (Page, error) + Get(context.Context, int64) (Paper, error) +} + +type service struct { + repository Repository + projects map[string]string + slots chan struct{} + timeout time.Duration +} + +// Register adds only GET endpoints; no credentials or database errors are sent +// to anonymous callers. A nil repository leaves the existing server unchanged. +func Register(mux *http.ServeMux, repository Repository, projects map[string]string) { + if repository == nil { + return + } + s := &service{repository: repository, projects: projects, slots: make(chan struct{}, 2), timeout: 8 * time.Second} + mux.HandleFunc("GET /projects/{project}/research/papers", s.serve) + mux.HandleFunc("GET /projects/{project}/research/papers/{id}", s.serve) +} + +func reply(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} + +func reject(w http.ResponseWriter, status int, code string) { + reply(w, status, map[string]string{"error": code}) +} + +var topicPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.-]{0,79}$`) +var positiveInteger = regexp.MustCompile(`^[1-9][0-9]*$`) + +func parseFilters(raw string) (Filters, error) { + f := Filters{SearchIn: "text", Sort: "recent", Page: 1, PageSize: 8} + values, err := url.ParseQuery(raw) + if err != nil { + return f, err + } + invalid := errors.New("invalid query") + for key, value := range values { + if len(value) != 1 { + return f, invalid + } + switch key { + case "q": + if !utf8.ValidString(value[0]) || utf8.RuneCountInString(value[0]) > 200 || len(strings.Fields(value[0])) > 20 || strings.ContainsRune(value[0], 0) { + return f, invalid + } + f.Query = strings.TrimSpace(value[0]) + case "searchIn": + if value[0] != "text" && value[0] != "author" { + return f, invalid + } + f.SearchIn = value[0] + case "topic": + if value[0] != "" && !topicPattern.MatchString(value[0]) { + return f, invalid + } + f.Topic = value[0] + case "sort": + if value[0] != "recent" && value[0] != "title" { + return f, invalid + } + f.Sort = value[0] + case "page", "pageSize": + if !positiveInteger.MatchString(value[0]) { + return f, invalid + } + n, err := strconv.Atoi(value[0]) + limit := 1000 + if key == "pageSize" { + limit = 50 + } + if err != nil || n > limit { + return f, invalid + } + if key == "page" { + f.Page = n + } else { + f.PageSize = n + } + default: + return f, invalid + } + } + return f, nil +} + +func (s *service) serve(w http.ResponseWriter, r *http.Request) { + if _, exists := s.projects[r.PathValue("project")]; !exists { + reject(w, 404, "not_found") + return + } + var filters Filters + var id int64 + var err error + if value := r.PathValue("id"); value != "" { + if !positiveInteger.MatchString(value) || r.URL.RawQuery != "" { + reject(w, 400, "invalid_request") + return + } + id, err = strconv.ParseInt(value, 10, 64) + } else { + filters, err = parseFilters(r.URL.RawQuery) + } + if err != nil { + reject(w, 400, "invalid_request") + return + } + select { + case s.slots <- struct{}{}: + defer func() { <-s.slots }() + default: + w.Header().Set("Retry-After", "1") + reject(w, 429, "research_busy") + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.timeout) + defer cancel() + var result any + if id != 0 { + result, err = s.repository.Get(ctx, id) + } else { + result, err = s.repository.List(ctx, filters) + } + if err == nil { + reply(w, 200, result) + return + } + var state interface{ SQLState() string } + switch { + case errors.Is(err, sql.ErrNoRows): + reject(w, 404, "not_found") + case errors.Is(ctx.Err(), context.DeadlineExceeded), errors.Is(err, context.DeadlineExceeded), errors.As(err, &state) && state.SQLState() == "57014": + reject(w, 504, "research_timeout") + default: + reject(w, 503, "research_unavailable") + } +} diff --git a/internal/research/http_test.go b/internal/research/http_test.go new file mode 100644 index 0000000..b9267fc --- /dev/null +++ b/internal/research/http_test.go @@ -0,0 +1,145 @@ +package research + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +type fixtureRepository struct { + filters Filters + id int64 + err error + wait bool + calls int +} + +func (f *fixtureRepository) List(ctx context.Context, filters Filters) (Page, error) { + f.calls++ + f.filters = filters + if f.wait { + <-ctx.Done() + return Page{}, ctx.Err() + } + return Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}, f.err +} + +func (f *fixtureRepository) Get(ctx context.Context, id int64) (Paper, error) { + f.calls++ + f.id = id + return Paper{ID: "42", Title: "Original title", Authors: []string{}, Topics: []string{}, Sources: []Source{}}, f.err +} + +func request(handler http.Handler, method, path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(method, path, nil)) + return w +} + +func TestAnonymousReadRoutes(t *testing.T) { + f := &fixtureRepository{} + mux := http.NewServeMux() + Register(mux, f, map[string]string{"thesisly": "http://localhost:17322/auth/callback"}) + w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&page=2&pageSize=10") + if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", Page: 2, PageSize: 10}) { + t.Fatalf("list: %d %+v", w.Code, f.filters) + } + var page Page + if err := json.Unmarshal(w.Body.Bytes(), &page); err != nil || page.Items == nil || page.Topics == nil { + t.Fatalf("invalid page: %s", w.Body.String()) + } + if w.Header().Get("Cache-Control") != "no-store" || w.Header().Get("Content-Type") != "application/json" { + t.Fatal("missing response headers") + } + w = request(mux, "GET", "/projects/thesisly/research/papers/42") + if w.Code != 200 || f.id != 42 || !strings.Contains(w.Body.String(), `"abstract":null`) { + t.Fatalf("detail: %d %s", w.Code, w.Body.String()) + } + if got := request(mux, "POST", "/projects/thesisly/research/papers").Code; got != 405 { + t.Fatalf("mutation status %d", got) + } + if got := request(mux, "GET", "/projects/other/research/papers").Code; got != 404 { + t.Fatalf("unknown project status %d", got) + } +} + +func TestInvalidQueriesNeverReachRepository(t *testing.T) { + queries := []string{"unknown=1", "q=x&q=y", "q=%ff", "q=%00", "q=%zz", "page=0", "page=1001", "page=-1", "page=1.5", "page=01", "pageSize=51", "pageSize=0", "pageSize=9999999999999999999999", "searchIn=sql", "sort=published", "topic=arxiv:cs.LG", "q=" + url.QueryEscape(strings.Repeat("字", 201)), "q=" + url.QueryEscape(strings.Repeat("word ", 21))} + for _, query := range queries { + t.Run(query, func(t *testing.T) { + f := &fixtureRepository{} + mux := http.NewServeMux() + Register(mux, f, map[string]string{"thesisly": "unused"}) + w := request(mux, "GET", "/projects/thesisly/research/papers?"+query) + if w.Code != 400 || f.calls != 0 { + t.Fatalf("status %d, calls %d", w.Code, f.calls) + } + }) + } + for _, path := range []string{"0", "-1", "1x", "9223372036854775808", "1?extra=1"} { + mux := http.NewServeMux() + f := &fixtureRepository{} + Register(mux, f, map[string]string{"thesisly": "unused"}) + if w := request(mux, "GET", "/projects/thesisly/research/papers/"+path); w.Code != 400 || f.calls != 0 { + t.Fatalf("invalid id %q: %d", path, w.Code) + } + } +} + +func TestQueryDefaultsAndUnicode(t *testing.T) { + f, err := parseFilters("q=" + url.QueryEscape(" 论文 Graph ")) + if err != nil || f.Query != "论文 Graph" || f.Page != 1 || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" { + t.Fatalf("%+v %v", f, err) + } +} + +type databaseFailure string + +func (failure databaseFailure) Error() string { return "private database error" } +func (failure databaseFailure) SQLState() string { return string(failure) } + +func TestStorageErrorsAreRedacted(t *testing.T) { + for _, test := range []struct { + err error + status int + code string + }{ + {sql.ErrNoRows, 404, "not_found"}, + {context.DeadlineExceeded, 504, "research_timeout"}, + {errors.Join(databaseFailure("57014"), errors.New("private statement")), 504, "research_timeout"}, + {databaseFailure("42501"), 503, "research_unavailable"}, + {errors.New("postgres://private-user:private-password@db/corpus"), 503, "research_unavailable"}, + } { + mux := http.NewServeMux() + Register(mux, &fixtureRepository{err: test.err}, map[string]string{"thesisly": "unused"}) + w := request(mux, "GET", "/projects/thesisly/research/papers/42") + if w.Code != test.status || !strings.Contains(w.Body.String(), test.code) || strings.Contains(w.Body.String(), "private") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + } +} + +func TestConcurrencyAndTimeoutReleaseSlots(t *testing.T) { + s := &service{repository: &fixtureRepository{wait: true}, projects: map[string]string{"thesisly": "unused"}, slots: make(chan struct{}, 2), timeout: time.Millisecond} + mux := http.NewServeMux() + mux.HandleFunc("GET /projects/{project}/research/papers", s.serve) + s.slots <- struct{}{} + s.slots <- struct{}{} + w := request(mux, "GET", "/projects/thesisly/research/papers") + if w.Code != 429 || w.Header().Get("Retry-After") != "1" { + t.Fatalf("busy %d", w.Code) + } + <-s.slots + <-s.slots + w = request(mux, "GET", "/projects/thesisly/research/papers") + if w.Code != 504 || len(s.slots) != 0 { + t.Fatalf("timeout %d, slots %d", w.Code, len(s.slots)) + } +} diff --git a/internal/research/postgres.go b/internal/research/postgres.go new file mode 100644 index 0000000..7198dc2 --- /dev/null +++ b/internal/research/postgres.go @@ -0,0 +1,389 @@ +package research + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "regexp" + "strconv" + "strings" +) + +// The fixed Paper Atlas migration a3b677b preserves business columns as TEXT +// and BIGINT and creates the English title/abstract GIN expression below. +// No schema creation, indexing or other writes belong to this repository. +type PostgresRepository struct { + db *sql.DB + prefix string + schema string + encoded bool +} + +var schemaName = regexp.MustCompile(`^[a-z][a-z0-9_]{0,35}$`) + +func NewPostgresRepository(db *sql.DB, schema string) (*PostgresRepository, error) { + if db == nil || !schemaName.MatchString(schema) { + return nil, errors.New("invalid research database configuration") + } + return &PostgresRepository{db: db, prefix: `"` + schema + `".`, schema: schema}, nil +} + +func (r *PostgresRepository) query(value string) string { + return strings.ReplaceAll(value, "{schema}", r.prefix) +} + +func (r *PostgresRepository) begin(ctx context.Context) (*sql.Tx, error) { + tx, err := r.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true}) + if err != nil { + return nil, err + } + _, err = tx.ExecContext(ctx, `SELECT set_config('statement_timeout','8000',true), + set_config('lock_timeout','1000',true), set_config('work_mem','2MB',true), + set_config('max_parallel_workers_per_gather','0',true), set_config('jit','off',true)`) + if err != nil { + _ = tx.Rollback() + return nil, err + } + return tx, nil +} + +// Check resolves required columns and SELECT permissions without scanning the +// corpus. The migration publishes the complete schema atomically. +func (r *PostgresRepository) Check(ctx context.Context) error { + tx, err := r.begin(ctx) + if err != nil { + return err + } + defer tx.Rollback() + var annotation sql.NullString + err = tx.QueryRowContext(ctx, `SELECT pg_catalog.obj_description(oid,'pg_namespace') + FROM pg_catalog.pg_namespace WHERE nspname=$1`, r.schema).Scan(&annotation) + if err != nil { + return err + } + encoded, err := encodedSchema(annotation) + if err != nil { + return err + } + rows, err := tx.QueryContext(ctx, r.query(`SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published,p.title_key, + r.external_id,r.url,r.metadata_json,so.name,i.scheme,i.value,t.namespace,t.value, + s.id,e.year,e.track,v.acronym,d.status,d.evidence_url,d.observed_at,d.decision_date + FROM {schema}papers p LEFT JOIN {schema}source_records r ON r.paper_id=p.id + LEFT JOIN {schema}sources so ON so.id=r.source_id LEFT JOIN {schema}identifiers i ON i.paper_id=p.id + LEFT JOIN {schema}paper_tags pt ON pt.paper_id=p.id LEFT JOIN {schema}tags t ON t.id=pt.tag_id + LEFT JOIN {schema}submissions s ON s.paper_id=p.id LEFT JOIN {schema}editions e ON e.id=s.edition_id + LEFT JOIN {schema}venues v ON v.id=e.venue_id LEFT JOIN {schema}decisions d ON d.submission_id=s.id WHERE false`)) + if err != nil { + return err + } + if err = rows.Close(); err != nil { + return err + } + if err = tx.Commit(); err != nil { + return err + } + r.encoded = encoded + return nil +} + +func whereClause(filters Filters) (string, []any) { + clauses, values := []string{}, []any{} + if filters.Query != "" { + values = append(values, filters.Query) + if filters.SearchIn == "author" { + // JSON stores author strings or [family,given,suffix] tuples. Plain + // full-text terms search names regardless of tuple display order. + clauses = append(clauses, `to_tsvector('simple'::regconfig,p.authors_json) @@ plainto_tsquery('simple'::regconfig,$1)`) + } else { + clauses = append(clauses, `to_tsvector('english'::regconfig,coalesce(p.title,'') || ' ' || coalesce(p.abstract,'')) @@ plainto_tsquery('english'::regconfig,$1)`) + } + } + if filters.Topic != "" { + values = append(values, filters.Topic) + clauses = append(clauses, fmt.Sprintf(`EXISTS(SELECT 1 FROM {schema}paper_tags pt + JOIN {schema}tags t ON t.id=pt.tag_id + WHERE pt.paper_id=p.id AND t.namespace='arxiv:category' AND t.value=$%d)`, len(values))) + } + if len(clauses) == 0 { + return "", values + } + return " WHERE " + strings.Join(clauses, " AND "), values +} + +type scanner interface{ Scan(...any) error } + +func (r *PostgresRepository) scanPaper(row scanner) (Paper, error) { + p := Paper{Authors: []string{}, Topics: []string{}, Sources: []Source{}} + var id int64 + var abstract, published sql.NullString + var authors string + if err := row.Scan(&id, r.text(&p.Title), r.text(&abstract), r.text(&authors), r.text(&published)); err != nil { + return p, err + } + p.ID = strconv.FormatInt(id, 10) + if abstract.Valid { + p.Abstract = &abstract.String + } + if published.Valid { + p.PublishedAt = &published.String + } + var err error + p.Authors, err = decodeAuthors(authors) + return p, err +} + +func decodeAuthors(raw string) ([]string, error) { + var entries []json.RawMessage + if err := json.Unmarshal([]byte(raw), &entries); err != nil { + return nil, errors.New("invalid stored authors") + } + names := []string{} + for _, entry := range entries { + var name string + if json.Unmarshal(entry, &name) == nil { + if name = strings.TrimSpace(name); name != "" { + names = append(names, name) + } + continue + } + var tuple []string + if json.Unmarshal(entry, &tuple) != nil || len(tuple) == 0 || len(tuple) > 4 { + return nil, errors.New("unsupported stored author") + } + // arXiv may append an affiliation after family, given and suffix. + // It is source metadata, not part of the display name. + if len(tuple) == 4 { + tuple = tuple[:3] + } + if len(tuple) >= 2 { + tuple[0], tuple[1] = tuple[1], tuple[0] + } + parts := []string{} + for _, part := range tuple { + if part = strings.TrimSpace(part); part != "" { + parts = append(parts, part) + } + } + if name = strings.Join(parts, " "); name != "" { + names = append(names, name) + } + } + return names, nil +} + +func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, error) { + result := Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize} + tx, err := r.begin(ctx) + if err != nil { + return result, err + } + defer tx.Rollback() + if r.encoded { + filters.Query = encodeText(filters.Query) + filters.Topic = encodeText(filters.Topic) + } + where, args := whereClause(filters) + if err = tx.QueryRowContext(ctx, r.query(`SELECT count(*) FROM {schema}papers p`+where), args...).Scan(&result.Total); err != nil { + return result, err + } + order := "p.id DESC" + if filters.Sort == "title" { + order = "p.title_key,p.id" + } + args = append(args, filters.PageSize, (filters.Page-1)*filters.PageSize) + // Select IDs before loading potentially TOASTed titles/abstracts. With + // sparse categories, selecting full rows here can make the planner walk + // hundreds of thousands of wide heap rows before finding eight matches. + candidateQuery := `SELECT p.id FROM {schema}papers p` + candidateOrder := order + prefix := "" + if where != "" { + columns := "p.id" + if filters.Sort == "title" { + columns += ",p.title_key" + } + // Apply search before LIMIT/ORDER BY so a sparse term cannot turn + // into a backward heap scan that recomputes vectors for millions. + prefix = fmt.Sprintf(`matches AS MATERIALIZED (SELECT %s FROM {schema}papers p%s), `, columns, where) + candidateQuery = "SELECT id FROM matches" + candidateOrder = strings.ReplaceAll(order, "p.", "") + } + query := fmt.Sprintf(`WITH %spage_ids AS MATERIALIZED ( + %s ORDER BY %s LIMIT $%d OFFSET $%d + ) SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published + FROM page_ids JOIN {schema}papers p ON p.id=page_ids.id ORDER BY %s`, prefix, candidateQuery, candidateOrder, len(args)-1, len(args), order) + rows, err := tx.QueryContext(ctx, r.query(query), args...) + if err != nil { + return result, err + } + for rows.Next() { + p, err := r.scanPaper(rows) + if err != nil { + _ = rows.Close() + return result, err + } + result.Items = append(result.Items, p) + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return result, err + } + if err = r.enrich(ctx, tx, result.Items); err != nil { + return result, err + } + rows, err = tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`)) + if err != nil { + return result, err + } + for rows.Next() { + var topic string + if err = rows.Scan(r.text(&topic)); err != nil { + _ = rows.Close() + return result, err + } + result.Topics = append(result.Topics, topic) + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return result, err + } + return result, tx.Commit() +} + +func (r *PostgresRepository) Get(ctx context.Context, id int64) (Paper, error) { + tx, err := r.begin(ctx) + if err != nil { + return Paper{}, err + } + defer tx.Rollback() + p, err := r.scanPaper(tx.QueryRowContext(ctx, r.query(`SELECT id,title,abstract,authors_json,first_published FROM {schema}papers WHERE id=$1`), id)) + if err != nil { + return p, err + } + papers := []Paper{p} + if err = r.enrich(ctx, tx, papers); err != nil { + return p, err + } + return papers[0], tx.Commit() +} + +// Batch metadata queries avoid per-paper round trips and share the count/page +// snapshot. User values are bound; only the validated schema is interpolated. +func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Paper) error { + if len(papers) == 0 { + return nil + } + ids := make([]int64, len(papers)) + byID := map[int64]*Paper{} + for i := range papers { + id, err := strconv.ParseInt(papers[i].ID, 10, 64) + if err != nil { + return err + } + ids[i] = id + byID[id] = &papers[i] + } + rows, err := tx.QueryContext(ctx, r.query(`SELECT DISTINCT pt.paper_id,t.value + FROM {schema}paper_tags pt JOIN {schema}tags t ON t.id=pt.tag_id + WHERE pt.paper_id=ANY($1) AND t.namespace='arxiv:category' ORDER BY pt.paper_id,t.value`), ids) + if err != nil { + return err + } + for rows.Next() { + var id int64 + var topic string + if err = rows.Scan(&id, r.text(&topic)); err != nil { + _ = rows.Close() + return err + } + byID[id].Topics = append(byID[id].Topics, topic) + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return err + } + rows, err = tx.QueryContext(ctx, r.query(`SELECT r.paper_id,s.name,r.external_id,r.url,r.metadata_json + FROM {schema}source_records r JOIN {schema}sources s ON s.id=r.source_id + WHERE r.paper_id=ANY($1) ORDER BY r.paper_id,r.id`), ids) + if err != nil { + return err + } + for rows.Next() { + var id int64 + var source Source + var metadata string + if err = rows.Scan(&id, r.text(&source.Label), r.text(&source.ExternalID), r.text(&source.URL), r.text(&metadata)); err != nil { + _ = rows.Close() + return err + } + p := byID[id] + switch source.Label { + case "arxiv": + source.Label = "arXiv" + case "openreview": + source.Label = "OpenReview" + case "pmlr": + source.Label = "PMLR" + } + p.Sources = append(p.Sources, source) + var fields map[string]json.RawMessage + if json.Unmarshal([]byte(metadata), &fields) == nil { + for key, target := range map[string]**string{"doi": &p.DOI, "journal-ref": &p.Journal, "comments": &p.Comments} { + var value string + if *target == nil && json.Unmarshal(fields[key], &value) == nil && strings.TrimSpace(value) != "" { + *target = &value + } + } + } + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return err + } + rows, err = tx.QueryContext(ctx, r.query(`SELECT paper_id,value FROM {schema}identifiers WHERE paper_id=ANY($1) AND scheme='doi' ORDER BY paper_id,value`), ids) + if err != nil { + return err + } + for rows.Next() { + var id int64 + var value string + if err = rows.Scan(&id, r.text(&value)); err != nil { + _ = rows.Close() + return err + } + if byID[id].DOI == nil && strings.TrimSpace(value) != "" { + byID[id].DOI = &value + } + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return err + } + rows, err = tx.QueryContext(ctx, r.query(`SELECT s.paper_id,v.acronym,e.year,e.track,coalesce(d.status,'unknown'),coalesce(d.evidence_url,'') + FROM {schema}submissions s JOIN {schema}editions e ON e.id=s.edition_id JOIN {schema}venues v ON v.id=e.venue_id + LEFT JOIN LATERAL (SELECT status,evidence_url FROM {schema}decisions WHERE submission_id=s.id + ORDER BY observed_at DESC,coalesce(decision_date,observed_at) DESC,id DESC LIMIT 1) d ON true + WHERE s.paper_id=ANY($1) ORDER BY s.paper_id,e.year DESC,v.acronym,e.track,s.id`), ids) + if err != nil { + return err + } + for rows.Next() { + var id int64 + var publication Publication + if err = rows.Scan(&id, r.text(&publication.Venue), &publication.Year, r.text(&publication.Track), r.text(&publication.Status), r.text(&publication.EvidenceURL)); err != nil { + _ = rows.Close() + return err + } + byID[id].Publications = append(byID[id].Publications, publication) + } + err = rows.Err() + _ = rows.Close() + return err +} diff --git a/internal/research/postgres_integration_test.go b/internal/research/postgres_integration_test.go new file mode 100644 index 0000000..65c999b --- /dev/null +++ b/internal/research/postgres_integration_test.go @@ -0,0 +1,186 @@ +package research + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "os" + "reflect" + "testing" + "time" + + _ "github.com/jackc/pgx/v5/stdlib" +) + +func integrationRepository(t *testing.T) (*PostgresRepository, context.Context) { + t.Helper() + dsn := os.Getenv("RESEARCH_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("requires the disposable research fixture database") + } + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + db.SetMaxOpenConns(2) + repository, err := NewPostgresRepository(db, "research_fixture") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + return repository, ctx +} + +func paperIDs(papers []Paper) []string { + ids := []string{} + for _, paper := range papers { + ids = append(ids, paper.ID) + } + return ids +} + +func TestPostgresPageSearchAndFilters(t *testing.T) { + r, ctx := integrationRepository(t) + if err := r.Check(ctx); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + filters Filters + total int64 + ids []string + }{ + {Filters{Page: 1, PageSize: 2}, 5, []string{"5", "4"}}, + {Filters{Page: 2, PageSize: 2}, 5, []string{"3", "2"}}, + {Filters{Page: 3, PageSize: 2}, 5, []string{"1"}}, + {Filters{Page: 1, PageSize: 2, Sort: "title"}, 5, []string{"2", "5"}}, + {Filters{Page: 1, PageSize: 8, Query: "graph"}, 2, []string{"3", "1"}}, + {Filters{Page: 1, PageSize: 8, Query: "graph OR zebra"}, 0, []string{}}, + {Filters{Page: 1, PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, 1, []string{"1"}}, + {Filters{Page: 1, PageSize: 8, Topic: "cs.LG"}, 1, []string{"1"}}, + {Filters{Page: 1, PageSize: 8, Topic: "cs.DC", Query: "graph"}, 1, []string{"3"}}, + {Filters{Page: 1, PageSize: 8, Query: `x'); DROP TABLE papers;--`}, 0, []string{}}, + } { + page, err := r.List(ctx, test.filters) + if err != nil { + t.Fatalf("%+v: %v", test.filters, err) + } + if page.Total != test.total || !reflect.DeepEqual(paperIDs(page.Items), test.ids) { + t.Fatalf("%+v: total=%d ids=%v", test.filters, page.Total, paperIDs(page.Items)) + } + if !reflect.DeepEqual(page.Topics, []string{"cs.AI", "cs.DC", "cs.LG"}) { + t.Fatalf("mixed tag namespaces: %v", page.Topics) + } + if page.Page != test.filters.Page || page.PageSize != test.filters.PageSize { + t.Fatal("lost paging metadata") + } + } +} + +func TestPostgresMetadataPreservesEvidence(t *testing.T) { + r, ctx := integrationRepository(t) + p, err := r.Get(ctx, 1) + if err != nil { + t.Fatal(err) + } + if p.Title != "Graph & learning " || p.Abstract == nil || *p.Abstract != "Neural graph models preserve original abstracts." || !reflect.DeepEqual(p.Authors, []string{"Ada Smith", "The Research Group"}) { + t.Fatalf("original content changed: %+v", p) + } + if !reflect.DeepEqual(p.Topics, []string{"cs.AI", "cs.LG"}) || len(p.Sources) != 1 || p.Sources[0].ExternalID != "fixture-1" || p.Sources[0].Label != "arXiv" { + t.Fatalf("incorrect source/tag joins: %+v", p) + } + if p.DOI == nil || *p.DOI != "10.fixture/canonical" || p.Journal == nil || *p.Journal != "Journal Fixture" || p.Comments == nil || *p.Comments != "Original comment" { + t.Fatalf("metadata missing: %+v", p) + } + if len(p.Publications) != 1 || p.Publications[0].Status != "withdrawn" || p.Publications[0].EvidenceURL != "https://example.test/decision/3" { + t.Fatalf("latest decision ordering changed: %+v", p.Publications) + } + p, err = r.Get(ctx, 3) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(p.Authors, []string{"Carla Network"}) || len(p.Sources) != 2 || p.DOI == nil || *p.DOI != "10.fixture/other" || len(p.Publications) != 1 || p.Publications[0].Status != "unknown" || p.Publications[0].EvidenceURL != "" { + t.Fatalf("unknown decision/source mapping: %+v", p) + } + p, err = r.Get(ctx, 2) + if err != nil || p.Abstract != nil || p.PublishedAt != nil || p.Sources == nil || p.Topics == nil { + t.Fatalf("nullable data changed: %+v %v", p, err) + } + if _, err = r.Get(ctx, 99999); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("missing paper: %v", err) + } +} + +func TestPostgresCorpusIsReadOnlyAndHTTPWorks(t *testing.T) { + r, ctx := integrationRepository(t) + tx, err := r.begin(ctx) + if err != nil { + t.Fatal(err) + } + var readOnly, isolation, jit string + if err = tx.QueryRowContext(ctx, "SHOW transaction_read_only").Scan(&readOnly); err != nil { + t.Fatal(err) + } + if err = tx.QueryRowContext(ctx, "SHOW transaction_isolation").Scan(&isolation); err != nil { + t.Fatal(err) + } + if err = tx.QueryRowContext(ctx, "SHOW jit").Scan(&jit); err != nil { + t.Fatal(err) + } + _ = tx.Rollback() + if readOnly != "on" || isolation != "repeatable read" || jit != "off" { + t.Fatalf("unexpected read transaction settings: %s %s jit=%s", readOnly, isolation, jit) + } + _, err = r.db.ExecContext(ctx, `UPDATE research_fixture.papers SET title='unexpected write' WHERE id=1`) + var state interface{ SQLState() string } + if !errors.As(err, &state) || state.SQLState() != "42501" { + t.Fatalf("expected database permission denial, got %v", err) + } + mux := http.NewServeMux() + Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"}) + w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph") + var page Page + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || page.Total != 2 { + t.Fatalf("SQL-backed HTTP list: %d %s", w.Code, w.Body.String()) + } + w = request(mux, "GET", "/projects/thesisly/research/papers/1") + var paper Paper + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &paper) != nil || paper.ID != "1" { + t.Fatalf("SQL-backed HTTP detail: %d %s", w.Code, w.Body.String()) + } + if w = request(mux, "GET", "/projects/thesisly/research/papers/99999"); w.Code != 404 { + t.Fatalf("missing HTTP status %d", w.Code) + } +} + +func TestPostgresPublishedTextCodec(t *testing.T) { + legacy, ctx := integrationRepository(t) + r, err := NewPostgresRepository(legacy.db, "research_encoded_fixture") + if err != nil { + t.Fatal(err) + } + if err = r.Check(ctx); err != nil { + t.Fatal(err) + } + p, err := r.Get(ctx, 1) + if err != nil { + t.Fatal(err) + } + if p.Title != "Graph\x00original\x01" || p.Abstract == nil || *p.Abstract != "Text\x00tail" || !reflect.DeepEqual(p.Authors, []string{"Ada\x00Smith", "Group\x01Name"}) || p.Comments == nil || *p.Comments != "Original\x00comment" { + t.Fatalf("published text was not restored: %+v", p) + } + mux := http.NewServeMux() + Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"}) + w := request(mux, "GET", "/projects/thesisly/research/papers/1") + var decoded Paper + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &decoded) != nil || decoded.Title != p.Title || decoded.Abstract == nil || *decoded.Abstract != *p.Abstract { + t.Fatalf("HTTP lost original control characters: %d %s", w.Code, w.Body.String()) + } + page, err := r.List(ctx, Filters{Page: 1, PageSize: 8, Query: "graph"}) + if err != nil || page.Total != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title { + t.Fatalf("encoded corpus search: %+v %v", page, err) + } +} diff --git a/internal/research/postgres_test.go b/internal/research/postgres_test.go new file mode 100644 index 0000000..e1bffb8 --- /dev/null +++ b/internal/research/postgres_test.go @@ -0,0 +1,78 @@ +package research + +import ( + "database/sql" + "encoding/json" + "reflect" + "strings" + "testing" +) + +func TestStoredAuthorShapes(t *testing.T) { + for _, test := range []struct { + raw string + want []string + }{ + {`[]`, []string{}}, + {`[["Buhrman","Harry","","CWI"],["Family","Given","Jr.","University"]]`, []string{"Harry Buhrman", "Given Family Jr."}}, + {`["The BABAR Collaboration","B. Aubert"]`, []string{"The BABAR Collaboration", "B. Aubert"}}, + {`[["Balázs","C.",""],["Berger","E. L.",""],["Smith","Ada","Jr."]]`, []string{"C. Balázs", "E. L. Berger", "Ada Smith Jr."}}, + {`[" 张三 ",["Only family"],["","Only given",""],""]`, []string{"张三", "Only family", "Only given"}}, + } { + got, err := decodeAuthors(test.raw) + if err != nil || !reflect.DeepEqual(got, test.want) { + t.Fatalf("%s: %#v, %v", test.raw, got, err) + } + encoded, _ := json.Marshal(got) + if string(encoded) == "null" { + t.Fatal("missing authors must remain an array") + } + } + for _, raw := range []string{`broken`, `{"unexpected":"object"}`, `[42]`, `[["a","b","c","d","e"]]`} { + if _, err := decodeAuthors(raw); err == nil { + t.Fatalf("accepted unsupported authors %s", raw) + } + } +} + +func TestQueryValuesRemainBound(t *testing.T) { + query := `graph'); DROP TABLE papers; -- OR author:*` + where, args := whereClause(Filters{Query: query, Topic: "cs.LG", SearchIn: "text"}) + if strings.Contains(where, query) || !reflect.DeepEqual(args, []any{query, "cs.LG"}) { + t.Fatalf("unbound user input: %s %#v", where, args) + } + if !strings.Contains(where, "plainto_tsquery('english'::regconfig,$1)") || !strings.Contains(where, "t.value=$2") { + t.Fatalf("unexpected search contract: %s", where) + } + where, args = whereClause(Filters{Query: "Ada Smith", SearchIn: "author"}) + if !strings.Contains(where, "p.authors_json") || strings.Contains(where, "p.abstract") || !reflect.DeepEqual(args, []any{"Ada Smith"}) { + t.Fatalf("author search includes unrelated text: %s", where) + } + where, args = whereClause(Filters{Topic: "cs.AI"}) + if !strings.Contains(where, "t.value=$1") || !reflect.DeepEqual(args, []any{"cs.AI"}) { + t.Fatalf("topic-only binding: %s %#v", where, args) + } + where, args = whereClause(Filters{}) + if where != "" || len(args) != 0 { + t.Fatal("empty filters changed the corpus") + } +} + +func TestSchemaCannotEscapeQualification(t *testing.T) { + for _, name := range []string{"", `research";DROP SCHEMA public;--`, "a.b", "Public", strings.Repeat("a", 37)} { + if _, err := NewPostgresRepository(&sql.DB{}, name); err == nil { + t.Fatalf("accepted schema %q", name) + } + } + if _, err := NewPostgresRepository(nil, "research"); err == nil { + t.Fatal("accepted nil database") + } + r, err := NewPostgresRepository(&sql.DB{}, "research_v2") + if err != nil { + t.Fatal(err) + } + got := r.query(`SELECT p.id FROM {schema}papers p JOIN {schema}source_records s ON s.paper_id=p.id`) + if got != `SELECT p.id FROM "research_v2".papers p JOIN "research_v2".source_records s ON s.paper_id=p.id` { + t.Fatalf("unqualified corpus tables: %s", got) + } +} diff --git a/internal/research/testdata/corpus.sql b/internal/research/testdata/corpus.sql new file mode 100644 index 0000000..3bbd5ac --- /dev/null +++ b/internal/research/testdata/corpus.sql @@ -0,0 +1,81 @@ +-- Synthetic fixture for the disposable, network-isolated CI database only. +-- Required columns/types follow Paper Atlas's fixed a3b677b migration. +CREATE SCHEMA research_fixture; +CREATE TABLE research_fixture.papers ( + id bigint PRIMARY KEY, title text NOT NULL, abstract text, title_key text COLLATE "C" NOT NULL, + authors_json text NOT NULL, first_published text +); +CREATE INDEX papers_title_key ON research_fixture.papers(title_key); +CREATE INDEX papers_search_gin ON research_fixture.papers USING gin + (to_tsvector('english'::regconfig,coalesce(title,'') || ' ' || coalesce(abstract,''))); +CREATE TABLE research_fixture.sources (id bigint PRIMARY KEY,name text NOT NULL); +CREATE TABLE research_fixture.source_records ( + id bigint PRIMARY KEY,source_id bigint REFERENCES research_fixture.sources(id), + paper_id bigint REFERENCES research_fixture.papers(id),external_id text,url text,metadata_json text +); +CREATE TABLE research_fixture.identifiers (scheme text,value text,paper_id bigint REFERENCES research_fixture.papers(id),PRIMARY KEY(scheme,value)); +CREATE TABLE research_fixture.tags (id bigint PRIMARY KEY,namespace text,value text,UNIQUE(namespace,value)); +CREATE TABLE research_fixture.paper_tags ( + paper_id bigint REFERENCES research_fixture.papers(id),tag_id bigint REFERENCES research_fixture.tags(id), + evidence_key text,PRIMARY KEY(paper_id,tag_id,evidence_key) +); +CREATE TABLE research_fixture.venues (id bigint PRIMARY KEY,acronym text); +CREATE TABLE research_fixture.editions (id bigint PRIMARY KEY,venue_id bigint REFERENCES research_fixture.venues(id),year bigint,track text); +CREATE TABLE research_fixture.submissions ( + id bigint PRIMARY KEY,paper_id bigint REFERENCES research_fixture.papers(id),edition_id bigint REFERENCES research_fixture.editions(id) +); +CREATE TABLE research_fixture.decisions ( + id bigint PRIMARY KEY,submission_id bigint REFERENCES research_fixture.submissions(id), + status text,evidence_url text,observed_at text,decision_date text +); +CREATE INDEX decisions_submission_order ON research_fixture.decisions(submission_id,observed_at DESC,id DESC); + +INSERT INTO research_fixture.papers VALUES + (1,'Graph & learning ','Neural graph models preserve original abstracts.','graph learning','[["Smith","Ada",""],"The Research Group"]','2024-01-02'), + (2,'Alpha topology',NULL,'alpha topology','["Bruno Park"]',NULL), + (3,'Graph systems','Graph systems and data.','graph systems','[["Network","Carla","","Fixture University"]]','2023-01-01'), + (4,'Zebra methods','Evidence and uncertainty.','zebra methods','[]','2022-01-01'), + (5,'Alpha Topology','Another independent record.','alpha topology','["Dana Other"]','2025-01-01'); +INSERT INTO research_fixture.sources VALUES(10,'arxiv'),(20,'pmlr'); +INSERT INTO research_fixture.source_records VALUES + (101,10,1,'fixture-1','https://example.test/papers/1','{"journal-ref":"Journal Fixture","comments":"Original comment"}'), + (202,20,3,'fixture-3','https://example.test/papers/3','{"doi":"10.fixture/other"}'), + (203,10,3,'fixture-3-arxiv','https://example.test/papers/3/arxiv','{}'); +INSERT INTO research_fixture.identifiers VALUES('doi','10.fixture/canonical',1); +INSERT INTO research_fixture.tags VALUES(1,'arxiv:category','cs.LG'),(2,'arxiv:category','cs.AI'),(3,'arxiv:category','cs.DC'),(4,'different:namespace','cs.LG'); +INSERT INTO research_fixture.paper_tags VALUES(1,1,'first'),(1,1,'second'),(1,2,'first'),(3,3,'first'),(2,4,'first'); +INSERT INTO research_fixture.venues VALUES(1,'ICLR'),(2,'ICML'); +INSERT INTO research_fixture.editions VALUES(7,1,2024,'main'),(8,2,2023,'main'); +INSERT INTO research_fixture.submissions VALUES(11,1,7),(12,3,8); +INSERT INTO research_fixture.decisions VALUES + (1,11,'accepted','https://example.test/decision/1','2024-02-01','2024-01-30'), + (2,11,'rejected','https://example.test/decision/2','2024-02-01','2024-01-31'), + (3,11,'withdrawn','https://example.test/decision/3','2024-02-01','2024-01-31'), + (4,11,'accepted','https://example.test/decision/4','2024-01-01','2099-01-01'); + +CREATE ROLE foundry_research_fixture LOGIN; +REVOKE ALL ON SCHEMA research_fixture FROM PUBLIC; +GRANT USAGE ON SCHEMA research_fixture TO foundry_research_fixture; +GRANT SELECT ON ALL TABLES IN SCHEMA research_fixture TO foundry_research_fixture; + +-- Separate encoded schema mirrors the published Paper Atlas nul-soh-v1 format. +CREATE SCHEMA research_encoded_fixture; +COMMENT ON SCHEMA research_encoded_fixture IS 'paper-atlas:text-codec=nul-soh-v1'; +DO $$ +DECLARE item text; +BEGIN + FOREACH item IN ARRAY ARRAY['papers','sources','source_records','identifiers','tags','paper_tags','venues','editions','submissions','decisions'] LOOP + EXECUTE format('CREATE TABLE research_encoded_fixture.%I (LIKE research_fixture.%I INCLUDING ALL)',item,item); + EXECUTE format('INSERT INTO research_encoded_fixture.%I SELECT * FROM research_fixture.%I',item,item); + END LOOP; +END $$; +UPDATE research_encoded_fixture.papers +SET title='Graph'||chr(1)||chr(1)||'original'||chr(1)||chr(2), + abstract='Text'||chr(1)||chr(1)||'tail', + authors_json='["Ada\u0000Smith","Group\u0001Name"]' +WHERE id=1; +UPDATE research_encoded_fixture.source_records +SET metadata_json='{"comments":"Original\u0000comment"}' WHERE id=101; +REVOKE ALL ON SCHEMA research_encoded_fixture FROM PUBLIC; +GRANT USAGE ON SCHEMA research_encoded_fixture TO foundry_research_fixture; +GRANT SELECT ON ALL TABLES IN SCHEMA research_encoded_fixture TO foundry_research_fixture; diff --git a/internal/research/text.go b/internal/research/text.go new file mode 100644 index 0000000..abebbbb --- /dev/null +++ b/internal/research/text.go @@ -0,0 +1,88 @@ +package research + +import ( + "database/sql" + "errors" + "strings" +) + +// Matches Paper Atlas repair 4e62710's nul-soh-v1 protocol: NUL -> SOH/SOH and literal +// SOH -> SOH/STX. All other bytes retain their original binary ordering. +const textCodec = "nul-soh-v1" + +func encodedSchema(annotation sql.NullString) (bool, error) { + const prefix = "paper-atlas:text-codec=" + if !annotation.Valid || !strings.HasPrefix(annotation.String, prefix) { + return false, nil + } + if strings.TrimPrefix(annotation.String, prefix) != textCodec { + return false, errors.New("unsupported research text codec") + } + return true, nil +} + +func encodeText(value string) string { + return strings.ReplaceAll(strings.ReplaceAll(value, "\x01", "\x01\x02"), "\x00", "\x01\x01") +} + +func decodeText(value string) (string, error) { + if !strings.ContainsRune(value, '\x01') { + return value, nil + } + var result strings.Builder + for { + position := strings.IndexByte(value, 1) + if position < 0 { + result.WriteString(value) + return result.String(), nil + } + result.WriteString(value[:position]) + if position+1 >= len(value) { + return "", errors.New("invalid research text escape") + } + switch value[position+1] { + case 1: + result.WriteByte(0) + case 2: + result.WriteByte(1) + default: + return "", errors.New("invalid research text escape") + } + value = value[position+2:] + } +} + +type textColumn struct { + encoded bool + target any +} + +func (column textColumn) Scan(source any) error { + var value sql.NullString + if err := value.Scan(source); err != nil { + return err + } + if value.Valid && column.encoded { + var err error + value.String, err = decodeText(value.String) + if err != nil { + return err + } + } + switch target := column.target.(type) { + case *string: + if !value.Valid { + return errors.New("unexpected null research text") + } + *target = value.String + case *sql.NullString: + *target = value + default: + return errors.New("unsupported research text destination") + } + return nil +} + +func (r *PostgresRepository) text(target any) sql.Scanner { + return textColumn{encoded: r.encoded, target: target} +} diff --git a/internal/research/text_test.go b/internal/research/text_test.go new file mode 100644 index 0000000..a972ef2 --- /dev/null +++ b/internal/research/text_test.go @@ -0,0 +1,69 @@ +package research + +import ( + "database/sql" + "strings" + "testing" +) + +func TestCorpusTextCodecRoundTripAndOrder(t *testing.T) { + values := []string{"", "\x00", "\x00\x01", "\x01", "\x01\x02", "\x02", "A", "A\x00", "A\x01", "Ada 张三 Ω", "Z\x00\x01\x02尾"} + for _, value := range values { + encoded := encodeText(value) + decoded, err := decodeText(encoded) + if err != nil || decoded != value || strings.ContainsRune(encoded, 0) { + t.Fatalf("round trip failed for %q", value) + } + for _, other := range values { + if strings.Compare(value, other) != strings.Compare(encoded, encodeText(other)) { + t.Fatalf("binary ordering changed for %q %q", value, other) + } + } + } + for _, invalid := range []string{"\x01", "prefix\x01x", "\x01\x00"} { + if _, err := decodeText(invalid); err == nil { + t.Fatalf("accepted malformed escape %q", invalid) + } + } +} + +func TestCorpusTextColumnsRespectCodecAndNulls(t *testing.T) { + var required string + if err := (textColumn{target: &required}).Scan("\x01\x01"); err != nil || required != "\x01\x01" { + t.Fatal("legacy text was decoded") + } + if err := (textColumn{encoded: true, target: &required}).Scan([]byte("before\x01\x01after\x01\x02")); err != nil || required != "before\x00after\x01" { + t.Fatalf("encoded text did not decode: %q %v", required, err) + } + var nullable sql.NullString + if err := (textColumn{encoded: true, target: &nullable}).Scan(nil); err != nil || nullable.Valid { + t.Fatal("null changed into empty text") + } + if err := (textColumn{encoded: true, target: &nullable}).Scan("\x01\x01"); err != nil || !nullable.Valid || nullable.String != "\x00" { + t.Fatal("nullable text was not restored") + } + if err := (textColumn{encoded: true, target: &required}).Scan(nil); err == nil { + t.Fatal("accepted null for required text") + } + if err := (textColumn{encoded: true, target: &required}).Scan("\x01x"); err == nil { + t.Fatal("accepted corrupt encoded text") + } +} + +func TestCorpusSchemaCodecMarker(t *testing.T) { + for _, annotation := range []sql.NullString{{}, {String: "unrelated schema description", Valid: true}} { + encoded, err := encodedSchema(annotation) + if encoded || err != nil { + t.Fatal("legacy schema changed encoding") + } + } + encoded, err := encodedSchema(sql.NullString{String: "paper-atlas:text-codec=nul-soh-v1", Valid: true}) + if !encoded || err != nil { + t.Fatal("published codec was not recognized") + } + for _, name := range []string{"unknown", "", "legacy"} { + if _, err := encodedSchema(sql.NullString{String: "paper-atlas:text-codec=" + name, Valid: true}); err == nil { + t.Fatal("unknown codec was accepted") + } + } +}