From 81cf7d9448f2eec45948f1ad2b2738bef35b4247 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sat, 7 Nov 2026 01:58:30 +0800 Subject: [PATCH] Add bounded public research HTTP contract and transport tests --- docs/research-api.md | 47 ++++++++ internal/research/http.go | 209 +++++++++++++++++++++++++++++++++ internal/research/http_test.go | 138 ++++++++++++++++++++++ 3 files changed, 394 insertions(+) create mode 100644 docs/research-api.md create mode 100644 internal/research/http.go create mode 100644 internal/research/http_test.go diff --git a/docs/research-api.md b/docs/research-api.md new file mode 100644 index 0000000..6b5c21a --- /dev/null +++ b/docs/research-api.md @@ -0,0 +1,47 @@ +# Shared research API (in progress) + +This work belongs to Thesisly 研迹 and uses Foundry Core rather than a separate +application backend. The corpus is public; account sessions and future private +collections remain project scoped. + +## Current implementation boundary + +`internal/research` implements the HTTP contract and repository boundary. It is +not registered by `cmd/foundry` yet and does not open any database connection. +The existing SQLite implementation was withdrawn. Wait for the separate Paper +Atlas PostgreSQL migration to finish, inspect the published schema and indexes, +then implement the repository with a least-privilege read-only connection and +register it in the existing server. Do not add `FOUNDRY_RESEARCH_DATABASE`. + +## Routes + +- `GET /projects/{project}/research/papers` +- `GET /projects/{project}/research/papers/{id}` + +Only configured Foundry projects are accepted. Reading the public corpus does +not require a session; these routes do not expose private data or accept writes. + +List parameters are `q` (200 Unicode characters, 20 words), `searchIn` +(`text` or `author`), `topic` (an exact arXiv category), `sort` (`recent` or +`title`), `page` (1–1000), and `pageSize` (1–50, default 8). Unknown, malformed +or repeated parameters return 400. `recent` means descending stable paper ID, +not publication date. Search terms must become bound query parameters, never +raw SQL or user-supplied full-text syntax. + +List responses contain `items`, `total`, `topics`, `page`, and `pageSize`. +Each paper preserves original title/abstract, authors, source dates, topics and +source links. Publication decisions need source evidence; absent decisions stay +unknown. No invented repositories, metrics or generated research conclusions. + +The repository must count and read each list in one read-only snapshot. The +HTTP layer permits two in-flight queries with an eight-second context budget. +Busy requests return 429 with `Retry-After: 1`; deadlines return 504; storage +errors return a generic 503 without database details. Missing papers return 404. + +## Validation + +`go test -race -count=1 ./internal/research` covers routing, query limits, +Unicode, anonymous reads, rejected mutations, project lookup, errors, saturation +and timeout slot release. Run compilation/tests only on the user-confirmed ECS +runner. These transport tests do not prove a database connection, full-corpus +query performance or production availability. diff --git a/internal/research/http.go b/internal/research/http.go new file mode 100644 index 0000000..3b339b6 --- /dev/null +++ b/internal/research/http.go @@ -0,0 +1,209 @@ +// Package research serves the shared, public paper corpus. Account data and +// private collections do not belong to these anonymous read-only endpoints. +package research + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +type Source struct { + Label string `json:"label"` + URL string `json:"url"` + ExternalID string `json:"externalId"` +} + +type Publication struct { + Venue string `json:"venue"` + Year int `json:"year"` + Track string `json:"track"` + Status string `json:"status"` + EvidenceURL string `json:"evidenceUrl,omitempty"` +} + +type Paper struct { + ID string `json:"id"` + Title string `json:"title"` + Abstract *string `json:"abstract"` + Authors []string `json:"authors"` + PublishedAt *string `json:"publishedAt"` + Topics []string `json:"topics"` + Sources []Source `json:"sources"` + Publications []Publication `json:"publications,omitempty"` + DOI *string `json:"doi,omitempty"` + Journal *string `json:"journal,omitempty"` + Comments *string `json:"comments,omitempty"` +} + +type Filters struct { + Query string + SearchIn string + Topic string + Sort string + Page int + PageSize int +} + +type Page struct { + Items []Paper `json:"items"` + Total int64 `json:"total"` + Topics []string `json:"topics"` + Page int `json:"page"` + PageSize int `json:"pageSize"` +} + +// Repository implementations must use a read-only transaction for each List, +// so its count, topics and page describe the same database snapshot. +type Repository interface { + List(context.Context, Filters) (Page, error) + Get(context.Context, int64) (Paper, error) +} + +type service struct { + repository Repository + projects map[string]string + slots chan struct{} + timeout time.Duration +} + +// Register adds only GET endpoints; no credentials or database errors are sent +// to anonymous callers. A nil repository leaves the existing server unchanged. +func Register(mux *http.ServeMux, repository Repository, projects map[string]string) { + if repository == nil { + return + } + s := &service{repository: repository, projects: projects, slots: make(chan struct{}, 2), timeout: 8 * time.Second} + mux.HandleFunc("GET /projects/{project}/research/papers", s.serve) + mux.HandleFunc("GET /projects/{project}/research/papers/{id}", s.serve) +} + +func reply(w http.ResponseWriter, status int, value any) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Cache-Control", "no-store") + w.Header().Set("X-Content-Type-Options", "nosniff") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(value) +} + +func reject(w http.ResponseWriter, status int, code string) { + reply(w, status, map[string]string{"error": code}) +} + +var topicPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.-]{0,79}$`) +var positiveInteger = regexp.MustCompile(`^[1-9][0-9]*$`) + +func parseFilters(raw string) (Filters, error) { + f := Filters{SearchIn: "text", Sort: "recent", Page: 1, PageSize: 8} + values, err := url.ParseQuery(raw) + if err != nil { + return f, err + } + invalid := errors.New("invalid query") + for key, value := range values { + if len(value) != 1 { + return f, invalid + } + switch key { + case "q": + if !utf8.ValidString(value[0]) || utf8.RuneCountInString(value[0]) > 200 || len(strings.Fields(value[0])) > 20 || strings.ContainsRune(value[0], 0) { + return f, invalid + } + f.Query = strings.TrimSpace(value[0]) + case "searchIn": + if value[0] != "text" && value[0] != "author" { + return f, invalid + } + f.SearchIn = value[0] + case "topic": + if value[0] != "" && !topicPattern.MatchString(value[0]) { + return f, invalid + } + f.Topic = value[0] + case "sort": + if value[0] != "recent" && value[0] != "title" { + return f, invalid + } + f.Sort = value[0] + case "page", "pageSize": + if !positiveInteger.MatchString(value[0]) { + return f, invalid + } + n, err := strconv.Atoi(value[0]) + limit := 1000 + if key == "pageSize" { + limit = 50 + } + if err != nil || n > limit { + return f, invalid + } + if key == "page" { + f.Page = n + } else { + f.PageSize = n + } + default: + return f, invalid + } + } + return f, nil +} + +func (s *service) serve(w http.ResponseWriter, r *http.Request) { + if _, exists := s.projects[r.PathValue("project")]; !exists { + reject(w, 404, "not_found") + return + } + var filters Filters + var id int64 + var err error + if value := r.PathValue("id"); value != "" { + if !positiveInteger.MatchString(value) || r.URL.RawQuery != "" { + reject(w, 400, "invalid_request") + return + } + id, err = strconv.ParseInt(value, 10, 64) + } else { + filters, err = parseFilters(r.URL.RawQuery) + } + if err != nil { + reject(w, 400, "invalid_request") + return + } + select { + case s.slots <- struct{}{}: + defer func() { <-s.slots }() + default: + w.Header().Set("Retry-After", "1") + reject(w, 429, "research_busy") + return + } + ctx, cancel := context.WithTimeout(r.Context(), s.timeout) + defer cancel() + var result any + if id != 0 { + result, err = s.repository.Get(ctx, id) + } else { + result, err = s.repository.List(ctx, filters) + } + if err == nil { + reply(w, 200, result) + return + } + switch { + case errors.Is(err, sql.ErrNoRows): + reject(w, 404, "not_found") + case errors.Is(ctx.Err(), context.DeadlineExceeded), errors.Is(err, context.DeadlineExceeded): + reject(w, 504, "research_timeout") + default: + reject(w, 503, "research_unavailable") + } +} diff --git a/internal/research/http_test.go b/internal/research/http_test.go new file mode 100644 index 0000000..47573cd --- /dev/null +++ b/internal/research/http_test.go @@ -0,0 +1,138 @@ +package research + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" +) + +type fixtureRepository struct { + filters Filters + id int64 + err error + wait bool + calls int +} + +func (f *fixtureRepository) List(ctx context.Context, filters Filters) (Page, error) { + f.calls++ + f.filters = filters + if f.wait { + <-ctx.Done() + return Page{}, ctx.Err() + } + return Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}, f.err +} + +func (f *fixtureRepository) Get(ctx context.Context, id int64) (Paper, error) { + f.calls++ + f.id = id + return Paper{ID: "42", Title: "Original title", Authors: []string{}, Topics: []string{}, Sources: []Source{}}, f.err +} + +func request(handler http.Handler, method, path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + handler.ServeHTTP(w, httptest.NewRequest(method, path, nil)) + return w +} + +func TestAnonymousReadRoutes(t *testing.T) { + f := &fixtureRepository{} + mux := http.NewServeMux() + Register(mux, f, map[string]string{"thesisly": "http://localhost:17322/auth/callback"}) + w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&page=2&pageSize=10") + if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", Page: 2, PageSize: 10}) { + t.Fatalf("list: %d %+v", w.Code, f.filters) + } + var page Page + if err := json.Unmarshal(w.Body.Bytes(), &page); err != nil || page.Items == nil || page.Topics == nil { + t.Fatalf("invalid page: %s", w.Body.String()) + } + if w.Header().Get("Cache-Control") != "no-store" || w.Header().Get("Content-Type") != "application/json" { + t.Fatal("missing response headers") + } + w = request(mux, "GET", "/projects/thesisly/research/papers/42") + if w.Code != 200 || f.id != 42 || !strings.Contains(w.Body.String(), `"abstract":null`) { + t.Fatalf("detail: %d %s", w.Code, w.Body.String()) + } + if got := request(mux, "POST", "/projects/thesisly/research/papers").Code; got != 405 { + t.Fatalf("mutation status %d", got) + } + if got := request(mux, "GET", "/projects/other/research/papers").Code; got != 404 { + t.Fatalf("unknown project status %d", got) + } +} + +func TestInvalidQueriesNeverReachRepository(t *testing.T) { + queries := []string{"unknown=1", "q=x&q=y", "q=%ff", "q=%00", "q=%zz", "page=0", "page=1001", "page=-1", "page=1.5", "page=01", "pageSize=51", "pageSize=0", "pageSize=9999999999999999999999", "searchIn=sql", "sort=published", "topic=arxiv:cs.LG", "q=" + url.QueryEscape(strings.Repeat("字", 201)), "q=" + url.QueryEscape(strings.Repeat("word ", 21))} + for _, query := range queries { + t.Run(query, func(t *testing.T) { + f := &fixtureRepository{} + mux := http.NewServeMux() + Register(mux, f, map[string]string{"thesisly": "unused"}) + w := request(mux, "GET", "/projects/thesisly/research/papers?"+query) + if w.Code != 400 || f.calls != 0 { + t.Fatalf("status %d, calls %d", w.Code, f.calls) + } + }) + } + for _, path := range []string{"0", "-1", "1x", "9223372036854775808", "1?extra=1"} { + mux := http.NewServeMux() + f := &fixtureRepository{} + Register(mux, f, map[string]string{"thesisly": "unused"}) + if w := request(mux, "GET", "/projects/thesisly/research/papers/"+path); w.Code != 400 || f.calls != 0 { + t.Fatalf("invalid id %q: %d", path, w.Code) + } + } +} + +func TestQueryDefaultsAndUnicode(t *testing.T) { + f, err := parseFilters("q=" + url.QueryEscape(" 论文 Graph ")) + if err != nil || f.Query != "论文 Graph" || f.Page != 1 || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" { + t.Fatalf("%+v %v", f, err) + } +} + +func TestStorageErrorsAreRedacted(t *testing.T) { + for _, test := range []struct { + err error + status int + code string + }{ + {sql.ErrNoRows, 404, "not_found"}, + {context.DeadlineExceeded, 504, "research_timeout"}, + {errors.New("postgres://private-user:private-password@db/corpus"), 503, "research_unavailable"}, + } { + mux := http.NewServeMux() + Register(mux, &fixtureRepository{err: test.err}, map[string]string{"thesisly": "unused"}) + w := request(mux, "GET", "/projects/thesisly/research/papers/42") + if w.Code != test.status || !strings.Contains(w.Body.String(), test.code) || strings.Contains(w.Body.String(), "private") { + t.Fatalf("%d %s", w.Code, w.Body.String()) + } + } +} + +func TestConcurrencyAndTimeoutReleaseSlots(t *testing.T) { + s := &service{repository: &fixtureRepository{wait: true}, projects: map[string]string{"thesisly": "unused"}, slots: make(chan struct{}, 2), timeout: time.Millisecond} + mux := http.NewServeMux() + mux.HandleFunc("GET /projects/{project}/research/papers", s.serve) + s.slots <- struct{}{} + s.slots <- struct{}{} + w := request(mux, "GET", "/projects/thesisly/research/papers") + if w.Code != 429 || w.Header().Get("Retry-After") != "1" { + t.Fatalf("busy %d", w.Code) + } + <-s.slots + <-s.slots + w = request(mux, "GET", "/projects/thesisly/research/papers") + if w.Code != 504 || len(s.slots) != 0 { + t.Fatalf("timeout %d, slots %d", w.Code, len(s.slots)) + } +}