Add bounded public research HTTP contract and transport tests
This commit is contained in:
@@ -0,0 +1,47 @@
|
||||
# Shared research API (in progress)
|
||||
|
||||
This work belongs to Thesisly 研迹 and uses Foundry Core rather than a separate
|
||||
application backend. The corpus is public; account sessions and future private
|
||||
collections remain project scoped.
|
||||
|
||||
## Current implementation boundary
|
||||
|
||||
`internal/research` implements the HTTP contract and repository boundary. It is
|
||||
not registered by `cmd/foundry` yet and does not open any database connection.
|
||||
The existing SQLite implementation was withdrawn. Wait for the separate Paper
|
||||
Atlas PostgreSQL migration to finish, inspect the published schema and indexes,
|
||||
then implement the repository with a least-privilege read-only connection and
|
||||
register it in the existing server. Do not add `FOUNDRY_RESEARCH_DATABASE`.
|
||||
|
||||
## Routes
|
||||
|
||||
- `GET /projects/{project}/research/papers`
|
||||
- `GET /projects/{project}/research/papers/{id}`
|
||||
|
||||
Only configured Foundry projects are accepted. Reading the public corpus does
|
||||
not require a session; these routes do not expose private data or accept writes.
|
||||
|
||||
List parameters are `q` (200 Unicode characters, 20 words), `searchIn`
|
||||
(`text` or `author`), `topic` (an exact arXiv category), `sort` (`recent` or
|
||||
`title`), `page` (1–1000), and `pageSize` (1–50, default 8). Unknown, malformed
|
||||
or repeated parameters return 400. `recent` means descending stable paper ID,
|
||||
not publication date. Search terms must become bound query parameters, never
|
||||
raw SQL or user-supplied full-text syntax.
|
||||
|
||||
List responses contain `items`, `total`, `topics`, `page`, and `pageSize`.
|
||||
Each paper preserves original title/abstract, authors, source dates, topics and
|
||||
source links. Publication decisions need source evidence; absent decisions stay
|
||||
unknown. No invented repositories, metrics or generated research conclusions.
|
||||
|
||||
The repository must count and read each list in one read-only snapshot. The
|
||||
HTTP layer permits two in-flight queries with an eight-second context budget.
|
||||
Busy requests return 429 with `Retry-After: 1`; deadlines return 504; storage
|
||||
errors return a generic 503 without database details. Missing papers return 404.
|
||||
|
||||
## Validation
|
||||
|
||||
`go test -race -count=1 ./internal/research` covers routing, query limits,
|
||||
Unicode, anonymous reads, rejected mutations, project lookup, errors, saturation
|
||||
and timeout slot release. Run compilation/tests only on the user-confirmed ECS
|
||||
runner. These transport tests do not prove a database connection, full-corpus
|
||||
query performance or production availability.
|
||||
@@ -0,0 +1,209 @@
|
||||
// Package research serves the shared, public paper corpus. Account data and
|
||||
// private collections do not belong to these anonymous read-only endpoints.
|
||||
package research
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
type Source struct {
|
||||
Label string `json:"label"`
|
||||
URL string `json:"url"`
|
||||
ExternalID string `json:"externalId"`
|
||||
}
|
||||
|
||||
type Publication struct {
|
||||
Venue string `json:"venue"`
|
||||
Year int `json:"year"`
|
||||
Track string `json:"track"`
|
||||
Status string `json:"status"`
|
||||
EvidenceURL string `json:"evidenceUrl,omitempty"`
|
||||
}
|
||||
|
||||
type Paper struct {
|
||||
ID string `json:"id"`
|
||||
Title string `json:"title"`
|
||||
Abstract *string `json:"abstract"`
|
||||
Authors []string `json:"authors"`
|
||||
PublishedAt *string `json:"publishedAt"`
|
||||
Topics []string `json:"topics"`
|
||||
Sources []Source `json:"sources"`
|
||||
Publications []Publication `json:"publications,omitempty"`
|
||||
DOI *string `json:"doi,omitempty"`
|
||||
Journal *string `json:"journal,omitempty"`
|
||||
Comments *string `json:"comments,omitempty"`
|
||||
}
|
||||
|
||||
type Filters struct {
|
||||
Query string
|
||||
SearchIn string
|
||||
Topic string
|
||||
Sort string
|
||||
Page int
|
||||
PageSize int
|
||||
}
|
||||
|
||||
type Page struct {
|
||||
Items []Paper `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Topics []string `json:"topics"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"pageSize"`
|
||||
}
|
||||
|
||||
// Repository implementations must use a read-only transaction for each List,
|
||||
// so its count, topics and page describe the same database snapshot.
|
||||
type Repository interface {
|
||||
List(context.Context, Filters) (Page, error)
|
||||
Get(context.Context, int64) (Paper, error)
|
||||
}
|
||||
|
||||
type service struct {
|
||||
repository Repository
|
||||
projects map[string]string
|
||||
slots chan struct{}
|
||||
timeout time.Duration
|
||||
}
|
||||
|
||||
// Register adds only GET endpoints; no credentials or database errors are sent
|
||||
// to anonymous callers. A nil repository leaves the existing server unchanged.
|
||||
func Register(mux *http.ServeMux, repository Repository, projects map[string]string) {
|
||||
if repository == nil {
|
||||
return
|
||||
}
|
||||
s := &service{repository: repository, projects: projects, slots: make(chan struct{}, 2), timeout: 8 * time.Second}
|
||||
mux.HandleFunc("GET /projects/{project}/research/papers", s.serve)
|
||||
mux.HandleFunc("GET /projects/{project}/research/papers/{id}", s.serve)
|
||||
}
|
||||
|
||||
func reply(w http.ResponseWriter, status int, value any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(value)
|
||||
}
|
||||
|
||||
func reject(w http.ResponseWriter, status int, code string) {
|
||||
reply(w, status, map[string]string{"error": code})
|
||||
}
|
||||
|
||||
var topicPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.-]{0,79}$`)
|
||||
var positiveInteger = regexp.MustCompile(`^[1-9][0-9]*$`)
|
||||
|
||||
func parseFilters(raw string) (Filters, error) {
|
||||
f := Filters{SearchIn: "text", Sort: "recent", Page: 1, PageSize: 8}
|
||||
values, err := url.ParseQuery(raw)
|
||||
if err != nil {
|
||||
return f, err
|
||||
}
|
||||
invalid := errors.New("invalid query")
|
||||
for key, value := range values {
|
||||
if len(value) != 1 {
|
||||
return f, invalid
|
||||
}
|
||||
switch key {
|
||||
case "q":
|
||||
if !utf8.ValidString(value[0]) || utf8.RuneCountInString(value[0]) > 200 || len(strings.Fields(value[0])) > 20 || strings.ContainsRune(value[0], 0) {
|
||||
return f, invalid
|
||||
}
|
||||
f.Query = strings.TrimSpace(value[0])
|
||||
case "searchIn":
|
||||
if value[0] != "text" && value[0] != "author" {
|
||||
return f, invalid
|
||||
}
|
||||
f.SearchIn = value[0]
|
||||
case "topic":
|
||||
if value[0] != "" && !topicPattern.MatchString(value[0]) {
|
||||
return f, invalid
|
||||
}
|
||||
f.Topic = value[0]
|
||||
case "sort":
|
||||
if value[0] != "recent" && value[0] != "title" {
|
||||
return f, invalid
|
||||
}
|
||||
f.Sort = value[0]
|
||||
case "page", "pageSize":
|
||||
if !positiveInteger.MatchString(value[0]) {
|
||||
return f, invalid
|
||||
}
|
||||
n, err := strconv.Atoi(value[0])
|
||||
limit := 1000
|
||||
if key == "pageSize" {
|
||||
limit = 50
|
||||
}
|
||||
if err != nil || n > limit {
|
||||
return f, invalid
|
||||
}
|
||||
if key == "page" {
|
||||
f.Page = n
|
||||
} else {
|
||||
f.PageSize = n
|
||||
}
|
||||
default:
|
||||
return f, invalid
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (s *service) serve(w http.ResponseWriter, r *http.Request) {
|
||||
if _, exists := s.projects[r.PathValue("project")]; !exists {
|
||||
reject(w, 404, "not_found")
|
||||
return
|
||||
}
|
||||
var filters Filters
|
||||
var id int64
|
||||
var err error
|
||||
if value := r.PathValue("id"); value != "" {
|
||||
if !positiveInteger.MatchString(value) || r.URL.RawQuery != "" {
|
||||
reject(w, 400, "invalid_request")
|
||||
return
|
||||
}
|
||||
id, err = strconv.ParseInt(value, 10, 64)
|
||||
} else {
|
||||
filters, err = parseFilters(r.URL.RawQuery)
|
||||
}
|
||||
if err != nil {
|
||||
reject(w, 400, "invalid_request")
|
||||
return
|
||||
}
|
||||
select {
|
||||
case s.slots <- struct{}{}:
|
||||
defer func() { <-s.slots }()
|
||||
default:
|
||||
w.Header().Set("Retry-After", "1")
|
||||
reject(w, 429, "research_busy")
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), s.timeout)
|
||||
defer cancel()
|
||||
var result any
|
||||
if id != 0 {
|
||||
result, err = s.repository.Get(ctx, id)
|
||||
} else {
|
||||
result, err = s.repository.List(ctx, filters)
|
||||
}
|
||||
if err == nil {
|
||||
reply(w, 200, result)
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case errors.Is(err, sql.ErrNoRows):
|
||||
reject(w, 404, "not_found")
|
||||
case errors.Is(ctx.Err(), context.DeadlineExceeded), errors.Is(err, context.DeadlineExceeded):
|
||||
reject(w, 504, "research_timeout")
|
||||
default:
|
||||
reject(w, 503, "research_unavailable")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
package research
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type fixtureRepository struct {
|
||||
filters Filters
|
||||
id int64
|
||||
err error
|
||||
wait bool
|
||||
calls int
|
||||
}
|
||||
|
||||
func (f *fixtureRepository) List(ctx context.Context, filters Filters) (Page, error) {
|
||||
f.calls++
|
||||
f.filters = filters
|
||||
if f.wait {
|
||||
<-ctx.Done()
|
||||
return Page{}, ctx.Err()
|
||||
}
|
||||
return Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}, f.err
|
||||
}
|
||||
|
||||
func (f *fixtureRepository) Get(ctx context.Context, id int64) (Paper, error) {
|
||||
f.calls++
|
||||
f.id = id
|
||||
return Paper{ID: "42", Title: "Original title", Authors: []string{}, Topics: []string{}, Sources: []Source{}}, f.err
|
||||
}
|
||||
|
||||
func request(handler http.Handler, method, path string) *httptest.ResponseRecorder {
|
||||
w := httptest.NewRecorder()
|
||||
handler.ServeHTTP(w, httptest.NewRequest(method, path, nil))
|
||||
return w
|
||||
}
|
||||
|
||||
func TestAnonymousReadRoutes(t *testing.T) {
|
||||
f := &fixtureRepository{}
|
||||
mux := http.NewServeMux()
|
||||
Register(mux, f, map[string]string{"thesisly": "http://localhost:17322/auth/callback"})
|
||||
w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&page=2&pageSize=10")
|
||||
if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", Page: 2, PageSize: 10}) {
|
||||
t.Fatalf("list: %d %+v", w.Code, f.filters)
|
||||
}
|
||||
var page Page
|
||||
if err := json.Unmarshal(w.Body.Bytes(), &page); err != nil || page.Items == nil || page.Topics == nil {
|
||||
t.Fatalf("invalid page: %s", w.Body.String())
|
||||
}
|
||||
if w.Header().Get("Cache-Control") != "no-store" || w.Header().Get("Content-Type") != "application/json" {
|
||||
t.Fatal("missing response headers")
|
||||
}
|
||||
w = request(mux, "GET", "/projects/thesisly/research/papers/42")
|
||||
if w.Code != 200 || f.id != 42 || !strings.Contains(w.Body.String(), `"abstract":null`) {
|
||||
t.Fatalf("detail: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
if got := request(mux, "POST", "/projects/thesisly/research/papers").Code; got != 405 {
|
||||
t.Fatalf("mutation status %d", got)
|
||||
}
|
||||
if got := request(mux, "GET", "/projects/other/research/papers").Code; got != 404 {
|
||||
t.Fatalf("unknown project status %d", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidQueriesNeverReachRepository(t *testing.T) {
|
||||
queries := []string{"unknown=1", "q=x&q=y", "q=%ff", "q=%00", "q=%zz", "page=0", "page=1001", "page=-1", "page=1.5", "page=01", "pageSize=51", "pageSize=0", "pageSize=9999999999999999999999", "searchIn=sql", "sort=published", "topic=arxiv:cs.LG", "q=" + url.QueryEscape(strings.Repeat("字", 201)), "q=" + url.QueryEscape(strings.Repeat("word ", 21))}
|
||||
for _, query := range queries {
|
||||
t.Run(query, func(t *testing.T) {
|
||||
f := &fixtureRepository{}
|
||||
mux := http.NewServeMux()
|
||||
Register(mux, f, map[string]string{"thesisly": "unused"})
|
||||
w := request(mux, "GET", "/projects/thesisly/research/papers?"+query)
|
||||
if w.Code != 400 || f.calls != 0 {
|
||||
t.Fatalf("status %d, calls %d", w.Code, f.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
for _, path := range []string{"0", "-1", "1x", "9223372036854775808", "1?extra=1"} {
|
||||
mux := http.NewServeMux()
|
||||
f := &fixtureRepository{}
|
||||
Register(mux, f, map[string]string{"thesisly": "unused"})
|
||||
if w := request(mux, "GET", "/projects/thesisly/research/papers/"+path); w.Code != 400 || f.calls != 0 {
|
||||
t.Fatalf("invalid id %q: %d", path, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryDefaultsAndUnicode(t *testing.T) {
|
||||
f, err := parseFilters("q=" + url.QueryEscape(" 论文 Graph "))
|
||||
if err != nil || f.Query != "论文 Graph" || f.Page != 1 || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" {
|
||||
t.Fatalf("%+v %v", f, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStorageErrorsAreRedacted(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
err error
|
||||
status int
|
||||
code string
|
||||
}{
|
||||
{sql.ErrNoRows, 404, "not_found"},
|
||||
{context.DeadlineExceeded, 504, "research_timeout"},
|
||||
{errors.New("postgres://private-user:private-password@db/corpus"), 503, "research_unavailable"},
|
||||
} {
|
||||
mux := http.NewServeMux()
|
||||
Register(mux, &fixtureRepository{err: test.err}, map[string]string{"thesisly": "unused"})
|
||||
w := request(mux, "GET", "/projects/thesisly/research/papers/42")
|
||||
if w.Code != test.status || !strings.Contains(w.Body.String(), test.code) || strings.Contains(w.Body.String(), "private") {
|
||||
t.Fatalf("%d %s", w.Code, w.Body.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestConcurrencyAndTimeoutReleaseSlots(t *testing.T) {
|
||||
s := &service{repository: &fixtureRepository{wait: true}, projects: map[string]string{"thesisly": "unused"}, slots: make(chan struct{}, 2), timeout: time.Millisecond}
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /projects/{project}/research/papers", s.serve)
|
||||
s.slots <- struct{}{}
|
||||
s.slots <- struct{}{}
|
||||
w := request(mux, "GET", "/projects/thesisly/research/papers")
|
||||
if w.Code != 429 || w.Header().Get("Retry-After") != "1" {
|
||||
t.Fatalf("busy %d", w.Code)
|
||||
}
|
||||
<-s.slots
|
||||
<-s.slots
|
||||
w = request(mux, "GET", "/projects/thesisly/research/papers")
|
||||
if w.Code != 504 || len(s.slots) != 0 {
|
||||
t.Fatalf("timeout %d, slots %d", w.Code, len(s.slots))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user