From 026ee22c1e1292087cbfa77bdc9a1ed5e40f1585 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sat, 7 Nov 2026 17:27:35 +0800 Subject: [PATCH] Page research without exact counts and bound candidate scans [skip ci] --- docs/research-api.md | 69 +++++- internal/research/cursor.go | 65 ++++++ internal/research/cursor_test.go | 34 +++ internal/research/http.go | 35 ++- internal/research/http_test.go | 22 +- internal/research/postgres.go | 205 +++++++++++++++--- .../research/postgres_integration_test.go | 197 +++++++++++++++-- internal/research/testdata/corpus.sql | 22 +- 8 files changed, 563 insertions(+), 86 deletions(-) create mode 100644 internal/research/cursor.go create mode 100644 internal/research/cursor_test.go diff --git a/docs/research-api.md b/docs/research-api.md index 08217ed..1d6f2eb 100644 --- a/docs/research-api.md +++ b/docs/research-api.md @@ -46,7 +46,8 @@ not require a session; these routes do not expose private data or accept writes. List parameters are `q` (200 Unicode characters, 20 words), `searchIn` (`text` or `author`), `topic` (an exact arXiv category), `sort` (`recent` or -`title`), `page` (1–1000), and `pageSize` (1–50, default 8). Unknown, malformed +`title`), `cursor` (optional, opaque, at most 32768 characters), and +`pageSize` (1–50, default 8). Unknown, malformed or repeated parameters return 400. `recent` means descending stable paper ID, not publication date. Search terms must become bound query parameters, never raw SQL or user-supplied full-text syntax. @@ -61,17 +62,35 @@ those two missing access paths. It is separate from Core startup and auth migrations; do not grant DDL to the runtime reader. Both indexes completed on the confirmed ECS (category 35.4 seconds, author 436.47 seconds); measured end-to-end checks are documented below. -List responses contain `items`, `total`, `topics`, `page`, and `pageSize`. +List responses contain `items`, `nextCursor`, `previousCursor`, `topics`, and +`pageSize`. Empty cursor strings mean that direction is unavailable. There is +no `total`, page number, total page count or deep-page cap. The old `page` +parameter is rejected; coordinate the frontend/backend contract update. Each paper preserves original title/abstract, authors, source dates, topics and source links. Publication decisions need source evidence; absent decisions stay unknown. No invented repositories, metrics or generated research conclusions. -The repository counts and reads each list in one read-only snapshot. -Paging first materializes matching IDs, then joins only that page to the wide -paper rows. Filtered searches first materialize matching IDs without an -ORDER BY/LIMIT, then sort/page those IDs, preventing sparse terms from choosing -a backward heap scan with repeated text-vector computation. Activity sampling showed that selecting full rows before LIMIT -caused sparse category queries to read large portions of the paper heap. The +Lists use a read-only repeatable-read snapshot without `count(*)` or `OFFSET`. +Keyset paging uses ID descending for recent order and `(title_key, id)` ascending +for title order, including a reverse query for previous pages. The title key is +kept in its stored codec and C collation. Cursors bind normalized query, search +scope, topic, sort and page size; changing filters requires a fresh cursor. They +are public row positions, not authorization tokens or guaranteed frozen snapshots +across requests. Concurrent corpus edits may alter later pages. + +Only pageSize+1 positions are selected; the extra position indicates more data, +then full rows and batched metadata are loaded for at most pageSize papers. +For recent order with only a topic filter, page directly through the existing +`(tag_id,paper_id)` index and deduplicate evidence before LIMIT. For text/author +search in recent order, try an ID interval of 512 near the cursor, expanding +once to 4096 only if there are some matches but fewer than pageSize+1. +Only a full page plus one proves that interval contains the next page. If neither bounded attempt fills the page, +all local candidates are discarded and the complete indexed search is used; +rare or older matches are never silently truncated. The fallback materializes +matching IDs before sorting, avoiding an unbounded ordered heap scan for sparse +terms. Title-order filtered queries still use that complete search and can time +out for broad terms. No global database or proxy resource settings are changed. +The HTTP layer permits two in-flight queries with an eight-second context budget. Busy requests return 429 with `Retry-After: 1`; deadlines return 504; storage errors return a generic 503 without database details. Missing papers return 404. @@ -95,7 +114,7 @@ and homepage HTTP routes have returned 200. The frontend has also rendered the actual 3,526,999-paper corpus through a loopback SSH tunnel: eight rows, matching detail title/source, desktop and mobile layout, no page exceptions or horizontal overflow in that check. -Author search returned 146 records in 0.336 seconds. After paging/author fixes, category `cs.AR` returned 9,457 records in 4.906 +The earlier count-based API returned 146 author records in 0.336 seconds. Before the cursor optimization, after paging/author fixes, category `cs.AR` returned 9,457 records in 4.906 seconds, `CartoonGAN` returned three in 0.332 seconds, and the multi-term search `attention is all you need` returned 8,653 in 6.325 seconds. These were sequential manual HTTP checks, not a load test. Very broad text searches can still exceed @@ -143,3 +162,35 @@ reload/new-tab persistence and logout behavior. Browser control was unavailable, so these browser results are attributed to user acceptance rather than an automated real-account test. This does not deploy the research work branch; mail delivery was not exercised. + + +## Cursor regression coverage + +The disposable PostgreSQL fixture covers forward/backward traversal, identical +title keys with different IDs, published control-character encoding, sparse +matches separated by a large ID gap, and fallback from the bounded interval. +Invalid, oversized, cross-filter and wrong-version cursors are rejected before +repository execution. The runtime reader retains SELECT-only privileges; no DDL +or writes are added. Run tests manually on the confirmed ECS; skip CI for this +iteration and retain `[skip ci]` in commit and PR titles. + + +## Manual cursor measurements (2026-11-07) + +On the confirmed ECS, after this task's compilation and disposable fixture had +finished, sequential direct requests to the temporary Core returned: homepage +65 ms, next/previous 39/21 ms, `graph` 345 ms, `CartoonGAN` 731 ms (3 papers), +`cs.AR` 145 ms, title order 180 ms, author `Buhrman` 769 ms, and +`attention is all you need` 1312 ms. Recent, topic and title cursor round trips +returned the same first page without overlap. These are individual manual +samples, not production latency guarantees; concurrent compilation earlier +produced multi-second cold measurements. No proxy, database memory, CPU or +production configuration was changed. + +The final source compiled manually on the ECS; package unit checks and all ten +selected cursor/PostgreSQL integration tests passed, including interval expansion +and complete fallback. No CI or local build ran. The local authorized Next dev +uses the temporary API through the existing SSH tunnel. Independent Chrome also +verified real homepage/search navigation, a preserved document, one mobile +header/main, no page exceptions and no horizontal overflow. The production Core +was not replaced by these temporary preview updates. diff --git a/internal/research/cursor.go b/internal/research/cursor.go new file mode 100644 index 0000000..4efd034 --- /dev/null +++ b/internal/research/cursor.go @@ -0,0 +1,65 @@ +package research + +import ( + "bytes" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "io" + "strconv" + "strings" +) + +// Cursors locate a public corpus row, not an account or authentication token. +// Binding to normalized filters prevents accidentally paging a different query. +type position struct { + Version int `json:"v"` + ID int64 `json:"id"` + Key string `json:"key"` + Before bool `json:"before"` + Filter string `json:"filter"` +} + +var errCursor = errors.New("invalid cursor") + +func filterKey(f Filters) string { + sort, search := f.Sort, f.SearchIn + if sort == "" { + sort = "recent" + } + if search == "" { + search = "text" + } + data, _ := json.Marshal([]string{strings.TrimSpace(f.Query), f.Topic, sort, search, strconv.Itoa(f.PageSize)}) + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:16]) +} +func encodeCursor(f Filters, p position, before bool) string { + p.Version, p.Before, p.Filter = 1, before, filterKey(f) + if f.Sort != "title" { + p.Key = "" + } + data, _ := json.Marshal(p) + return base64.RawURLEncoding.EncodeToString(data) +} +func decodeCursor(f Filters) (position, error) { + var p position + if f.Cursor == "" { + return p, nil + } + if len(f.Cursor) > 32768 { + return p, errCursor + } + data, err := base64.RawURLEncoding.Strict().DecodeString(f.Cursor) + if err != nil { + return p, errCursor + } + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if decoder.Decode(&p) != nil || decoder.Decode(new(any)) != io.EOF || p.Version != 1 || p.ID <= 0 || p.Filter != filterKey(f) || strings.ContainsRune(p.Key, 0) || (f.Sort != "title" && p.Key != "") { + return position{}, errCursor + } + return p, nil +} diff --git a/internal/research/cursor_test.go b/internal/research/cursor_test.go new file mode 100644 index 0000000..4f26d16 --- /dev/null +++ b/internal/research/cursor_test.go @@ -0,0 +1,34 @@ +package research + +import ( + "encoding/base64" + "net/url" + "strings" + "testing" +) + +func TestCursorValidation(t *testing.T) { + f := Filters{Query: "graph", Sort: "title", SearchIn: "text", PageSize: 8} + f.Cursor = encodeCursor(f, position{ID: 9223372036854775807, Key: "duplicate\x01title"}, true) + p, err := decodeCursor(f) + if err != nil || !p.Before || p.ID != 9223372036854775807 || p.Key != "duplicate\x01title" { + t.Fatalf("roundtrip %+v %v", p, err) + } + if _, err := parseFilters("q=graph&sort=title&cursor=" + url.QueryEscape(f.Cursor)); err != nil { + t.Fatal(err) + } + for _, raw := range []string{"!", strings.Repeat("a", 32769), base64.RawURLEncoding.EncodeToString([]byte(`{"v":2}`)), f.Cursor + "="} { + bad := f + bad.Cursor = raw + if _, err := decodeCursor(bad); err == nil { + t.Fatal("accepted invalid cursor") + } + } + for _, change := range []func(*Filters){func(f *Filters) { f.Query = "other" }, func(f *Filters) { f.Sort = "recent" }, func(f *Filters) { f.PageSize = 9 }, func(f *Filters) { f.SearchIn = "author" }, func(f *Filters) { f.Topic = "cs.AI" }} { + bad := f + change(&bad) + if _, err := decodeCursor(bad); err == nil { + t.Fatal("accepted mismatched filter") + } + } +} diff --git a/internal/research/http.go b/internal/research/http.go index 9f86ac1..97820b8 100644 --- a/internal/research/http.go +++ b/internal/research/http.go @@ -49,20 +49,20 @@ type Filters struct { SearchIn string Topic string Sort string - Page int + Cursor string PageSize int } type Page struct { - Items []Paper `json:"items"` - Total int64 `json:"total"` - Topics []string `json:"topics"` - Page int `json:"page"` - PageSize int `json:"pageSize"` + Items []Paper `json:"items"` + NextCursor string `json:"nextCursor"` + PreviousCursor string `json:"previousCursor"` + Topics []string `json:"topics"` + PageSize int `json:"pageSize"` } // Repository implementations must use a read-only transaction for each List, -// so its count, topics and page describe the same database snapshot. +// so its metadata and page describe the same database snapshot. type Repository interface { List(context.Context, Filters) (Page, error) Get(context.Context, int64) (Paper, error) @@ -102,7 +102,7 @@ var topicPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.-]{0,79}$`) var positiveInteger = regexp.MustCompile(`^[1-9][0-9]*$`) func parseFilters(raw string) (Filters, error) { - f := Filters{SearchIn: "text", Sort: "recent", Page: 1, PageSize: 8} + f := Filters{SearchIn: "text", Sort: "recent", PageSize: 8} values, err := url.ParseQuery(raw) if err != nil { return f, err @@ -133,28 +133,23 @@ func parseFilters(raw string) (Filters, error) { return f, invalid } f.Sort = value[0] - case "page", "pageSize": + case "cursor": + f.Cursor = value[0] + case "pageSize": if !positiveInteger.MatchString(value[0]) { return f, invalid } n, err := strconv.Atoi(value[0]) - limit := 1000 - if key == "pageSize" { - limit = 50 - } - if err != nil || n > limit { + if err != nil || n > 50 { return f, invalid } - if key == "page" { - f.Page = n - } else { - f.PageSize = n - } + f.PageSize = n default: return f, invalid } } - return f, nil + _, err = decodeCursor(f) + return f, err } func (s *service) serve(w http.ResponseWriter, r *http.Request) { diff --git a/internal/research/http_test.go b/internal/research/http_test.go index b9267fc..09873e8 100644 --- a/internal/research/http_test.go +++ b/internal/research/http_test.go @@ -28,7 +28,7 @@ func (f *fixtureRepository) List(ctx context.Context, filters Filters) (Page, er <-ctx.Done() return Page{}, ctx.Err() } - return Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}, f.err + return Page{Items: []Paper{}, Topics: []string{}, PageSize: filters.PageSize}, f.err } func (f *fixtureRepository) Get(ctx context.Context, id int64) (Paper, error) { @@ -47,8 +47,8 @@ func TestAnonymousReadRoutes(t *testing.T) { f := &fixtureRepository{} mux := http.NewServeMux() Register(mux, f, map[string]string{"thesisly": "http://localhost:17322/auth/callback"}) - w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&page=2&pageSize=10") - if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", Page: 2, PageSize: 10}) { + w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&pageSize=10") + if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", PageSize: 10}) { t.Fatalf("list: %d %+v", w.Code, f.filters) } var page Page @@ -95,7 +95,7 @@ func TestInvalidQueriesNeverReachRepository(t *testing.T) { func TestQueryDefaultsAndUnicode(t *testing.T) { f, err := parseFilters("q=" + url.QueryEscape(" 论文 Graph ")) - if err != nil || f.Query != "论文 Graph" || f.Page != 1 || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" { + if err != nil || f.Query != "论文 Graph" || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" { t.Fatalf("%+v %v", f, err) } } @@ -143,3 +143,17 @@ func TestConcurrencyAndTimeoutReleaseSlots(t *testing.T) { t.Fatalf("timeout %d, slots %d", w.Code, len(s.slots)) } } + +func TestInvalidCursorsNeverReachRepository(t *testing.T) { + f := Filters{Query: "graph", Sort: "recent", SearchIn: "text", PageSize: 8} + valid := encodeCursor(f, position{ID: 42}, false) + for _, query := range []string{"cursor=!", "cursor=" + strings.Repeat("a", 32769), "q=different&cursor=" + valid, "cursor=" + valid + "&cursor=" + valid, "page=1"} { + repository := &fixtureRepository{} + mux := http.NewServeMux() + Register(mux, repository, map[string]string{"thesisly": "unused"}) + w := request(mux, "GET", "/projects/thesisly/research/papers?"+query) + if w.Code != 400 || repository.calls != 0 { + t.Fatalf("invalid cursor reached storage: status=%d calls=%d", w.Code, repository.calls) + } + } +} diff --git a/internal/research/postgres.go b/internal/research/postgres.go index 7198dc2..feb4898 100644 --- a/internal/research/postgres.go +++ b/internal/research/postgres.go @@ -173,8 +173,31 @@ func decodeAuthors(raw string) ([]string, error) { return names, nil } +func (r *PostgresRepository) positions(ctx context.Context, tx *sql.Tx, query string, args []any) ([]position, error) { + rows, err := tx.QueryContext(ctx, r.query(query), args...) + if err != nil { + return nil, err + } + defer rows.Close() + positions := []position{} + for rows.Next() { + var p position + // Preserve the stored title key, including the published text codec. + if err = rows.Scan(&p.ID, &p.Key); err != nil { + return nil, err + } + positions = append(positions, p) + } + return positions, rows.Err() +} + func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, error) { - result := Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize} + result := Page{Items: []Paper{}, Topics: []string{}, PageSize: filters.PageSize} + cursor, err := decodeCursor(filters) + if err != nil { + return result, err + } + original := filters tx, err := r.begin(ctx) if err != nil { return result, err @@ -185,56 +208,168 @@ func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, e filters.Topic = encodeText(filters.Topic) } where, args := whereClause(filters) - if err = tx.QueryRowContext(ctx, r.query(`SELECT count(*) FROM {schema}papers p`+where), args...).Scan(&result.Total); err != nil { - return result, err + if cursor.ID != 0 { + clause := " AND " + if where == "" { + clause = " WHERE " + } + op := "<" + if cursor.Before { + op = ">" + } + args = append(args, cursor.ID) + if filters.Sort == "title" { + if cursor.Before { + op = "<" + } else { + op = ">" + } + args = append(args, cursor.Key) + where += fmt.Sprintf("%s(p.title_key,p.id) %s ($%d::text COLLATE \"C\",$%d)", clause, op, len(args), len(args)-1) + } else { + where += fmt.Sprintf("%sp.id %s $%d", clause, op, len(args)) + } } order := "p.id DESC" if filters.Sort == "title" { order = "p.title_key,p.id" } - args = append(args, filters.PageSize, (filters.Page-1)*filters.PageSize) - // Select IDs before loading potentially TOASTed titles/abstracts. With - // sparse categories, selecting full rows here can make the planner walk - // hundreds of thousands of wide heap rows before finding eight matches. - candidateQuery := `SELECT p.id FROM {schema}papers p` - candidateOrder := order - prefix := "" - if where != "" { - columns := "p.id" + if cursor.Before { + order = "p.id" if filters.Sort == "title" { - columns += ",p.title_key" + order = "p.title_key DESC,p.id DESC" } - // Apply search before LIMIT/ORDER BY so a sparse term cannot turn - // into a backward heap scan that recomputes vectors for millions. - prefix = fmt.Sprintf(`matches AS MATERIALIZED (SELECT %s FROM {schema}papers p%s), `, columns, where) - candidateQuery = "SELECT id FROM matches" - candidateOrder = strings.ReplaceAll(order, "p.", "") } - query := fmt.Sprintf(`WITH %spage_ids AS MATERIALIZED ( - %s ORDER BY %s LIMIT $%d OFFSET $%d - ) SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published - FROM page_ids JOIN {schema}papers p ON p.id=page_ids.id ORDER BY %s`, prefix, candidateQuery, candidateOrder, len(args)-1, len(args), order) - rows, err := tx.QueryContext(ctx, r.query(query), args...) - if err != nil { - return result, err + args = append(args, filters.PageSize+1) + columns := "p.id,p.title_key" + if filters.Sort != "title" { + columns = "p.id, ''::text AS title_key" } - for rows.Next() { - p, err := r.scanPaper(rows) + // Search candidates are materialized to avoid scanning millions of wide + // rows in ID order for sparse terms. There is no count or deep OFFSET. + query := fmt.Sprintf("SELECT %s FROM {schema}papers p%s ORDER BY %s LIMIT $%d", columns, where, order, len(args)) + if filters.Query != "" || filters.Topic != "" { + query = fmt.Sprintf("WITH matches AS MATERIALIZED (SELECT %s FROM {schema}papers p%s) SELECT p.id,p.title_key FROM matches p ORDER BY %s LIMIT $%d", columns, where, order, len(args)) + } + positions := []position{} + topicOnly := filters.Sort != "title" && filters.Topic != "" && filters.Query == "" + if topicOnly { + // The published paper_tags foreign key guarantees these IDs exist. + // Walk the (tag_id,paper_id) index instead of joining all matching + // paper rows before LIMIT. DISTINCT removes duplicate source evidence. + topicArgs := []any{filters.Topic, filters.PageSize + 1} + predicate := "" + if cursor.ID != 0 { + op := "<" + if cursor.Before { + op = ">" + } + topicArgs = append(topicArgs, cursor.ID) + predicate = " AND pt.paper_id " + op + " $3" + } + direction := "DESC" + if cursor.Before { + direction = "ASC" + } + query := fmt.Sprintf(`SELECT DISTINCT pt.paper_id,''::text AS title_key FROM {schema}paper_tags pt + WHERE pt.tag_id=(SELECT id FROM {schema}tags WHERE namespace='arxiv:category' AND value=$1)%s + ORDER BY pt.paper_id %s LIMIT $2`, predicate, direction) + positions, err = r.positions(ctx, tx, query, topicArgs) if err != nil { - _ = rows.Close() return result, err } - result.Items = append(result.Items, p) } - err = rows.Err() - _ = rows.Close() - if err != nil { - return result, err + if !topicOnly && filters.Sort != "title" && (filters.Query != "" || filters.Topic != "") { + // Common terms can fill a page near its cursor. Probe a bounded ID + // interval first; never scan the entire heap in order for a rare term. + // If the interval has fewer than size+1 matches, discard it and use + // the complete indexed search below. No matches are silently omitted. + anchor := cursor.ID + if anchor == 0 { + if err = tx.QueryRowContext(ctx, r.query(`SELECT coalesce(max(id),0) FROM {schema}papers`)).Scan(&anchor); err != nil { + return result, err + } + } + for _, window := range []int64{512, 4096} { + bound, op := int64(0), ">=" + if anchor > window { + bound = anchor - window + } + if cursor.Before { + bound, op = int64(9223372036854775807), "<=" + if anchor <= 9223372036854775807-window { + bound = anchor + window + } + } + boundedArgs := append(append([]any{}, args...), bound) + bounded := fmt.Sprintf("SELECT %s FROM {schema}papers p%s AND p.id %s $%d ORDER BY %s LIMIT $%d", columns, where, op, len(boundedArgs), order, len(args)) + positions, err = r.positions(ctx, tx, bounded, boundedArgs) + if err != nil { + return result, err + } + // Expand once only when there is evidence of nearby matches. + // Zero matches goes straight to the sparse-term index fallback. + if len(positions) == 0 || len(positions) >= filters.PageSize+1 { + break + } + } + } + if !topicOnly && len(positions) < filters.PageSize+1 { + positions, err = r.positions(ctx, tx, query, args) + if err != nil { + return result, err + } + } + more := len(positions) > filters.PageSize + if more { + positions = positions[:filters.PageSize] + } + if cursor.Before { + for i, j := 0, len(positions)-1; i < j; i, j = i+1, j-1 { + positions[i], positions[j] = positions[j], positions[i] + } + } + if len(positions) > 0 { + if (!cursor.Before && cursor.ID != 0) || (cursor.Before && more) { + result.PreviousCursor = encodeCursor(original, positions[0], true) + } + if (!cursor.Before && more) || (cursor.Before && cursor.ID != 0) { + result.NextCursor = encodeCursor(original, positions[len(positions)-1], false) + } + ids := make([]int64, len(positions)) + byID := map[string]Paper{} + for i, p := range positions { + ids[i] = p.ID + } + rows, err := tx.QueryContext(ctx, r.query(`SELECT id,title,abstract,authors_json,first_published FROM {schema}papers WHERE id=ANY($1)`), ids) + if err != nil { + return result, err + } + for rows.Next() { + p, scanErr := r.scanPaper(rows) + if scanErr != nil { + _ = rows.Close() + return result, scanErr + } + byID[p.ID] = p + } + err = rows.Err() + _ = rows.Close() + if err != nil { + return result, err + } + for _, p := range positions { + paper, exists := byID[strconv.FormatInt(p.ID, 10)] + if !exists { + return result, errors.New("inconsistent paper index") + } + result.Items = append(result.Items, paper) + } } if err = r.enrich(ctx, tx, result.Items); err != nil { return result, err } - rows, err = tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`)) + rows, err := tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`)) if err != nil { return result, err } @@ -271,7 +406,7 @@ func (r *PostgresRepository) Get(ctx context.Context, id int64) (Paper, error) { return papers[0], tx.Commit() } -// Batch metadata queries avoid per-paper round trips and share the count/page +// Batch metadata queries avoid per-paper round trips and share the page // snapshot. User values are bound; only the validated schema is interpolated. func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Paper) error { if len(papers) == 0 { diff --git a/internal/research/postgres_integration_test.go b/internal/research/postgres_integration_test.go index 65c999b..97ee7e4 100644 --- a/internal/research/postgres_integration_test.go +++ b/internal/research/postgres_integration_test.go @@ -50,31 +50,28 @@ func TestPostgresPageSearchAndFilters(t *testing.T) { } for _, test := range []struct { filters Filters - total int64 ids []string }{ - {Filters{Page: 1, PageSize: 2}, 5, []string{"5", "4"}}, - {Filters{Page: 2, PageSize: 2}, 5, []string{"3", "2"}}, - {Filters{Page: 3, PageSize: 2}, 5, []string{"1"}}, - {Filters{Page: 1, PageSize: 2, Sort: "title"}, 5, []string{"2", "5"}}, - {Filters{Page: 1, PageSize: 8, Query: "graph"}, 2, []string{"3", "1"}}, - {Filters{Page: 1, PageSize: 8, Query: "graph OR zebra"}, 0, []string{}}, - {Filters{Page: 1, PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, 1, []string{"1"}}, - {Filters{Page: 1, PageSize: 8, Topic: "cs.LG"}, 1, []string{"1"}}, - {Filters{Page: 1, PageSize: 8, Topic: "cs.DC", Query: "graph"}, 1, []string{"3"}}, - {Filters{Page: 1, PageSize: 8, Query: `x'); DROP TABLE papers;--`}, 0, []string{}}, + {Filters{PageSize: 2}, []string{"5", "4"}}, + {Filters{PageSize: 2, Sort: "title"}, []string{"2", "5"}}, + {Filters{PageSize: 8, Query: "graph"}, []string{"3", "1"}}, + {Filters{PageSize: 8, Query: "graph OR zebra"}, []string{}}, + {Filters{PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, []string{"1"}}, + {Filters{PageSize: 8, Topic: "cs.LG"}, []string{"1"}}, + {Filters{PageSize: 8, Topic: "cs.DC", Query: "graph"}, []string{"3"}}, + {Filters{PageSize: 8, Query: `x'); DROP TABLE papers;--`}, []string{}}, } { page, err := r.List(ctx, test.filters) if err != nil { t.Fatalf("%+v: %v", test.filters, err) } - if page.Total != test.total || !reflect.DeepEqual(paperIDs(page.Items), test.ids) { - t.Fatalf("%+v: total=%d ids=%v", test.filters, page.Total, paperIDs(page.Items)) + if !reflect.DeepEqual(paperIDs(page.Items), test.ids) { + t.Fatalf("%+v: ids=%v", test.filters, paperIDs(page.Items)) } if !reflect.DeepEqual(page.Topics, []string{"cs.AI", "cs.DC", "cs.LG"}) { t.Fatalf("mixed tag namespaces: %v", page.Topics) } - if page.Page != test.filters.Page || page.PageSize != test.filters.PageSize { + if page.PageSize != test.filters.PageSize { t.Fatal("lost paging metadata") } } @@ -143,7 +140,7 @@ func TestPostgresCorpusIsReadOnlyAndHTTPWorks(t *testing.T) { Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"}) w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph") var page Page - if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || page.Total != 2 { + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || len(page.Items) != 2 { t.Fatalf("SQL-backed HTTP list: %d %s", w.Code, w.Body.String()) } w = request(mux, "GET", "/projects/thesisly/research/papers/1") @@ -179,8 +176,174 @@ func TestPostgresPublishedTextCodec(t *testing.T) { if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &decoded) != nil || decoded.Title != p.Title || decoded.Abstract == nil || *decoded.Abstract != *p.Abstract { t.Fatalf("HTTP lost original control characters: %d %s", w.Code, w.Body.String()) } - page, err := r.List(ctx, Filters{Page: 1, PageSize: 8, Query: "graph"}) - if err != nil || page.Total != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title { + page, err := r.List(ctx, Filters{PageSize: 8, Query: "graph"}) + if err != nil || len(page.Items) != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title { t.Fatalf("encoded corpus search: %+v %v", page, err) } } + +func TestPostgresCursorsTraverseBothDirections(t *testing.T) { + r, ctx := integrationRepository(t) + for _, sort := range []string{"recent", "title"} { + f := Filters{Sort: sort, PageSize: 1} + seen := []string{} + cursors := []string{""} + for { + page, err := r.List(ctx, f) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 { + t.Fatalf("unexpected page %+v", page) + } + seen = append(seen, page.Items[0].ID) + if len(seen) > 5 { + t.Fatal("cursor cycle") + } + if len(seen) > 1 { + back := f + back.Cursor = page.PreviousCursor + prior, err := r.List(ctx, back) + if err != nil || len(prior.Items) != 1 || prior.Items[0].ID != seen[len(seen)-2] { + t.Fatalf("reverse cursor: %+v %v", prior, err) + } + } else if page.PreviousCursor != "" { + t.Fatal("first page has previous") + } + if page.NextCursor == "" { + break + } + f.Cursor = page.NextCursor + cursors = append(cursors, f.Cursor) + } + want := []string{"5", "4", "3", "2", "1"} + if sort == "title" { + want = []string{"2", "5", "1", "3", "4"} + } + if !reflect.DeepEqual(seen, want) { + t.Fatalf("%s: %v", sort, seen) + } + f.Cursor = cursors[1] + f.Query = "different" + if _, err := r.List(ctx, f); err == nil { + t.Fatal("cursor reused across filters") + } + } +} + +func TestPostgresEncodedTitleCursors(t *testing.T) { + legacy, ctx := integrationRepository(t) + r, err := NewPostgresRepository(legacy.db, "research_encoded_fixture") + if err != nil { + t.Fatal(err) + } + if err = r.Check(ctx); err != nil { + t.Fatal(err) + } + f := Filters{Sort: "title", PageSize: 1} + seen := []string{} + for { + page, err := r.List(ctx, f) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 { + t.Fatalf("invalid encoded title page %+v", page) + } + seen = append(seen, page.Items[0].ID) + if len(seen) > 5 { + t.Fatal("cursor cycle") + } + if page.NextCursor == "" { + break + } + f.Cursor = page.NextCursor + } + if !reflect.DeepEqual(seen, []string{"2", "5", "1", "3", "4"}) { + t.Fatalf("encoded key paging %v", seen) + } +} + +func TestPostgresBoundedProbeDoesNotTruncateSparseMatches(t *testing.T) { + legacy, ctx := integrationRepository(t) + r, err := NewPostgresRepository(legacy.db, "research_sparse_fixture") + if err != nil { + t.Fatal(err) + } + if err = r.Check(ctx); err != nil { + t.Fatal(err) + } + for _, size := range []int{1, 2, 3} { + f := Filters{Query: "graph", PageSize: size} + seen := []string{} + var previous []string + for { + page, err := r.List(ctx, f) + if err != nil { + t.Fatal(err) + } + ids := paperIDs(page.Items) + if len(ids) == 0 || len(ids) > size { + t.Fatalf("invalid sparse page %+v", page) + } + if previous != nil { + back := f + back.Cursor = page.PreviousCursor + prior, err := r.List(ctx, back) + if err != nil || !reflect.DeepEqual(paperIDs(prior.Items), previous) { + t.Fatalf("reverse gap %+v %v", prior, err) + } + } + seen = append(seen, ids...) + if len(seen) > 5 { + t.Fatal("cursor cycle") + } + previous = ids + if page.NextCursor == "" { + break + } + f.Cursor = page.NextCursor + } + if !reflect.DeepEqual(seen, []string{"10001", "10000", "8000", "3", "1"}) { + t.Fatalf("size=%d lost sparse matches %v", size, seen) + } + } +} + +func TestPostgresTopicCursorSkipsDuplicateEvidence(t *testing.T) { + legacy, ctx := integrationRepository(t) + r, err := NewPostgresRepository(legacy.db, "research_sparse_fixture") + if err != nil { + t.Fatal(err) + } + f := Filters{Topic: "cs.LG", PageSize: 1} + seen := []string{} + for { + page, err := r.List(ctx, f) + if err != nil { + t.Fatal(err) + } + if len(page.Items) != 1 { + t.Fatalf("invalid topic page %+v", page) + } + seen = append(seen, page.Items[0].ID) + if len(seen) > 3 { + t.Fatal("cursor cycle") + } + if len(seen) > 1 { + back := f + back.Cursor = page.PreviousCursor + prior, err := r.List(ctx, back) + if err != nil || len(prior.Items) != 1 || prior.Items[0].ID != seen[len(seen)-2] { + t.Fatalf("reverse topic %+v %v", prior, err) + } + } + if page.NextCursor == "" { + break + } + f.Cursor = page.NextCursor + } + if !reflect.DeepEqual(seen, []string{"10001", "10000", "1"}) { + t.Fatalf("duplicate evidence or lost topic matches %v", seen) + } +} diff --git a/internal/research/testdata/corpus.sql b/internal/research/testdata/corpus.sql index 3bbd5ac..e890668 100644 --- a/internal/research/testdata/corpus.sql +++ b/internal/research/testdata/corpus.sql @@ -70,7 +70,8 @@ BEGIN END LOOP; END $$; UPDATE research_encoded_fixture.papers -SET title='Graph'||chr(1)||chr(1)||'original'||chr(1)||chr(2), +SET title_key='graph'||chr(1)||chr(1)||'key'||chr(1)||chr(2), + title='Graph'||chr(1)||chr(1)||'original'||chr(1)||chr(2), abstract='Text'||chr(1)||chr(1)||'tail', authors_json='["Ada\u0000Smith","Group\u0001Name"]' WHERE id=1; @@ -79,3 +80,22 @@ SET metadata_json='{"comments":"Original\u0000comment"}' WHERE id=101; REVOKE ALL ON SCHEMA research_encoded_fixture FROM PUBLIC; GRANT USAGE ON SCHEMA research_encoded_fixture TO foundry_research_fixture; GRANT SELECT ON ALL TABLES IN SCHEMA research_encoded_fixture TO foundry_research_fixture; + +-- A large ID gap proves the bounded probe cannot truncate sparse results. +CREATE SCHEMA research_sparse_fixture; +DO $$ +DECLARE item text; +BEGIN + FOREACH item IN ARRAY ARRAY['papers','sources','source_records','identifiers','tags','paper_tags','venues','editions','submissions','decisions'] LOOP + EXECUTE format('CREATE TABLE research_sparse_fixture.%I (LIKE research_fixture.%I INCLUDING ALL)',item,item); + EXECUTE format('INSERT INTO research_sparse_fixture.%I SELECT * FROM research_fixture.%I',item,item); + END LOOP; +END $$; +INSERT INTO research_sparse_fixture.papers VALUES + (8000,'Graph in expanded interval','Graph evidence.','graph expanded','[]',NULL), + (10000,'Graph near cursor','Graph evidence.','graph near cursor','[]',NULL), + (10001,'Graph first','Graph evidence.','graph first','[]',NULL); +INSERT INTO research_sparse_fixture.paper_tags VALUES(10000,1,'first'),(10000,1,'second'),(10001,1,'first'); +REVOKE ALL ON SCHEMA research_sparse_fixture FROM PUBLIC; +GRANT USAGE ON SCHEMA research_sparse_fixture TO foundry_research_fixture; +GRANT SELECT ON ALL TABLES IN SCHEMA research_sparse_fixture TO foundry_research_fixture; -- 2.52.0