Page research without exact counts and bound candidate scans [skip ci]

This commit is contained in:
2026-11-07 17:27:35 +08:00
parent 142c7b2d86
commit 026ee22c1e
8 changed files with 563 additions and 86 deletions
+65
View File
@@ -0,0 +1,65 @@
package research
import (
"bytes"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"io"
"strconv"
"strings"
)
// Cursors locate a public corpus row, not an account or authentication token.
// Binding to normalized filters prevents accidentally paging a different query.
type position struct {
Version int `json:"v"`
ID int64 `json:"id"`
Key string `json:"key"`
Before bool `json:"before"`
Filter string `json:"filter"`
}
var errCursor = errors.New("invalid cursor")
func filterKey(f Filters) string {
sort, search := f.Sort, f.SearchIn
if sort == "" {
sort = "recent"
}
if search == "" {
search = "text"
}
data, _ := json.Marshal([]string{strings.TrimSpace(f.Query), f.Topic, sort, search, strconv.Itoa(f.PageSize)})
sum := sha256.Sum256(data)
return hex.EncodeToString(sum[:16])
}
func encodeCursor(f Filters, p position, before bool) string {
p.Version, p.Before, p.Filter = 1, before, filterKey(f)
if f.Sort != "title" {
p.Key = ""
}
data, _ := json.Marshal(p)
return base64.RawURLEncoding.EncodeToString(data)
}
func decodeCursor(f Filters) (position, error) {
var p position
if f.Cursor == "" {
return p, nil
}
if len(f.Cursor) > 32768 {
return p, errCursor
}
data, err := base64.RawURLEncoding.Strict().DecodeString(f.Cursor)
if err != nil {
return p, errCursor
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
if decoder.Decode(&p) != nil || decoder.Decode(new(any)) != io.EOF || p.Version != 1 || p.ID <= 0 || p.Filter != filterKey(f) || strings.ContainsRune(p.Key, 0) || (f.Sort != "title" && p.Key != "") {
return position{}, errCursor
}
return p, nil
}
+34
View File
@@ -0,0 +1,34 @@
package research
import (
"encoding/base64"
"net/url"
"strings"
"testing"
)
func TestCursorValidation(t *testing.T) {
f := Filters{Query: "graph", Sort: "title", SearchIn: "text", PageSize: 8}
f.Cursor = encodeCursor(f, position{ID: 9223372036854775807, Key: "duplicate\x01title"}, true)
p, err := decodeCursor(f)
if err != nil || !p.Before || p.ID != 9223372036854775807 || p.Key != "duplicate\x01title" {
t.Fatalf("roundtrip %+v %v", p, err)
}
if _, err := parseFilters("q=graph&sort=title&cursor=" + url.QueryEscape(f.Cursor)); err != nil {
t.Fatal(err)
}
for _, raw := range []string{"!", strings.Repeat("a", 32769), base64.RawURLEncoding.EncodeToString([]byte(`{"v":2}`)), f.Cursor + "="} {
bad := f
bad.Cursor = raw
if _, err := decodeCursor(bad); err == nil {
t.Fatal("accepted invalid cursor")
}
}
for _, change := range []func(*Filters){func(f *Filters) { f.Query = "other" }, func(f *Filters) { f.Sort = "recent" }, func(f *Filters) { f.PageSize = 9 }, func(f *Filters) { f.SearchIn = "author" }, func(f *Filters) { f.Topic = "cs.AI" }} {
bad := f
change(&bad)
if _, err := decodeCursor(bad); err == nil {
t.Fatal("accepted mismatched filter")
}
}
}
+15 -20
View File
@@ -49,20 +49,20 @@ type Filters struct {
SearchIn string
Topic string
Sort string
Page int
Cursor string
PageSize int
}
type Page struct {
Items []Paper `json:"items"`
Total int64 `json:"total"`
Topics []string `json:"topics"`
Page int `json:"page"`
PageSize int `json:"pageSize"`
Items []Paper `json:"items"`
NextCursor string `json:"nextCursor"`
PreviousCursor string `json:"previousCursor"`
Topics []string `json:"topics"`
PageSize int `json:"pageSize"`
}
// Repository implementations must use a read-only transaction for each List,
// so its count, topics and page describe the same database snapshot.
// so its metadata and page describe the same database snapshot.
type Repository interface {
List(context.Context, Filters) (Page, error)
Get(context.Context, int64) (Paper, error)
@@ -102,7 +102,7 @@ var topicPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.-]{0,79}$`)
var positiveInteger = regexp.MustCompile(`^[1-9][0-9]*$`)
func parseFilters(raw string) (Filters, error) {
f := Filters{SearchIn: "text", Sort: "recent", Page: 1, PageSize: 8}
f := Filters{SearchIn: "text", Sort: "recent", PageSize: 8}
values, err := url.ParseQuery(raw)
if err != nil {
return f, err
@@ -133,28 +133,23 @@ func parseFilters(raw string) (Filters, error) {
return f, invalid
}
f.Sort = value[0]
case "page", "pageSize":
case "cursor":
f.Cursor = value[0]
case "pageSize":
if !positiveInteger.MatchString(value[0]) {
return f, invalid
}
n, err := strconv.Atoi(value[0])
limit := 1000
if key == "pageSize" {
limit = 50
}
if err != nil || n > limit {
if err != nil || n > 50 {
return f, invalid
}
if key == "page" {
f.Page = n
} else {
f.PageSize = n
}
f.PageSize = n
default:
return f, invalid
}
}
return f, nil
_, err = decodeCursor(f)
return f, err
}
func (s *service) serve(w http.ResponseWriter, r *http.Request) {
+18 -4
View File
@@ -28,7 +28,7 @@ func (f *fixtureRepository) List(ctx context.Context, filters Filters) (Page, er
<-ctx.Done()
return Page{}, ctx.Err()
}
return Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}, f.err
return Page{Items: []Paper{}, Topics: []string{}, PageSize: filters.PageSize}, f.err
}
func (f *fixtureRepository) Get(ctx context.Context, id int64) (Paper, error) {
@@ -47,8 +47,8 @@ func TestAnonymousReadRoutes(t *testing.T) {
f := &fixtureRepository{}
mux := http.NewServeMux()
Register(mux, f, map[string]string{"thesisly": "http://localhost:17322/auth/callback"})
w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&page=2&pageSize=10")
if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", Page: 2, PageSize: 10}) {
w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&pageSize=10")
if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", PageSize: 10}) {
t.Fatalf("list: %d %+v", w.Code, f.filters)
}
var page Page
@@ -95,7 +95,7 @@ func TestInvalidQueriesNeverReachRepository(t *testing.T) {
func TestQueryDefaultsAndUnicode(t *testing.T) {
f, err := parseFilters("q=" + url.QueryEscape(" 论文 Graph "))
if err != nil || f.Query != "论文 Graph" || f.Page != 1 || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" {
if err != nil || f.Query != "论文 Graph" || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" {
t.Fatalf("%+v %v", f, err)
}
}
@@ -143,3 +143,17 @@ func TestConcurrencyAndTimeoutReleaseSlots(t *testing.T) {
t.Fatalf("timeout %d, slots %d", w.Code, len(s.slots))
}
}
func TestInvalidCursorsNeverReachRepository(t *testing.T) {
f := Filters{Query: "graph", Sort: "recent", SearchIn: "text", PageSize: 8}
valid := encodeCursor(f, position{ID: 42}, false)
for _, query := range []string{"cursor=!", "cursor=" + strings.Repeat("a", 32769), "q=different&cursor=" + valid, "cursor=" + valid + "&cursor=" + valid, "page=1"} {
repository := &fixtureRepository{}
mux := http.NewServeMux()
Register(mux, repository, map[string]string{"thesisly": "unused"})
w := request(mux, "GET", "/projects/thesisly/research/papers?"+query)
if w.Code != 400 || repository.calls != 0 {
t.Fatalf("invalid cursor reached storage: status=%d calls=%d", w.Code, repository.calls)
}
}
}
+170 -35
View File
@@ -173,8 +173,31 @@ func decodeAuthors(raw string) ([]string, error) {
return names, nil
}
func (r *PostgresRepository) positions(ctx context.Context, tx *sql.Tx, query string, args []any) ([]position, error) {
rows, err := tx.QueryContext(ctx, r.query(query), args...)
if err != nil {
return nil, err
}
defer rows.Close()
positions := []position{}
for rows.Next() {
var p position
// Preserve the stored title key, including the published text codec.
if err = rows.Scan(&p.ID, &p.Key); err != nil {
return nil, err
}
positions = append(positions, p)
}
return positions, rows.Err()
}
func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, error) {
result := Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}
result := Page{Items: []Paper{}, Topics: []string{}, PageSize: filters.PageSize}
cursor, err := decodeCursor(filters)
if err != nil {
return result, err
}
original := filters
tx, err := r.begin(ctx)
if err != nil {
return result, err
@@ -185,56 +208,168 @@ func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, e
filters.Topic = encodeText(filters.Topic)
}
where, args := whereClause(filters)
if err = tx.QueryRowContext(ctx, r.query(`SELECT count(*) FROM {schema}papers p`+where), args...).Scan(&result.Total); err != nil {
return result, err
if cursor.ID != 0 {
clause := " AND "
if where == "" {
clause = " WHERE "
}
op := "<"
if cursor.Before {
op = ">"
}
args = append(args, cursor.ID)
if filters.Sort == "title" {
if cursor.Before {
op = "<"
} else {
op = ">"
}
args = append(args, cursor.Key)
where += fmt.Sprintf("%s(p.title_key,p.id) %s ($%d::text COLLATE \"C\",$%d)", clause, op, len(args), len(args)-1)
} else {
where += fmt.Sprintf("%sp.id %s $%d", clause, op, len(args))
}
}
order := "p.id DESC"
if filters.Sort == "title" {
order = "p.title_key,p.id"
}
args = append(args, filters.PageSize, (filters.Page-1)*filters.PageSize)
// Select IDs before loading potentially TOASTed titles/abstracts. With
// sparse categories, selecting full rows here can make the planner walk
// hundreds of thousands of wide heap rows before finding eight matches.
candidateQuery := `SELECT p.id FROM {schema}papers p`
candidateOrder := order
prefix := ""
if where != "" {
columns := "p.id"
if cursor.Before {
order = "p.id"
if filters.Sort == "title" {
columns += ",p.title_key"
order = "p.title_key DESC,p.id DESC"
}
// Apply search before LIMIT/ORDER BY so a sparse term cannot turn
// into a backward heap scan that recomputes vectors for millions.
prefix = fmt.Sprintf(`matches AS MATERIALIZED (SELECT %s FROM {schema}papers p%s), `, columns, where)
candidateQuery = "SELECT id FROM matches"
candidateOrder = strings.ReplaceAll(order, "p.", "")
}
query := fmt.Sprintf(`WITH %spage_ids AS MATERIALIZED (
%s ORDER BY %s LIMIT $%d OFFSET $%d
) SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published
FROM page_ids JOIN {schema}papers p ON p.id=page_ids.id ORDER BY %s`, prefix, candidateQuery, candidateOrder, len(args)-1, len(args), order)
rows, err := tx.QueryContext(ctx, r.query(query), args...)
if err != nil {
return result, err
args = append(args, filters.PageSize+1)
columns := "p.id,p.title_key"
if filters.Sort != "title" {
columns = "p.id, ''::text AS title_key"
}
for rows.Next() {
p, err := r.scanPaper(rows)
// Search candidates are materialized to avoid scanning millions of wide
// rows in ID order for sparse terms. There is no count or deep OFFSET.
query := fmt.Sprintf("SELECT %s FROM {schema}papers p%s ORDER BY %s LIMIT $%d", columns, where, order, len(args))
if filters.Query != "" || filters.Topic != "" {
query = fmt.Sprintf("WITH matches AS MATERIALIZED (SELECT %s FROM {schema}papers p%s) SELECT p.id,p.title_key FROM matches p ORDER BY %s LIMIT $%d", columns, where, order, len(args))
}
positions := []position{}
topicOnly := filters.Sort != "title" && filters.Topic != "" && filters.Query == ""
if topicOnly {
// The published paper_tags foreign key guarantees these IDs exist.
// Walk the (tag_id,paper_id) index instead of joining all matching
// paper rows before LIMIT. DISTINCT removes duplicate source evidence.
topicArgs := []any{filters.Topic, filters.PageSize + 1}
predicate := ""
if cursor.ID != 0 {
op := "<"
if cursor.Before {
op = ">"
}
topicArgs = append(topicArgs, cursor.ID)
predicate = " AND pt.paper_id " + op + " $3"
}
direction := "DESC"
if cursor.Before {
direction = "ASC"
}
query := fmt.Sprintf(`SELECT DISTINCT pt.paper_id,''::text AS title_key FROM {schema}paper_tags pt
WHERE pt.tag_id=(SELECT id FROM {schema}tags WHERE namespace='arxiv:category' AND value=$1)%s
ORDER BY pt.paper_id %s LIMIT $2`, predicate, direction)
positions, err = r.positions(ctx, tx, query, topicArgs)
if err != nil {
_ = rows.Close()
return result, err
}
result.Items = append(result.Items, p)
}
err = rows.Err()
_ = rows.Close()
if err != nil {
return result, err
if !topicOnly && filters.Sort != "title" && (filters.Query != "" || filters.Topic != "") {
// Common terms can fill a page near its cursor. Probe a bounded ID
// interval first; never scan the entire heap in order for a rare term.
// If the interval has fewer than size+1 matches, discard it and use
// the complete indexed search below. No matches are silently omitted.
anchor := cursor.ID
if anchor == 0 {
if err = tx.QueryRowContext(ctx, r.query(`SELECT coalesce(max(id),0) FROM {schema}papers`)).Scan(&anchor); err != nil {
return result, err
}
}
for _, window := range []int64{512, 4096} {
bound, op := int64(0), ">="
if anchor > window {
bound = anchor - window
}
if cursor.Before {
bound, op = int64(9223372036854775807), "<="
if anchor <= 9223372036854775807-window {
bound = anchor + window
}
}
boundedArgs := append(append([]any{}, args...), bound)
bounded := fmt.Sprintf("SELECT %s FROM {schema}papers p%s AND p.id %s $%d ORDER BY %s LIMIT $%d", columns, where, op, len(boundedArgs), order, len(args))
positions, err = r.positions(ctx, tx, bounded, boundedArgs)
if err != nil {
return result, err
}
// Expand once only when there is evidence of nearby matches.
// Zero matches goes straight to the sparse-term index fallback.
if len(positions) == 0 || len(positions) >= filters.PageSize+1 {
break
}
}
}
if !topicOnly && len(positions) < filters.PageSize+1 {
positions, err = r.positions(ctx, tx, query, args)
if err != nil {
return result, err
}
}
more := len(positions) > filters.PageSize
if more {
positions = positions[:filters.PageSize]
}
if cursor.Before {
for i, j := 0, len(positions)-1; i < j; i, j = i+1, j-1 {
positions[i], positions[j] = positions[j], positions[i]
}
}
if len(positions) > 0 {
if (!cursor.Before && cursor.ID != 0) || (cursor.Before && more) {
result.PreviousCursor = encodeCursor(original, positions[0], true)
}
if (!cursor.Before && more) || (cursor.Before && cursor.ID != 0) {
result.NextCursor = encodeCursor(original, positions[len(positions)-1], false)
}
ids := make([]int64, len(positions))
byID := map[string]Paper{}
for i, p := range positions {
ids[i] = p.ID
}
rows, err := tx.QueryContext(ctx, r.query(`SELECT id,title,abstract,authors_json,first_published FROM {schema}papers WHERE id=ANY($1)`), ids)
if err != nil {
return result, err
}
for rows.Next() {
p, scanErr := r.scanPaper(rows)
if scanErr != nil {
_ = rows.Close()
return result, scanErr
}
byID[p.ID] = p
}
err = rows.Err()
_ = rows.Close()
if err != nil {
return result, err
}
for _, p := range positions {
paper, exists := byID[strconv.FormatInt(p.ID, 10)]
if !exists {
return result, errors.New("inconsistent paper index")
}
result.Items = append(result.Items, paper)
}
}
if err = r.enrich(ctx, tx, result.Items); err != nil {
return result, err
}
rows, err = tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`))
rows, err := tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`))
if err != nil {
return result, err
}
@@ -271,7 +406,7 @@ func (r *PostgresRepository) Get(ctx context.Context, id int64) (Paper, error) {
return papers[0], tx.Commit()
}
// Batch metadata queries avoid per-paper round trips and share the count/page
// Batch metadata queries avoid per-paper round trips and share the page
// snapshot. User values are bound; only the validated schema is interpolated.
func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Paper) error {
if len(papers) == 0 {
+180 -17
View File
@@ -50,31 +50,28 @@ func TestPostgresPageSearchAndFilters(t *testing.T) {
}
for _, test := range []struct {
filters Filters
total int64
ids []string
}{
{Filters{Page: 1, PageSize: 2}, 5, []string{"5", "4"}},
{Filters{Page: 2, PageSize: 2}, 5, []string{"3", "2"}},
{Filters{Page: 3, PageSize: 2}, 5, []string{"1"}},
{Filters{Page: 1, PageSize: 2, Sort: "title"}, 5, []string{"2", "5"}},
{Filters{Page: 1, PageSize: 8, Query: "graph"}, 2, []string{"3", "1"}},
{Filters{Page: 1, PageSize: 8, Query: "graph OR zebra"}, 0, []string{}},
{Filters{Page: 1, PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, 1, []string{"1"}},
{Filters{Page: 1, PageSize: 8, Topic: "cs.LG"}, 1, []string{"1"}},
{Filters{Page: 1, PageSize: 8, Topic: "cs.DC", Query: "graph"}, 1, []string{"3"}},
{Filters{Page: 1, PageSize: 8, Query: `x'); DROP TABLE papers;--`}, 0, []string{}},
{Filters{PageSize: 2}, []string{"5", "4"}},
{Filters{PageSize: 2, Sort: "title"}, []string{"2", "5"}},
{Filters{PageSize: 8, Query: "graph"}, []string{"3", "1"}},
{Filters{PageSize: 8, Query: "graph OR zebra"}, []string{}},
{Filters{PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, []string{"1"}},
{Filters{PageSize: 8, Topic: "cs.LG"}, []string{"1"}},
{Filters{PageSize: 8, Topic: "cs.DC", Query: "graph"}, []string{"3"}},
{Filters{PageSize: 8, Query: `x'); DROP TABLE papers;--`}, []string{}},
} {
page, err := r.List(ctx, test.filters)
if err != nil {
t.Fatalf("%+v: %v", test.filters, err)
}
if page.Total != test.total || !reflect.DeepEqual(paperIDs(page.Items), test.ids) {
t.Fatalf("%+v: total=%d ids=%v", test.filters, page.Total, paperIDs(page.Items))
if !reflect.DeepEqual(paperIDs(page.Items), test.ids) {
t.Fatalf("%+v: ids=%v", test.filters, paperIDs(page.Items))
}
if !reflect.DeepEqual(page.Topics, []string{"cs.AI", "cs.DC", "cs.LG"}) {
t.Fatalf("mixed tag namespaces: %v", page.Topics)
}
if page.Page != test.filters.Page || page.PageSize != test.filters.PageSize {
if page.PageSize != test.filters.PageSize {
t.Fatal("lost paging metadata")
}
}
@@ -143,7 +140,7 @@ func TestPostgresCorpusIsReadOnlyAndHTTPWorks(t *testing.T) {
Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"})
w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph")
var page Page
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || page.Total != 2 {
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || len(page.Items) != 2 {
t.Fatalf("SQL-backed HTTP list: %d %s", w.Code, w.Body.String())
}
w = request(mux, "GET", "/projects/thesisly/research/papers/1")
@@ -179,8 +176,174 @@ func TestPostgresPublishedTextCodec(t *testing.T) {
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &decoded) != nil || decoded.Title != p.Title || decoded.Abstract == nil || *decoded.Abstract != *p.Abstract {
t.Fatalf("HTTP lost original control characters: %d %s", w.Code, w.Body.String())
}
page, err := r.List(ctx, Filters{Page: 1, PageSize: 8, Query: "graph"})
if err != nil || page.Total != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title {
page, err := r.List(ctx, Filters{PageSize: 8, Query: "graph"})
if err != nil || len(page.Items) != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title {
t.Fatalf("encoded corpus search: %+v %v", page, err)
}
}
func TestPostgresCursorsTraverseBothDirections(t *testing.T) {
r, ctx := integrationRepository(t)
for _, sort := range []string{"recent", "title"} {
f := Filters{Sort: sort, PageSize: 1}
seen := []string{}
cursors := []string{""}
for {
page, err := r.List(ctx, f)
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 1 {
t.Fatalf("unexpected page %+v", page)
}
seen = append(seen, page.Items[0].ID)
if len(seen) > 5 {
t.Fatal("cursor cycle")
}
if len(seen) > 1 {
back := f
back.Cursor = page.PreviousCursor
prior, err := r.List(ctx, back)
if err != nil || len(prior.Items) != 1 || prior.Items[0].ID != seen[len(seen)-2] {
t.Fatalf("reverse cursor: %+v %v", prior, err)
}
} else if page.PreviousCursor != "" {
t.Fatal("first page has previous")
}
if page.NextCursor == "" {
break
}
f.Cursor = page.NextCursor
cursors = append(cursors, f.Cursor)
}
want := []string{"5", "4", "3", "2", "1"}
if sort == "title" {
want = []string{"2", "5", "1", "3", "4"}
}
if !reflect.DeepEqual(seen, want) {
t.Fatalf("%s: %v", sort, seen)
}
f.Cursor = cursors[1]
f.Query = "different"
if _, err := r.List(ctx, f); err == nil {
t.Fatal("cursor reused across filters")
}
}
}
func TestPostgresEncodedTitleCursors(t *testing.T) {
legacy, ctx := integrationRepository(t)
r, err := NewPostgresRepository(legacy.db, "research_encoded_fixture")
if err != nil {
t.Fatal(err)
}
if err = r.Check(ctx); err != nil {
t.Fatal(err)
}
f := Filters{Sort: "title", PageSize: 1}
seen := []string{}
for {
page, err := r.List(ctx, f)
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 1 {
t.Fatalf("invalid encoded title page %+v", page)
}
seen = append(seen, page.Items[0].ID)
if len(seen) > 5 {
t.Fatal("cursor cycle")
}
if page.NextCursor == "" {
break
}
f.Cursor = page.NextCursor
}
if !reflect.DeepEqual(seen, []string{"2", "5", "1", "3", "4"}) {
t.Fatalf("encoded key paging %v", seen)
}
}
func TestPostgresBoundedProbeDoesNotTruncateSparseMatches(t *testing.T) {
legacy, ctx := integrationRepository(t)
r, err := NewPostgresRepository(legacy.db, "research_sparse_fixture")
if err != nil {
t.Fatal(err)
}
if err = r.Check(ctx); err != nil {
t.Fatal(err)
}
for _, size := range []int{1, 2, 3} {
f := Filters{Query: "graph", PageSize: size}
seen := []string{}
var previous []string
for {
page, err := r.List(ctx, f)
if err != nil {
t.Fatal(err)
}
ids := paperIDs(page.Items)
if len(ids) == 0 || len(ids) > size {
t.Fatalf("invalid sparse page %+v", page)
}
if previous != nil {
back := f
back.Cursor = page.PreviousCursor
prior, err := r.List(ctx, back)
if err != nil || !reflect.DeepEqual(paperIDs(prior.Items), previous) {
t.Fatalf("reverse gap %+v %v", prior, err)
}
}
seen = append(seen, ids...)
if len(seen) > 5 {
t.Fatal("cursor cycle")
}
previous = ids
if page.NextCursor == "" {
break
}
f.Cursor = page.NextCursor
}
if !reflect.DeepEqual(seen, []string{"10001", "10000", "8000", "3", "1"}) {
t.Fatalf("size=%d lost sparse matches %v", size, seen)
}
}
}
func TestPostgresTopicCursorSkipsDuplicateEvidence(t *testing.T) {
legacy, ctx := integrationRepository(t)
r, err := NewPostgresRepository(legacy.db, "research_sparse_fixture")
if err != nil {
t.Fatal(err)
}
f := Filters{Topic: "cs.LG", PageSize: 1}
seen := []string{}
for {
page, err := r.List(ctx, f)
if err != nil {
t.Fatal(err)
}
if len(page.Items) != 1 {
t.Fatalf("invalid topic page %+v", page)
}
seen = append(seen, page.Items[0].ID)
if len(seen) > 3 {
t.Fatal("cursor cycle")
}
if len(seen) > 1 {
back := f
back.Cursor = page.PreviousCursor
prior, err := r.List(ctx, back)
if err != nil || len(prior.Items) != 1 || prior.Items[0].ID != seen[len(seen)-2] {
t.Fatalf("reverse topic %+v %v", prior, err)
}
}
if page.NextCursor == "" {
break
}
f.Cursor = page.NextCursor
}
if !reflect.DeepEqual(seen, []string{"10001", "10000", "1"}) {
t.Fatalf("duplicate evidence or lost topic matches %v", seen)
}
}
+21 -1
View File
@@ -70,7 +70,8 @@ BEGIN
END LOOP;
END $$;
UPDATE research_encoded_fixture.papers
SET title='Graph'||chr(1)||chr(1)||'original'||chr(1)||chr(2),
SET title_key='graph'||chr(1)||chr(1)||'key'||chr(1)||chr(2),
title='Graph'||chr(1)||chr(1)||'original'||chr(1)||chr(2),
abstract='Text'||chr(1)||chr(1)||'tail',
authors_json='["Ada\u0000Smith","Group\u0001Name"]'
WHERE id=1;
@@ -79,3 +80,22 @@ SET metadata_json='{"comments":"Original\u0000comment"}' WHERE id=101;
REVOKE ALL ON SCHEMA research_encoded_fixture FROM PUBLIC;
GRANT USAGE ON SCHEMA research_encoded_fixture TO foundry_research_fixture;
GRANT SELECT ON ALL TABLES IN SCHEMA research_encoded_fixture TO foundry_research_fixture;
-- A large ID gap proves the bounded probe cannot truncate sparse results.
CREATE SCHEMA research_sparse_fixture;
DO $$
DECLARE item text;
BEGIN
FOREACH item IN ARRAY ARRAY['papers','sources','source_records','identifiers','tags','paper_tags','venues','editions','submissions','decisions'] LOOP
EXECUTE format('CREATE TABLE research_sparse_fixture.%I (LIKE research_fixture.%I INCLUDING ALL)',item,item);
EXECUTE format('INSERT INTO research_sparse_fixture.%I SELECT * FROM research_fixture.%I',item,item);
END LOOP;
END $$;
INSERT INTO research_sparse_fixture.papers VALUES
(8000,'Graph in expanded interval','Graph evidence.','graph expanded','[]',NULL),
(10000,'Graph near cursor','Graph evidence.','graph near cursor','[]',NULL),
(10001,'Graph first','Graph evidence.','graph first','[]',NULL);
INSERT INTO research_sparse_fixture.paper_tags VALUES(10000,1,'first'),(10000,1,'second'),(10001,1,'first');
REVOKE ALL ON SCHEMA research_sparse_fixture FROM PUBLIC;
GRANT USAGE ON SCHEMA research_sparse_fixture TO foundry_research_fixture;
GRANT SELECT ON ALL TABLES IN SCHEMA research_sparse_fixture TO foundry_research_fixture;