Page research without exact counts and bound candidate scans [skip ci]
This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
package research
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Cursors locate a public corpus row, not an account or authentication token.
|
||||
// Binding to normalized filters prevents accidentally paging a different query.
|
||||
type position struct {
|
||||
Version int `json:"v"`
|
||||
ID int64 `json:"id"`
|
||||
Key string `json:"key"`
|
||||
Before bool `json:"before"`
|
||||
Filter string `json:"filter"`
|
||||
}
|
||||
|
||||
var errCursor = errors.New("invalid cursor")
|
||||
|
||||
func filterKey(f Filters) string {
|
||||
sort, search := f.Sort, f.SearchIn
|
||||
if sort == "" {
|
||||
sort = "recent"
|
||||
}
|
||||
if search == "" {
|
||||
search = "text"
|
||||
}
|
||||
data, _ := json.Marshal([]string{strings.TrimSpace(f.Query), f.Topic, sort, search, strconv.Itoa(f.PageSize)})
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:16])
|
||||
}
|
||||
func encodeCursor(f Filters, p position, before bool) string {
|
||||
p.Version, p.Before, p.Filter = 1, before, filterKey(f)
|
||||
if f.Sort != "title" {
|
||||
p.Key = ""
|
||||
}
|
||||
data, _ := json.Marshal(p)
|
||||
return base64.RawURLEncoding.EncodeToString(data)
|
||||
}
|
||||
func decodeCursor(f Filters) (position, error) {
|
||||
var p position
|
||||
if f.Cursor == "" {
|
||||
return p, nil
|
||||
}
|
||||
if len(f.Cursor) > 32768 {
|
||||
return p, errCursor
|
||||
}
|
||||
data, err := base64.RawURLEncoding.Strict().DecodeString(f.Cursor)
|
||||
if err != nil {
|
||||
return p, errCursor
|
||||
}
|
||||
decoder := json.NewDecoder(bytes.NewReader(data))
|
||||
decoder.DisallowUnknownFields()
|
||||
if decoder.Decode(&p) != nil || decoder.Decode(new(any)) != io.EOF || p.Version != 1 || p.ID <= 0 || p.Filter != filterKey(f) || strings.ContainsRune(p.Key, 0) || (f.Sort != "title" && p.Key != "") {
|
||||
return position{}, errCursor
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package research
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestCursorValidation(t *testing.T) {
|
||||
f := Filters{Query: "graph", Sort: "title", SearchIn: "text", PageSize: 8}
|
||||
f.Cursor = encodeCursor(f, position{ID: 9223372036854775807, Key: "duplicate\x01title"}, true)
|
||||
p, err := decodeCursor(f)
|
||||
if err != nil || !p.Before || p.ID != 9223372036854775807 || p.Key != "duplicate\x01title" {
|
||||
t.Fatalf("roundtrip %+v %v", p, err)
|
||||
}
|
||||
if _, err := parseFilters("q=graph&sort=title&cursor=" + url.QueryEscape(f.Cursor)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, raw := range []string{"!", strings.Repeat("a", 32769), base64.RawURLEncoding.EncodeToString([]byte(`{"v":2}`)), f.Cursor + "="} {
|
||||
bad := f
|
||||
bad.Cursor = raw
|
||||
if _, err := decodeCursor(bad); err == nil {
|
||||
t.Fatal("accepted invalid cursor")
|
||||
}
|
||||
}
|
||||
for _, change := range []func(*Filters){func(f *Filters) { f.Query = "other" }, func(f *Filters) { f.Sort = "recent" }, func(f *Filters) { f.PageSize = 9 }, func(f *Filters) { f.SearchIn = "author" }, func(f *Filters) { f.Topic = "cs.AI" }} {
|
||||
bad := f
|
||||
change(&bad)
|
||||
if _, err := decodeCursor(bad); err == nil {
|
||||
t.Fatal("accepted mismatched filter")
|
||||
}
|
||||
}
|
||||
}
|
||||
+15
-20
@@ -49,20 +49,20 @@ type Filters struct {
|
||||
SearchIn string
|
||||
Topic string
|
||||
Sort string
|
||||
Page int
|
||||
Cursor string
|
||||
PageSize int
|
||||
}
|
||||
|
||||
type Page struct {
|
||||
Items []Paper `json:"items"`
|
||||
Total int64 `json:"total"`
|
||||
Topics []string `json:"topics"`
|
||||
Page int `json:"page"`
|
||||
PageSize int `json:"pageSize"`
|
||||
Items []Paper `json:"items"`
|
||||
NextCursor string `json:"nextCursor"`
|
||||
PreviousCursor string `json:"previousCursor"`
|
||||
Topics []string `json:"topics"`
|
||||
PageSize int `json:"pageSize"`
|
||||
}
|
||||
|
||||
// Repository implementations must use a read-only transaction for each List,
|
||||
// so its count, topics and page describe the same database snapshot.
|
||||
// so its metadata and page describe the same database snapshot.
|
||||
type Repository interface {
|
||||
List(context.Context, Filters) (Page, error)
|
||||
Get(context.Context, int64) (Paper, error)
|
||||
@@ -102,7 +102,7 @@ var topicPattern = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9.-]{0,79}$`)
|
||||
var positiveInteger = regexp.MustCompile(`^[1-9][0-9]*$`)
|
||||
|
||||
func parseFilters(raw string) (Filters, error) {
|
||||
f := Filters{SearchIn: "text", Sort: "recent", Page: 1, PageSize: 8}
|
||||
f := Filters{SearchIn: "text", Sort: "recent", PageSize: 8}
|
||||
values, err := url.ParseQuery(raw)
|
||||
if err != nil {
|
||||
return f, err
|
||||
@@ -133,28 +133,23 @@ func parseFilters(raw string) (Filters, error) {
|
||||
return f, invalid
|
||||
}
|
||||
f.Sort = value[0]
|
||||
case "page", "pageSize":
|
||||
case "cursor":
|
||||
f.Cursor = value[0]
|
||||
case "pageSize":
|
||||
if !positiveInteger.MatchString(value[0]) {
|
||||
return f, invalid
|
||||
}
|
||||
n, err := strconv.Atoi(value[0])
|
||||
limit := 1000
|
||||
if key == "pageSize" {
|
||||
limit = 50
|
||||
}
|
||||
if err != nil || n > limit {
|
||||
if err != nil || n > 50 {
|
||||
return f, invalid
|
||||
}
|
||||
if key == "page" {
|
||||
f.Page = n
|
||||
} else {
|
||||
f.PageSize = n
|
||||
}
|
||||
f.PageSize = n
|
||||
default:
|
||||
return f, invalid
|
||||
}
|
||||
}
|
||||
return f, nil
|
||||
_, err = decodeCursor(f)
|
||||
return f, err
|
||||
}
|
||||
|
||||
func (s *service) serve(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -28,7 +28,7 @@ func (f *fixtureRepository) List(ctx context.Context, filters Filters) (Page, er
|
||||
<-ctx.Done()
|
||||
return Page{}, ctx.Err()
|
||||
}
|
||||
return Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}, f.err
|
||||
return Page{Items: []Paper{}, Topics: []string{}, PageSize: filters.PageSize}, f.err
|
||||
}
|
||||
|
||||
func (f *fixtureRepository) Get(ctx context.Context, id int64) (Paper, error) {
|
||||
@@ -47,8 +47,8 @@ func TestAnonymousReadRoutes(t *testing.T) {
|
||||
f := &fixtureRepository{}
|
||||
mux := http.NewServeMux()
|
||||
Register(mux, f, map[string]string{"thesisly": "http://localhost:17322/auth/callback"})
|
||||
w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&page=2&pageSize=10")
|
||||
if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", Page: 2, PageSize: 10}) {
|
||||
w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph&searchIn=author&sort=title&topic=cs.LG&pageSize=10")
|
||||
if w.Code != 200 || f.filters != (Filters{Query: "graph", SearchIn: "author", Topic: "cs.LG", Sort: "title", PageSize: 10}) {
|
||||
t.Fatalf("list: %d %+v", w.Code, f.filters)
|
||||
}
|
||||
var page Page
|
||||
@@ -95,7 +95,7 @@ func TestInvalidQueriesNeverReachRepository(t *testing.T) {
|
||||
|
||||
func TestQueryDefaultsAndUnicode(t *testing.T) {
|
||||
f, err := parseFilters("q=" + url.QueryEscape(" 论文 Graph "))
|
||||
if err != nil || f.Query != "论文 Graph" || f.Page != 1 || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" {
|
||||
if err != nil || f.Query != "论文 Graph" || f.PageSize != 8 || f.SearchIn != "text" || f.Sort != "recent" {
|
||||
t.Fatalf("%+v %v", f, err)
|
||||
}
|
||||
}
|
||||
@@ -143,3 +143,17 @@ func TestConcurrencyAndTimeoutReleaseSlots(t *testing.T) {
|
||||
t.Fatalf("timeout %d, slots %d", w.Code, len(s.slots))
|
||||
}
|
||||
}
|
||||
|
||||
func TestInvalidCursorsNeverReachRepository(t *testing.T) {
|
||||
f := Filters{Query: "graph", Sort: "recent", SearchIn: "text", PageSize: 8}
|
||||
valid := encodeCursor(f, position{ID: 42}, false)
|
||||
for _, query := range []string{"cursor=!", "cursor=" + strings.Repeat("a", 32769), "q=different&cursor=" + valid, "cursor=" + valid + "&cursor=" + valid, "page=1"} {
|
||||
repository := &fixtureRepository{}
|
||||
mux := http.NewServeMux()
|
||||
Register(mux, repository, map[string]string{"thesisly": "unused"})
|
||||
w := request(mux, "GET", "/projects/thesisly/research/papers?"+query)
|
||||
if w.Code != 400 || repository.calls != 0 {
|
||||
t.Fatalf("invalid cursor reached storage: status=%d calls=%d", w.Code, repository.calls)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+170
-35
@@ -173,8 +173,31 @@ func decodeAuthors(raw string) ([]string, error) {
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) positions(ctx context.Context, tx *sql.Tx, query string, args []any) ([]position, error) {
|
||||
rows, err := tx.QueryContext(ctx, r.query(query), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
positions := []position{}
|
||||
for rows.Next() {
|
||||
var p position
|
||||
// Preserve the stored title key, including the published text codec.
|
||||
if err = rows.Scan(&p.ID, &p.Key); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
positions = append(positions, p)
|
||||
}
|
||||
return positions, rows.Err()
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, error) {
|
||||
result := Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}
|
||||
result := Page{Items: []Paper{}, Topics: []string{}, PageSize: filters.PageSize}
|
||||
cursor, err := decodeCursor(filters)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
original := filters
|
||||
tx, err := r.begin(ctx)
|
||||
if err != nil {
|
||||
return result, err
|
||||
@@ -185,56 +208,168 @@ func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, e
|
||||
filters.Topic = encodeText(filters.Topic)
|
||||
}
|
||||
where, args := whereClause(filters)
|
||||
if err = tx.QueryRowContext(ctx, r.query(`SELECT count(*) FROM {schema}papers p`+where), args...).Scan(&result.Total); err != nil {
|
||||
return result, err
|
||||
if cursor.ID != 0 {
|
||||
clause := " AND "
|
||||
if where == "" {
|
||||
clause = " WHERE "
|
||||
}
|
||||
op := "<"
|
||||
if cursor.Before {
|
||||
op = ">"
|
||||
}
|
||||
args = append(args, cursor.ID)
|
||||
if filters.Sort == "title" {
|
||||
if cursor.Before {
|
||||
op = "<"
|
||||
} else {
|
||||
op = ">"
|
||||
}
|
||||
args = append(args, cursor.Key)
|
||||
where += fmt.Sprintf("%s(p.title_key,p.id) %s ($%d::text COLLATE \"C\",$%d)", clause, op, len(args), len(args)-1)
|
||||
} else {
|
||||
where += fmt.Sprintf("%sp.id %s $%d", clause, op, len(args))
|
||||
}
|
||||
}
|
||||
order := "p.id DESC"
|
||||
if filters.Sort == "title" {
|
||||
order = "p.title_key,p.id"
|
||||
}
|
||||
args = append(args, filters.PageSize, (filters.Page-1)*filters.PageSize)
|
||||
// Select IDs before loading potentially TOASTed titles/abstracts. With
|
||||
// sparse categories, selecting full rows here can make the planner walk
|
||||
// hundreds of thousands of wide heap rows before finding eight matches.
|
||||
candidateQuery := `SELECT p.id FROM {schema}papers p`
|
||||
candidateOrder := order
|
||||
prefix := ""
|
||||
if where != "" {
|
||||
columns := "p.id"
|
||||
if cursor.Before {
|
||||
order = "p.id"
|
||||
if filters.Sort == "title" {
|
||||
columns += ",p.title_key"
|
||||
order = "p.title_key DESC,p.id DESC"
|
||||
}
|
||||
// Apply search before LIMIT/ORDER BY so a sparse term cannot turn
|
||||
// into a backward heap scan that recomputes vectors for millions.
|
||||
prefix = fmt.Sprintf(`matches AS MATERIALIZED (SELECT %s FROM {schema}papers p%s), `, columns, where)
|
||||
candidateQuery = "SELECT id FROM matches"
|
||||
candidateOrder = strings.ReplaceAll(order, "p.", "")
|
||||
}
|
||||
query := fmt.Sprintf(`WITH %spage_ids AS MATERIALIZED (
|
||||
%s ORDER BY %s LIMIT $%d OFFSET $%d
|
||||
) SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published
|
||||
FROM page_ids JOIN {schema}papers p ON p.id=page_ids.id ORDER BY %s`, prefix, candidateQuery, candidateOrder, len(args)-1, len(args), order)
|
||||
rows, err := tx.QueryContext(ctx, r.query(query), args...)
|
||||
if err != nil {
|
||||
return result, err
|
||||
args = append(args, filters.PageSize+1)
|
||||
columns := "p.id,p.title_key"
|
||||
if filters.Sort != "title" {
|
||||
columns = "p.id, ''::text AS title_key"
|
||||
}
|
||||
for rows.Next() {
|
||||
p, err := r.scanPaper(rows)
|
||||
// Search candidates are materialized to avoid scanning millions of wide
|
||||
// rows in ID order for sparse terms. There is no count or deep OFFSET.
|
||||
query := fmt.Sprintf("SELECT %s FROM {schema}papers p%s ORDER BY %s LIMIT $%d", columns, where, order, len(args))
|
||||
if filters.Query != "" || filters.Topic != "" {
|
||||
query = fmt.Sprintf("WITH matches AS MATERIALIZED (SELECT %s FROM {schema}papers p%s) SELECT p.id,p.title_key FROM matches p ORDER BY %s LIMIT $%d", columns, where, order, len(args))
|
||||
}
|
||||
positions := []position{}
|
||||
topicOnly := filters.Sort != "title" && filters.Topic != "" && filters.Query == ""
|
||||
if topicOnly {
|
||||
// The published paper_tags foreign key guarantees these IDs exist.
|
||||
// Walk the (tag_id,paper_id) index instead of joining all matching
|
||||
// paper rows before LIMIT. DISTINCT removes duplicate source evidence.
|
||||
topicArgs := []any{filters.Topic, filters.PageSize + 1}
|
||||
predicate := ""
|
||||
if cursor.ID != 0 {
|
||||
op := "<"
|
||||
if cursor.Before {
|
||||
op = ">"
|
||||
}
|
||||
topicArgs = append(topicArgs, cursor.ID)
|
||||
predicate = " AND pt.paper_id " + op + " $3"
|
||||
}
|
||||
direction := "DESC"
|
||||
if cursor.Before {
|
||||
direction = "ASC"
|
||||
}
|
||||
query := fmt.Sprintf(`SELECT DISTINCT pt.paper_id,''::text AS title_key FROM {schema}paper_tags pt
|
||||
WHERE pt.tag_id=(SELECT id FROM {schema}tags WHERE namespace='arxiv:category' AND value=$1)%s
|
||||
ORDER BY pt.paper_id %s LIMIT $2`, predicate, direction)
|
||||
positions, err = r.positions(ctx, tx, query, topicArgs)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
return result, err
|
||||
}
|
||||
result.Items = append(result.Items, p)
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return result, err
|
||||
if !topicOnly && filters.Sort != "title" && (filters.Query != "" || filters.Topic != "") {
|
||||
// Common terms can fill a page near its cursor. Probe a bounded ID
|
||||
// interval first; never scan the entire heap in order for a rare term.
|
||||
// If the interval has fewer than size+1 matches, discard it and use
|
||||
// the complete indexed search below. No matches are silently omitted.
|
||||
anchor := cursor.ID
|
||||
if anchor == 0 {
|
||||
if err = tx.QueryRowContext(ctx, r.query(`SELECT coalesce(max(id),0) FROM {schema}papers`)).Scan(&anchor); err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
for _, window := range []int64{512, 4096} {
|
||||
bound, op := int64(0), ">="
|
||||
if anchor > window {
|
||||
bound = anchor - window
|
||||
}
|
||||
if cursor.Before {
|
||||
bound, op = int64(9223372036854775807), "<="
|
||||
if anchor <= 9223372036854775807-window {
|
||||
bound = anchor + window
|
||||
}
|
||||
}
|
||||
boundedArgs := append(append([]any{}, args...), bound)
|
||||
bounded := fmt.Sprintf("SELECT %s FROM {schema}papers p%s AND p.id %s $%d ORDER BY %s LIMIT $%d", columns, where, op, len(boundedArgs), order, len(args))
|
||||
positions, err = r.positions(ctx, tx, bounded, boundedArgs)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
// Expand once only when there is evidence of nearby matches.
|
||||
// Zero matches goes straight to the sparse-term index fallback.
|
||||
if len(positions) == 0 || len(positions) >= filters.PageSize+1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !topicOnly && len(positions) < filters.PageSize+1 {
|
||||
positions, err = r.positions(ctx, tx, query, args)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
}
|
||||
more := len(positions) > filters.PageSize
|
||||
if more {
|
||||
positions = positions[:filters.PageSize]
|
||||
}
|
||||
if cursor.Before {
|
||||
for i, j := 0, len(positions)-1; i < j; i, j = i+1, j-1 {
|
||||
positions[i], positions[j] = positions[j], positions[i]
|
||||
}
|
||||
}
|
||||
if len(positions) > 0 {
|
||||
if (!cursor.Before && cursor.ID != 0) || (cursor.Before && more) {
|
||||
result.PreviousCursor = encodeCursor(original, positions[0], true)
|
||||
}
|
||||
if (!cursor.Before && more) || (cursor.Before && cursor.ID != 0) {
|
||||
result.NextCursor = encodeCursor(original, positions[len(positions)-1], false)
|
||||
}
|
||||
ids := make([]int64, len(positions))
|
||||
byID := map[string]Paper{}
|
||||
for i, p := range positions {
|
||||
ids[i] = p.ID
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, r.query(`SELECT id,title,abstract,authors_json,first_published FROM {schema}papers WHERE id=ANY($1)`), ids)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
for rows.Next() {
|
||||
p, scanErr := r.scanPaper(rows)
|
||||
if scanErr != nil {
|
||||
_ = rows.Close()
|
||||
return result, scanErr
|
||||
}
|
||||
byID[p.ID] = p
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
for _, p := range positions {
|
||||
paper, exists := byID[strconv.FormatInt(p.ID, 10)]
|
||||
if !exists {
|
||||
return result, errors.New("inconsistent paper index")
|
||||
}
|
||||
result.Items = append(result.Items, paper)
|
||||
}
|
||||
}
|
||||
if err = r.enrich(ctx, tx, result.Items); err != nil {
|
||||
return result, err
|
||||
}
|
||||
rows, err = tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`))
|
||||
rows, err := tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`))
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -271,7 +406,7 @@ func (r *PostgresRepository) Get(ctx context.Context, id int64) (Paper, error) {
|
||||
return papers[0], tx.Commit()
|
||||
}
|
||||
|
||||
// Batch metadata queries avoid per-paper round trips and share the count/page
|
||||
// Batch metadata queries avoid per-paper round trips and share the page
|
||||
// snapshot. User values are bound; only the validated schema is interpolated.
|
||||
func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Paper) error {
|
||||
if len(papers) == 0 {
|
||||
|
||||
@@ -50,31 +50,28 @@ func TestPostgresPageSearchAndFilters(t *testing.T) {
|
||||
}
|
||||
for _, test := range []struct {
|
||||
filters Filters
|
||||
total int64
|
||||
ids []string
|
||||
}{
|
||||
{Filters{Page: 1, PageSize: 2}, 5, []string{"5", "4"}},
|
||||
{Filters{Page: 2, PageSize: 2}, 5, []string{"3", "2"}},
|
||||
{Filters{Page: 3, PageSize: 2}, 5, []string{"1"}},
|
||||
{Filters{Page: 1, PageSize: 2, Sort: "title"}, 5, []string{"2", "5"}},
|
||||
{Filters{Page: 1, PageSize: 8, Query: "graph"}, 2, []string{"3", "1"}},
|
||||
{Filters{Page: 1, PageSize: 8, Query: "graph OR zebra"}, 0, []string{}},
|
||||
{Filters{Page: 1, PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, 1, []string{"1"}},
|
||||
{Filters{Page: 1, PageSize: 8, Topic: "cs.LG"}, 1, []string{"1"}},
|
||||
{Filters{Page: 1, PageSize: 8, Topic: "cs.DC", Query: "graph"}, 1, []string{"3"}},
|
||||
{Filters{Page: 1, PageSize: 8, Query: `x'); DROP TABLE papers;--`}, 0, []string{}},
|
||||
{Filters{PageSize: 2}, []string{"5", "4"}},
|
||||
{Filters{PageSize: 2, Sort: "title"}, []string{"2", "5"}},
|
||||
{Filters{PageSize: 8, Query: "graph"}, []string{"3", "1"}},
|
||||
{Filters{PageSize: 8, Query: "graph OR zebra"}, []string{}},
|
||||
{Filters{PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, []string{"1"}},
|
||||
{Filters{PageSize: 8, Topic: "cs.LG"}, []string{"1"}},
|
||||
{Filters{PageSize: 8, Topic: "cs.DC", Query: "graph"}, []string{"3"}},
|
||||
{Filters{PageSize: 8, Query: `x'); DROP TABLE papers;--`}, []string{}},
|
||||
} {
|
||||
page, err := r.List(ctx, test.filters)
|
||||
if err != nil {
|
||||
t.Fatalf("%+v: %v", test.filters, err)
|
||||
}
|
||||
if page.Total != test.total || !reflect.DeepEqual(paperIDs(page.Items), test.ids) {
|
||||
t.Fatalf("%+v: total=%d ids=%v", test.filters, page.Total, paperIDs(page.Items))
|
||||
if !reflect.DeepEqual(paperIDs(page.Items), test.ids) {
|
||||
t.Fatalf("%+v: ids=%v", test.filters, paperIDs(page.Items))
|
||||
}
|
||||
if !reflect.DeepEqual(page.Topics, []string{"cs.AI", "cs.DC", "cs.LG"}) {
|
||||
t.Fatalf("mixed tag namespaces: %v", page.Topics)
|
||||
}
|
||||
if page.Page != test.filters.Page || page.PageSize != test.filters.PageSize {
|
||||
if page.PageSize != test.filters.PageSize {
|
||||
t.Fatal("lost paging metadata")
|
||||
}
|
||||
}
|
||||
@@ -143,7 +140,7 @@ func TestPostgresCorpusIsReadOnlyAndHTTPWorks(t *testing.T) {
|
||||
Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"})
|
||||
w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph")
|
||||
var page Page
|
||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || page.Total != 2 {
|
||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || len(page.Items) != 2 {
|
||||
t.Fatalf("SQL-backed HTTP list: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
w = request(mux, "GET", "/projects/thesisly/research/papers/1")
|
||||
@@ -179,8 +176,174 @@ func TestPostgresPublishedTextCodec(t *testing.T) {
|
||||
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &decoded) != nil || decoded.Title != p.Title || decoded.Abstract == nil || *decoded.Abstract != *p.Abstract {
|
||||
t.Fatalf("HTTP lost original control characters: %d %s", w.Code, w.Body.String())
|
||||
}
|
||||
page, err := r.List(ctx, Filters{Page: 1, PageSize: 8, Query: "graph"})
|
||||
if err != nil || page.Total != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title {
|
||||
page, err := r.List(ctx, Filters{PageSize: 8, Query: "graph"})
|
||||
if err != nil || len(page.Items) != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title {
|
||||
t.Fatalf("encoded corpus search: %+v %v", page, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresCursorsTraverseBothDirections(t *testing.T) {
|
||||
r, ctx := integrationRepository(t)
|
||||
for _, sort := range []string{"recent", "title"} {
|
||||
f := Filters{Sort: sort, PageSize: 1}
|
||||
seen := []string{}
|
||||
cursors := []string{""}
|
||||
for {
|
||||
page, err := r.List(ctx, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 1 {
|
||||
t.Fatalf("unexpected page %+v", page)
|
||||
}
|
||||
seen = append(seen, page.Items[0].ID)
|
||||
if len(seen) > 5 {
|
||||
t.Fatal("cursor cycle")
|
||||
}
|
||||
if len(seen) > 1 {
|
||||
back := f
|
||||
back.Cursor = page.PreviousCursor
|
||||
prior, err := r.List(ctx, back)
|
||||
if err != nil || len(prior.Items) != 1 || prior.Items[0].ID != seen[len(seen)-2] {
|
||||
t.Fatalf("reverse cursor: %+v %v", prior, err)
|
||||
}
|
||||
} else if page.PreviousCursor != "" {
|
||||
t.Fatal("first page has previous")
|
||||
}
|
||||
if page.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
f.Cursor = page.NextCursor
|
||||
cursors = append(cursors, f.Cursor)
|
||||
}
|
||||
want := []string{"5", "4", "3", "2", "1"}
|
||||
if sort == "title" {
|
||||
want = []string{"2", "5", "1", "3", "4"}
|
||||
}
|
||||
if !reflect.DeepEqual(seen, want) {
|
||||
t.Fatalf("%s: %v", sort, seen)
|
||||
}
|
||||
f.Cursor = cursors[1]
|
||||
f.Query = "different"
|
||||
if _, err := r.List(ctx, f); err == nil {
|
||||
t.Fatal("cursor reused across filters")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresEncodedTitleCursors(t *testing.T) {
|
||||
legacy, ctx := integrationRepository(t)
|
||||
r, err := NewPostgresRepository(legacy.db, "research_encoded_fixture")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = r.Check(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := Filters{Sort: "title", PageSize: 1}
|
||||
seen := []string{}
|
||||
for {
|
||||
page, err := r.List(ctx, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 1 {
|
||||
t.Fatalf("invalid encoded title page %+v", page)
|
||||
}
|
||||
seen = append(seen, page.Items[0].ID)
|
||||
if len(seen) > 5 {
|
||||
t.Fatal("cursor cycle")
|
||||
}
|
||||
if page.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
f.Cursor = page.NextCursor
|
||||
}
|
||||
if !reflect.DeepEqual(seen, []string{"2", "5", "1", "3", "4"}) {
|
||||
t.Fatalf("encoded key paging %v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresBoundedProbeDoesNotTruncateSparseMatches(t *testing.T) {
|
||||
legacy, ctx := integrationRepository(t)
|
||||
r, err := NewPostgresRepository(legacy.db, "research_sparse_fixture")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = r.Check(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, size := range []int{1, 2, 3} {
|
||||
f := Filters{Query: "graph", PageSize: size}
|
||||
seen := []string{}
|
||||
var previous []string
|
||||
for {
|
||||
page, err := r.List(ctx, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ids := paperIDs(page.Items)
|
||||
if len(ids) == 0 || len(ids) > size {
|
||||
t.Fatalf("invalid sparse page %+v", page)
|
||||
}
|
||||
if previous != nil {
|
||||
back := f
|
||||
back.Cursor = page.PreviousCursor
|
||||
prior, err := r.List(ctx, back)
|
||||
if err != nil || !reflect.DeepEqual(paperIDs(prior.Items), previous) {
|
||||
t.Fatalf("reverse gap %+v %v", prior, err)
|
||||
}
|
||||
}
|
||||
seen = append(seen, ids...)
|
||||
if len(seen) > 5 {
|
||||
t.Fatal("cursor cycle")
|
||||
}
|
||||
previous = ids
|
||||
if page.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
f.Cursor = page.NextCursor
|
||||
}
|
||||
if !reflect.DeepEqual(seen, []string{"10001", "10000", "8000", "3", "1"}) {
|
||||
t.Fatalf("size=%d lost sparse matches %v", size, seen)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresTopicCursorSkipsDuplicateEvidence(t *testing.T) {
|
||||
legacy, ctx := integrationRepository(t)
|
||||
r, err := NewPostgresRepository(legacy.db, "research_sparse_fixture")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := Filters{Topic: "cs.LG", PageSize: 1}
|
||||
seen := []string{}
|
||||
for {
|
||||
page, err := r.List(ctx, f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(page.Items) != 1 {
|
||||
t.Fatalf("invalid topic page %+v", page)
|
||||
}
|
||||
seen = append(seen, page.Items[0].ID)
|
||||
if len(seen) > 3 {
|
||||
t.Fatal("cursor cycle")
|
||||
}
|
||||
if len(seen) > 1 {
|
||||
back := f
|
||||
back.Cursor = page.PreviousCursor
|
||||
prior, err := r.List(ctx, back)
|
||||
if err != nil || len(prior.Items) != 1 || prior.Items[0].ID != seen[len(seen)-2] {
|
||||
t.Fatalf("reverse topic %+v %v", prior, err)
|
||||
}
|
||||
}
|
||||
if page.NextCursor == "" {
|
||||
break
|
||||
}
|
||||
f.Cursor = page.NextCursor
|
||||
}
|
||||
if !reflect.DeepEqual(seen, []string{"10001", "10000", "1"}) {
|
||||
t.Fatalf("duplicate evidence or lost topic matches %v", seen)
|
||||
}
|
||||
}
|
||||
|
||||
+21
-1
@@ -70,7 +70,8 @@ BEGIN
|
||||
END LOOP;
|
||||
END $$;
|
||||
UPDATE research_encoded_fixture.papers
|
||||
SET title='Graph'||chr(1)||chr(1)||'original'||chr(1)||chr(2),
|
||||
SET title_key='graph'||chr(1)||chr(1)||'key'||chr(1)||chr(2),
|
||||
title='Graph'||chr(1)||chr(1)||'original'||chr(1)||chr(2),
|
||||
abstract='Text'||chr(1)||chr(1)||'tail',
|
||||
authors_json='["Ada\u0000Smith","Group\u0001Name"]'
|
||||
WHERE id=1;
|
||||
@@ -79,3 +80,22 @@ SET metadata_json='{"comments":"Original\u0000comment"}' WHERE id=101;
|
||||
REVOKE ALL ON SCHEMA research_encoded_fixture FROM PUBLIC;
|
||||
GRANT USAGE ON SCHEMA research_encoded_fixture TO foundry_research_fixture;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA research_encoded_fixture TO foundry_research_fixture;
|
||||
|
||||
-- A large ID gap proves the bounded probe cannot truncate sparse results.
|
||||
CREATE SCHEMA research_sparse_fixture;
|
||||
DO $$
|
||||
DECLARE item text;
|
||||
BEGIN
|
||||
FOREACH item IN ARRAY ARRAY['papers','sources','source_records','identifiers','tags','paper_tags','venues','editions','submissions','decisions'] LOOP
|
||||
EXECUTE format('CREATE TABLE research_sparse_fixture.%I (LIKE research_fixture.%I INCLUDING ALL)',item,item);
|
||||
EXECUTE format('INSERT INTO research_sparse_fixture.%I SELECT * FROM research_fixture.%I',item,item);
|
||||
END LOOP;
|
||||
END $$;
|
||||
INSERT INTO research_sparse_fixture.papers VALUES
|
||||
(8000,'Graph in expanded interval','Graph evidence.','graph expanded','[]',NULL),
|
||||
(10000,'Graph near cursor','Graph evidence.','graph near cursor','[]',NULL),
|
||||
(10001,'Graph first','Graph evidence.','graph first','[]',NULL);
|
||||
INSERT INTO research_sparse_fixture.paper_tags VALUES(10000,1,'first'),(10000,1,'second'),(10001,1,'first');
|
||||
REVOKE ALL ON SCHEMA research_sparse_fixture FROM PUBLIC;
|
||||
GRANT USAGE ON SCHEMA research_sparse_fixture TO foundry_research_fixture;
|
||||
GRANT SELECT ON ALL TABLES IN SCHEMA research_sparse_fixture TO foundry_research_fixture;
|
||||
|
||||
Reference in New Issue
Block a user