350 lines
11 KiB
Go
350 lines
11 KiB
Go
package research
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"os"
|
|
"reflect"
|
|
"testing"
|
|
"time"
|
|
|
|
_ "github.com/jackc/pgx/v5/stdlib"
|
|
)
|
|
|
|
func integrationRepository(t *testing.T) (*PostgresRepository, context.Context) {
|
|
t.Helper()
|
|
dsn := os.Getenv("RESEARCH_TEST_DATABASE_URL")
|
|
if dsn == "" {
|
|
t.Skip("requires the disposable research fixture database")
|
|
}
|
|
db, err := sql.Open("pgx", dsn)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
t.Cleanup(func() { _ = db.Close() })
|
|
db.SetMaxOpenConns(2)
|
|
repository, err := NewPostgresRepository(db, "research_fixture")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
|
|
t.Cleanup(cancel)
|
|
return repository, ctx
|
|
}
|
|
|
|
func paperIDs(papers []Paper) []string {
|
|
ids := []string{}
|
|
for _, paper := range papers {
|
|
ids = append(ids, paper.ID)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func TestPostgresPageSearchAndFilters(t *testing.T) {
|
|
r, ctx := integrationRepository(t)
|
|
if err := r.Check(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, test := range []struct {
|
|
filters Filters
|
|
ids []string
|
|
}{
|
|
{Filters{PageSize: 2}, []string{"5", "4"}},
|
|
{Filters{PageSize: 2, Sort: "title"}, []string{"2", "5"}},
|
|
{Filters{PageSize: 8, Query: "graph"}, []string{"3", "1"}},
|
|
{Filters{PageSize: 8, Query: "graph OR zebra"}, []string{}},
|
|
{Filters{PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, []string{"1"}},
|
|
{Filters{PageSize: 8, Topic: "cs.LG"}, []string{"1"}},
|
|
{Filters{PageSize: 8, Topic: "cs.DC", Query: "graph"}, []string{"3"}},
|
|
{Filters{PageSize: 8, Query: `x'); DROP TABLE papers;--`}, []string{}},
|
|
} {
|
|
page, err := r.List(ctx, test.filters)
|
|
if err != nil {
|
|
t.Fatalf("%+v: %v", test.filters, err)
|
|
}
|
|
if !reflect.DeepEqual(paperIDs(page.Items), test.ids) {
|
|
t.Fatalf("%+v: ids=%v", test.filters, paperIDs(page.Items))
|
|
}
|
|
if !reflect.DeepEqual(page.Topics, []string{"cs.AI", "cs.DC", "cs.LG"}) {
|
|
t.Fatalf("mixed tag namespaces: %v", page.Topics)
|
|
}
|
|
if page.PageSize != test.filters.PageSize {
|
|
t.Fatal("lost paging metadata")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPostgresMetadataPreservesEvidence(t *testing.T) {
|
|
r, ctx := integrationRepository(t)
|
|
p, err := r.Get(ctx, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p.Title != "Graph & learning <original>" || p.Abstract == nil || *p.Abstract != "Neural graph models preserve original abstracts." || !reflect.DeepEqual(p.Authors, []string{"Ada Smith", "The Research Group"}) {
|
|
t.Fatalf("original content changed: %+v", p)
|
|
}
|
|
if !reflect.DeepEqual(p.Topics, []string{"cs.AI", "cs.LG"}) || len(p.Sources) != 1 || p.Sources[0].ExternalID != "fixture-1" || p.Sources[0].Label != "arXiv" {
|
|
t.Fatalf("incorrect source/tag joins: %+v", p)
|
|
}
|
|
if p.DOI == nil || *p.DOI != "10.fixture/canonical" || p.Journal == nil || *p.Journal != "Journal Fixture" || p.Comments == nil || *p.Comments != "Original comment" {
|
|
t.Fatalf("metadata missing: %+v", p)
|
|
}
|
|
if len(p.Publications) != 1 || p.Publications[0].Status != "withdrawn" || p.Publications[0].EvidenceURL != "https://example.test/decision/3" {
|
|
t.Fatalf("latest decision ordering changed: %+v", p.Publications)
|
|
}
|
|
p, err = r.Get(ctx, 3)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !reflect.DeepEqual(p.Authors, []string{"Carla Network"}) || len(p.Sources) != 2 || p.DOI == nil || *p.DOI != "10.fixture/other" || len(p.Publications) != 1 || p.Publications[0].Status != "unknown" || p.Publications[0].EvidenceURL != "" {
|
|
t.Fatalf("unknown decision/source mapping: %+v", p)
|
|
}
|
|
p, err = r.Get(ctx, 2)
|
|
if err != nil || p.Abstract != nil || p.PublishedAt != nil || p.Sources == nil || p.Topics == nil {
|
|
t.Fatalf("nullable data changed: %+v %v", p, err)
|
|
}
|
|
if _, err = r.Get(ctx, 99999); !errors.Is(err, sql.ErrNoRows) {
|
|
t.Fatalf("missing paper: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestPostgresCorpusIsReadOnlyAndHTTPWorks(t *testing.T) {
|
|
r, ctx := integrationRepository(t)
|
|
tx, err := r.begin(ctx)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var readOnly, isolation, jit string
|
|
if err = tx.QueryRowContext(ctx, "SHOW transaction_read_only").Scan(&readOnly); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = tx.QueryRowContext(ctx, "SHOW transaction_isolation").Scan(&isolation); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = tx.QueryRowContext(ctx, "SHOW jit").Scan(&jit); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_ = tx.Rollback()
|
|
if readOnly != "on" || isolation != "repeatable read" || jit != "off" {
|
|
t.Fatalf("unexpected read transaction settings: %s %s jit=%s", readOnly, isolation, jit)
|
|
}
|
|
_, err = r.db.ExecContext(ctx, `UPDATE research_fixture.papers SET title='unexpected write' WHERE id=1`)
|
|
var state interface{ SQLState() string }
|
|
if !errors.As(err, &state) || state.SQLState() != "42501" {
|
|
t.Fatalf("expected database permission denial, got %v", err)
|
|
}
|
|
mux := http.NewServeMux()
|
|
Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"})
|
|
w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph")
|
|
var page Page
|
|
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || len(page.Items) != 2 {
|
|
t.Fatalf("SQL-backed HTTP list: %d %s", w.Code, w.Body.String())
|
|
}
|
|
w = request(mux, "GET", "/projects/thesisly/research/papers/1")
|
|
var paper Paper
|
|
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &paper) != nil || paper.ID != "1" {
|
|
t.Fatalf("SQL-backed HTTP detail: %d %s", w.Code, w.Body.String())
|
|
}
|
|
if w = request(mux, "GET", "/projects/thesisly/research/papers/99999"); w.Code != 404 {
|
|
t.Fatalf("missing HTTP status %d", w.Code)
|
|
}
|
|
}
|
|
|
|
func TestPostgresPublishedTextCodec(t *testing.T) {
|
|
legacy, ctx := integrationRepository(t)
|
|
r, err := NewPostgresRepository(legacy.db, "research_encoded_fixture")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = r.Check(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
p, err := r.Get(ctx, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if p.Title != "Graph\x00original\x01" || p.Abstract == nil || *p.Abstract != "Text\x00tail" || !reflect.DeepEqual(p.Authors, []string{"Ada\x00Smith", "Group\x01Name"}) || p.Comments == nil || *p.Comments != "Original\x00comment" {
|
|
t.Fatalf("published text was not restored: %+v", p)
|
|
}
|
|
mux := http.NewServeMux()
|
|
Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"})
|
|
w := request(mux, "GET", "/projects/thesisly/research/papers/1")
|
|
var decoded Paper
|
|
if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &decoded) != nil || decoded.Title != p.Title || decoded.Abstract == nil || *decoded.Abstract != *p.Abstract {
|
|
t.Fatalf("HTTP lost original control characters: %d %s", w.Code, w.Body.String())
|
|
}
|
|
page, err := r.List(ctx, Filters{PageSize: 8, Query: "graph"})
|
|
if err != nil || len(page.Items) != 2 || len(page.Items) != 2 || page.Items[1].Title != p.Title {
|
|
t.Fatalf("encoded corpus search: %+v %v", page, err)
|
|
}
|
|
}
|
|
|
|
func TestPostgresCursorsTraverseBothDirections(t *testing.T) {
|
|
r, ctx := integrationRepository(t)
|
|
for _, sort := range []string{"recent", "title"} {
|
|
f := Filters{Sort: sort, PageSize: 1}
|
|
seen := []string{}
|
|
cursors := []string{""}
|
|
for {
|
|
page, err := r.List(ctx, f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(page.Items) != 1 {
|
|
t.Fatalf("unexpected page %+v", page)
|
|
}
|
|
seen = append(seen, page.Items[0].ID)
|
|
if len(seen) > 5 {
|
|
t.Fatal("cursor cycle")
|
|
}
|
|
if len(seen) > 1 {
|
|
back := f
|
|
back.Cursor = page.PreviousCursor
|
|
prior, err := r.List(ctx, back)
|
|
if err != nil || len(prior.Items) != 1 || prior.Items[0].ID != seen[len(seen)-2] {
|
|
t.Fatalf("reverse cursor: %+v %v", prior, err)
|
|
}
|
|
} else if page.PreviousCursor != "" {
|
|
t.Fatal("first page has previous")
|
|
}
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
f.Cursor = page.NextCursor
|
|
cursors = append(cursors, f.Cursor)
|
|
}
|
|
want := []string{"5", "4", "3", "2", "1"}
|
|
if sort == "title" {
|
|
want = []string{"2", "5", "1", "3", "4"}
|
|
}
|
|
if !reflect.DeepEqual(seen, want) {
|
|
t.Fatalf("%s: %v", sort, seen)
|
|
}
|
|
f.Cursor = cursors[1]
|
|
f.Query = "different"
|
|
if _, err := r.List(ctx, f); err == nil {
|
|
t.Fatal("cursor reused across filters")
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPostgresEncodedTitleCursors(t *testing.T) {
|
|
legacy, ctx := integrationRepository(t)
|
|
r, err := NewPostgresRepository(legacy.db, "research_encoded_fixture")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = r.Check(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f := Filters{Sort: "title", PageSize: 1}
|
|
seen := []string{}
|
|
for {
|
|
page, err := r.List(ctx, f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(page.Items) != 1 {
|
|
t.Fatalf("invalid encoded title page %+v", page)
|
|
}
|
|
seen = append(seen, page.Items[0].ID)
|
|
if len(seen) > 5 {
|
|
t.Fatal("cursor cycle")
|
|
}
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
f.Cursor = page.NextCursor
|
|
}
|
|
if !reflect.DeepEqual(seen, []string{"2", "5", "1", "3", "4"}) {
|
|
t.Fatalf("encoded key paging %v", seen)
|
|
}
|
|
}
|
|
|
|
func TestPostgresBoundedProbeDoesNotTruncateSparseMatches(t *testing.T) {
|
|
legacy, ctx := integrationRepository(t)
|
|
r, err := NewPostgresRepository(legacy.db, "research_sparse_fixture")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err = r.Check(ctx); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, size := range []int{1, 2, 3} {
|
|
f := Filters{Query: "graph", PageSize: size}
|
|
seen := []string{}
|
|
var previous []string
|
|
for {
|
|
page, err := r.List(ctx, f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
ids := paperIDs(page.Items)
|
|
if len(ids) == 0 || len(ids) > size {
|
|
t.Fatalf("invalid sparse page %+v", page)
|
|
}
|
|
if previous != nil {
|
|
back := f
|
|
back.Cursor = page.PreviousCursor
|
|
prior, err := r.List(ctx, back)
|
|
if err != nil || !reflect.DeepEqual(paperIDs(prior.Items), previous) {
|
|
t.Fatalf("reverse gap %+v %v", prior, err)
|
|
}
|
|
}
|
|
seen = append(seen, ids...)
|
|
if len(seen) > 5 {
|
|
t.Fatal("cursor cycle")
|
|
}
|
|
previous = ids
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
f.Cursor = page.NextCursor
|
|
}
|
|
if !reflect.DeepEqual(seen, []string{"10001", "10000", "8000", "3", "1"}) {
|
|
t.Fatalf("size=%d lost sparse matches %v", size, seen)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPostgresTopicCursorSkipsDuplicateEvidence(t *testing.T) {
|
|
legacy, ctx := integrationRepository(t)
|
|
r, err := NewPostgresRepository(legacy.db, "research_sparse_fixture")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
f := Filters{Topic: "cs.LG", PageSize: 1}
|
|
seen := []string{}
|
|
for {
|
|
page, err := r.List(ctx, f)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(page.Items) != 1 {
|
|
t.Fatalf("invalid topic page %+v", page)
|
|
}
|
|
seen = append(seen, page.Items[0].ID)
|
|
if len(seen) > 3 {
|
|
t.Fatal("cursor cycle")
|
|
}
|
|
if len(seen) > 1 {
|
|
back := f
|
|
back.Cursor = page.PreviousCursor
|
|
prior, err := r.List(ctx, back)
|
|
if err != nil || len(prior.Items) != 1 || prior.Items[0].ID != seen[len(seen)-2] {
|
|
t.Fatalf("reverse topic %+v %v", prior, err)
|
|
}
|
|
}
|
|
if page.NextCursor == "" {
|
|
break
|
|
}
|
|
f.Cursor = page.NextCursor
|
|
}
|
|
if !reflect.DeepEqual(seen, []string{"10001", "10000", "1"}) {
|
|
t.Fatalf("duplicate evidence or lost topic matches %v", seen)
|
|
}
|
|
}
|