diff --git a/deploy/check-schema.sh b/deploy/check-schema.sh index ab87d48..941baa7 100644 --- a/deploy/check-schema.sh +++ b/deploy/check-schema.sh @@ -67,12 +67,14 @@ END $$; ROLLBACK; SQL printf 'Migration preserved existing logins and enforced project isolation.\n' -for package in platform auth; do +psql < internal/research/testdata/corpus.sql +for package in platform auth research; do CGO_ENABLED=0 go test -mod=readonly -c -o "bin/$package-tests" "./internal/$package" docker run --rm --network "container:$container" --user 1000:1000 \ --read-only --cap-drop ALL --security-opt no-new-privileges \ --tmpfs /tmp:rw,nosuid,nodev,noexec,size=16m,mode=1777 \ --mount "type=bind,source=$(pwd)/bin/$package-tests,target=/opt/integration-tests,readonly" \ -e PLATFORM_TEST_DATABASE_URL=postgres://foundry_fixture@127.0.0.1:5432/postgres?sslmode=disable \ + -e RESEARCH_TEST_DATABASE_URL=postgres://foundry_research_fixture@127.0.0.1:5432/postgres?sslmode=disable \ --entrypoint /opt/integration-tests postgres:17-alpine -test.v done diff --git a/docs/research-api.md b/docs/research-api.md index 16570b1..d15ee56 100644 --- a/docs/research-api.md +++ b/docs/research-api.md @@ -75,4 +75,13 @@ given name, family name and suffix; null abstracts/dates remain null. Publicatio status comes from the latest observed decision, using decision date and ID as tie-breakers, and stays unknown when no decision exists. Unit checks cover author shapes, bound user values and schema qualification; real PostgreSQL -integration checks remain pending the migration. +corpus checks remain pending the migration. + +The existing `deploy/check-schema.sh` also loads the synthetic fixture in +`internal/research/testdata/corpus.sql` inside its disposable PostgreSQL +container (`--network none`, no host port). The research test process receives a +separate SELECT-only fixture role via `RESEARCH_TEST_DATABASE_URL`. These +integration checks execute real SQL for filtering, stable paging, nullable +content, source/tag/identifier joins, latest decision tie-breakers, permission +denial and SQL-backed HTTP routes. They never connect to the shared migration +instance and do not establish full-corpus performance or live-data correctness. diff --git a/internal/research/postgres_integration_test.go b/internal/research/postgres_integration_test.go new file mode 100644 index 0000000..e93c097 --- /dev/null +++ b/internal/research/postgres_integration_test.go @@ -0,0 +1,154 @@ +package research + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "net/http" + "os" + "reflect" + "testing" + "time" + + _ "github.com/jackc/pgx/v5/stdlib" +) + +func integrationRepository(t *testing.T) (*PostgresRepository, context.Context) { + t.Helper() + dsn := os.Getenv("RESEARCH_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("requires the disposable research fixture database") + } + db, err := sql.Open("pgx", dsn) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + db.SetMaxOpenConns(2) + repository, err := NewPostgresRepository(db, "research_fixture") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + t.Cleanup(cancel) + return repository, ctx +} + +func paperIDs(papers []Paper) []string { + ids := []string{} + for _, paper := range papers { + ids = append(ids, paper.ID) + } + return ids +} + +func TestPostgresPageSearchAndFilters(t *testing.T) { + r, ctx := integrationRepository(t) + if err := r.Check(ctx); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + filters Filters + total int64 + ids []string + }{ + {Filters{Page: 1, PageSize: 2}, 5, []string{"5", "4"}}, + {Filters{Page: 2, PageSize: 2}, 5, []string{"3", "2"}}, + {Filters{Page: 3, PageSize: 2}, 5, []string{"1"}}, + {Filters{Page: 1, PageSize: 2, Sort: "title"}, 5, []string{"2", "5"}}, + {Filters{Page: 1, PageSize: 8, Query: "graph"}, 2, []string{"3", "1"}}, + {Filters{Page: 1, PageSize: 8, Query: "graph OR zebra"}, 0, []string{}}, + {Filters{Page: 1, PageSize: 8, Query: "Ada Smith", SearchIn: "author"}, 1, []string{"1"}}, + {Filters{Page: 1, PageSize: 8, Topic: "cs.LG"}, 1, []string{"1"}}, + {Filters{Page: 1, PageSize: 8, Topic: "cs.DC", Query: "graph"}, 1, []string{"3"}}, + {Filters{Page: 1, PageSize: 8, Query: `x'); DROP TABLE papers;--`}, 0, []string{}}, + } { + page, err := r.List(ctx, test.filters) + if err != nil { + t.Fatalf("%+v: %v", test.filters, err) + } + if page.Total != test.total || !reflect.DeepEqual(paperIDs(page.Items), test.ids) { + t.Fatalf("%+v: total=%d ids=%v", test.filters, page.Total, paperIDs(page.Items)) + } + if !reflect.DeepEqual(page.Topics, []string{"cs.AI", "cs.DC", "cs.LG"}) { + t.Fatalf("mixed tag namespaces: %v", page.Topics) + } + if page.Page != test.filters.Page || page.PageSize != test.filters.PageSize { + t.Fatal("lost paging metadata") + } + } +} + +func TestPostgresMetadataPreservesEvidence(t *testing.T) { + r, ctx := integrationRepository(t) + p, err := r.Get(ctx, 1) + if err != nil { + t.Fatal(err) + } + if p.Title != "Graph & learning " || p.Abstract == nil || *p.Abstract != "Neural graph models preserve original abstracts." || !reflect.DeepEqual(p.Authors, []string{"Ada Smith", "The Research Group"}) { + t.Fatalf("original content changed: %+v", p) + } + if !reflect.DeepEqual(p.Topics, []string{"cs.AI", "cs.LG"}) || len(p.Sources) != 1 || p.Sources[0].ExternalID != "fixture-1" || p.Sources[0].Label != "arXiv" { + t.Fatalf("incorrect source/tag joins: %+v", p) + } + if p.DOI == nil || *p.DOI != "10.fixture/canonical" || p.Journal == nil || *p.Journal != "Journal Fixture" || p.Comments == nil || *p.Comments != "Original comment" { + t.Fatalf("metadata missing: %+v", p) + } + if len(p.Publications) != 1 || p.Publications[0].Status != "withdrawn" || p.Publications[0].EvidenceURL != "https://example.test/decision/3" { + t.Fatalf("latest decision ordering changed: %+v", p.Publications) + } + p, err = r.Get(ctx, 3) + if err != nil { + t.Fatal(err) + } + if len(p.Sources) != 2 || p.DOI == nil || *p.DOI != "10.fixture/other" || len(p.Publications) != 1 || p.Publications[0].Status != "unknown" || p.Publications[0].EvidenceURL != "" { + t.Fatalf("unknown decision/source mapping: %+v", p) + } + p, err = r.Get(ctx, 2) + if err != nil || p.Abstract != nil || p.PublishedAt != nil || p.Sources == nil || p.Topics == nil { + t.Fatalf("nullable data changed: %+v %v", p, err) + } + if _, err = r.Get(ctx, 99999); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("missing paper: %v", err) + } +} + +func TestPostgresCorpusIsReadOnlyAndHTTPWorks(t *testing.T) { + r, ctx := integrationRepository(t) + tx, err := r.begin(ctx) + if err != nil { + t.Fatal(err) + } + var readOnly, isolation string + if err = tx.QueryRowContext(ctx, "SHOW transaction_read_only").Scan(&readOnly); err != nil { + t.Fatal(err) + } + if err = tx.QueryRowContext(ctx, "SHOW transaction_isolation").Scan(&isolation); err != nil { + t.Fatal(err) + } + _ = tx.Rollback() + if readOnly != "on" || isolation != "repeatable read" { + t.Fatalf("unsafe read transaction: %s %s", readOnly, isolation) + } + _, err = r.db.ExecContext(ctx, `UPDATE research_fixture.papers SET title='unexpected write' WHERE id=1`) + var state interface{ SQLState() string } + if !errors.As(err, &state) || state.SQLState() != "42501" { + t.Fatalf("expected database permission denial, got %v", err) + } + mux := http.NewServeMux() + Register(mux, r, map[string]string{"thesisly": "http://localhost:17322/auth/callback"}) + w := request(mux, "GET", "/projects/thesisly/research/papers?q=graph") + var page Page + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &page) != nil || page.Total != 2 { + t.Fatalf("SQL-backed HTTP list: %d %s", w.Code, w.Body.String()) + } + w = request(mux, "GET", "/projects/thesisly/research/papers/1") + var paper Paper + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &paper) != nil || paper.ID != "1" { + t.Fatalf("SQL-backed HTTP detail: %d %s", w.Code, w.Body.String()) + } + if w = request(mux, "GET", "/projects/thesisly/research/papers/99999"); w.Code != 404 { + t.Fatalf("missing HTTP status %d", w.Code) + } +} diff --git a/internal/research/testdata/corpus.sql b/internal/research/testdata/corpus.sql new file mode 100644 index 0000000..48323aa --- /dev/null +++ b/internal/research/testdata/corpus.sql @@ -0,0 +1,59 @@ +-- Synthetic fixture for the disposable, network-isolated CI database only. +-- Required columns/types follow Paper Atlas's fixed a3b677b migration. +CREATE SCHEMA research_fixture; +CREATE TABLE research_fixture.papers ( + id bigint PRIMARY KEY, title text NOT NULL, abstract text, title_key text COLLATE "C" NOT NULL, + authors_json text NOT NULL, first_published text +); +CREATE INDEX papers_title_key ON research_fixture.papers(title_key); +CREATE INDEX papers_search_gin ON research_fixture.papers USING gin + (to_tsvector('english'::regconfig,coalesce(title,'') || ' ' || coalesce(abstract,''))); +CREATE TABLE research_fixture.sources (id bigint PRIMARY KEY,name text NOT NULL); +CREATE TABLE research_fixture.source_records ( + id bigint PRIMARY KEY,source_id bigint REFERENCES research_fixture.sources(id), + paper_id bigint REFERENCES research_fixture.papers(id),external_id text,url text,metadata_json text +); +CREATE TABLE research_fixture.identifiers (scheme text,value text,paper_id bigint REFERENCES research_fixture.papers(id),PRIMARY KEY(scheme,value)); +CREATE TABLE research_fixture.tags (id bigint PRIMARY KEY,namespace text,value text,UNIQUE(namespace,value)); +CREATE TABLE research_fixture.paper_tags ( + paper_id bigint REFERENCES research_fixture.papers(id),tag_id bigint REFERENCES research_fixture.tags(id), + evidence_key text,PRIMARY KEY(paper_id,tag_id,evidence_key) +); +CREATE TABLE research_fixture.venues (id bigint PRIMARY KEY,acronym text); +CREATE TABLE research_fixture.editions (id bigint PRIMARY KEY,venue_id bigint REFERENCES research_fixture.venues(id),year bigint,track text); +CREATE TABLE research_fixture.submissions ( + id bigint PRIMARY KEY,paper_id bigint REFERENCES research_fixture.papers(id),edition_id bigint REFERENCES research_fixture.editions(id) +); +CREATE TABLE research_fixture.decisions ( + id bigint PRIMARY KEY,submission_id bigint REFERENCES research_fixture.submissions(id), + status text,evidence_url text,observed_at text,decision_date text +); +CREATE INDEX decisions_submission_order ON research_fixture.decisions(submission_id,observed_at DESC,id DESC); + +INSERT INTO research_fixture.papers VALUES + (1,'Graph & learning ','Neural graph models preserve original abstracts.','graph learning','[["Smith","Ada",""],"The Research Group"]','2024-01-02'), + (2,'Alpha topology',NULL,'alpha topology','["Bruno Park"]',NULL), + (3,'Graph systems','Graph systems and data.','graph systems','["Carla Network"]','2023-01-01'), + (4,'Zebra methods','Evidence and uncertainty.','zebra methods','[]','2022-01-01'), + (5,'Alpha Topology','Another independent record.','alpha topology','["Dana Other"]','2025-01-01'); +INSERT INTO research_fixture.sources VALUES(10,'arxiv'),(20,'pmlr'); +INSERT INTO research_fixture.source_records VALUES + (101,10,1,'fixture-1','https://example.test/papers/1','{"journal-ref":"Journal Fixture","comments":"Original comment"}'), + (202,20,3,'fixture-3','https://example.test/papers/3','{"doi":"10.fixture/other"}'), + (203,10,3,'fixture-3-arxiv','https://example.test/papers/3/arxiv','{}'); +INSERT INTO research_fixture.identifiers VALUES('doi','10.fixture/canonical',1); +INSERT INTO research_fixture.tags VALUES(1,'arxiv:category','cs.LG'),(2,'arxiv:category','cs.AI'),(3,'arxiv:category','cs.DC'),(4,'different:namespace','cs.LG'); +INSERT INTO research_fixture.paper_tags VALUES(1,1,'first'),(1,1,'second'),(1,2,'first'),(3,3,'first'),(2,4,'first'); +INSERT INTO research_fixture.venues VALUES(1,'ICLR'),(2,'ICML'); +INSERT INTO research_fixture.editions VALUES(7,1,2024,'main'),(8,2,2023,'main'); +INSERT INTO research_fixture.submissions VALUES(11,1,7),(12,3,8); +INSERT INTO research_fixture.decisions VALUES + (1,11,'accepted','https://example.test/decision/1','2024-02-01','2024-01-30'), + (2,11,'rejected','https://example.test/decision/2','2024-02-01','2024-01-31'), + (3,11,'withdrawn','https://example.test/decision/3','2024-02-01','2024-01-31'), + (4,11,'accepted','https://example.test/decision/4','2024-01-01','2099-01-01'); + +CREATE ROLE foundry_research_fixture LOGIN; +REVOKE ALL ON SCHEMA research_fixture FROM PUBLIC; +GRANT USAGE ON SCHEMA research_fixture TO foundry_research_fixture; +GRANT SELECT ON ALL TABLES IN SCHEMA research_fixture TO foundry_research_fixture;