Prepare read-only PostgreSQL paper queries and optional Core wiring
This commit is contained in:
@@ -16,3 +16,10 @@ SMTP_FROM=noreply@notice.entroia.com
|
||||
# Optional Sign in with Slack.
|
||||
SLACK_CLIENT_ID=replace-me
|
||||
SLACK_CLIENT_SECRET=replace-me
|
||||
|
||||
# Optional shared paper corpus; enable only after PostgreSQL publication.
|
||||
# Store the real read-only connection string in the protected server research.env.
|
||||
FOUNDRY_RESEARCH_DATABASE_URL=
|
||||
FOUNDRY_RESEARCH_SCHEMA=research
|
||||
# Deployment migration container; independent of database/schema names.
|
||||
FOUNDRY_POSTGRES_CONTAINER=foundry-postgres
|
||||
|
||||
@@ -74,6 +74,7 @@ jobs:
|
||||
env:
|
||||
PUBLIC_ORIGIN: https://api.example.com
|
||||
DATABASE_URL: postgres://foundry:fixture@127.0.0.1:5432/foundry
|
||||
FOUNDRY_RESEARCH_DATABASE_URL: ''
|
||||
GOOGLE_CLIENT_ID: fixture
|
||||
GOOGLE_CLIENT_SECRET: fixture
|
||||
FOUNDRY_PROJECTS: '{"notes":"http://localhost:3000/auth/callback"}'
|
||||
|
||||
@@ -10,6 +10,10 @@
|
||||
4. 由 Gitea Actions 在 `ces-control` 上使用 Go 1.25 构建并检查。部署工作流仅处理 `main` 上影响后端或部署的改动,将二进制启动在服务器 Docker 私网,由 Traefik 提供 HTTPS。通过 `GET /healthz` 检查进程,通过 `GET /readyz` 检查认证库。
|
||||
5. 代理避免记录 Google 回调的查询参数及认证请求头。设置入口限流,并定时删除三张临时表(`oauth_attempts`、`login_grants`、`project_sessions`)中 `expires_at <= now()` 的记录。
|
||||
|
||||
## 共享论文库(Thesisly)
|
||||
|
||||
可选只读接入见[共享论文 API](docs/research-api.md)。Thesisly 使用同一后端的 `/projects/{project}/research/papers` 列表、搜索和详情;只有配置服务器上的 `FOUNDRY_RESEARCH_DATABASE_URL` 时才启用。连接串保存在受保护的 `research.env`,不放入前端或 Actions Variables。当前仍等待 Paper Atlas 全量 PostgreSQL 迁移完成及真实数据验收,尚未启用线上论文接口。
|
||||
|
||||
## Actions 部署配置
|
||||
|
||||
在仓库的 Actions Secrets 配置 `GOOGLE_CLIENT_ID` 和 `GOOGLE_CLIENT_SECRET`,在 Actions Variables 配置 `FOUNDRY_PROJECTS`。变量值为 JSON 对象,例如 `{"prompt":"http://localhost:17321/auth/callback"}`。部署读取服务器上预先配置的 `/opt/endoleap-infra/foundry/database.env`,OAuth 值由 Actions 在启动容器时注入。
|
||||
|
||||
@@ -13,6 +13,7 @@ import (
|
||||
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/auth"
|
||||
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/codex"
|
||||
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/platform"
|
||||
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/research"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
)
|
||||
@@ -61,6 +62,33 @@ func main() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
var papers research.Repository
|
||||
if researchDSN := os.Getenv("FOUNDRY_RESEARCH_DATABASE_URL"); researchDSN != "" {
|
||||
researchConfig, err := pgx.ParseConfig(researchDSN)
|
||||
if err != nil {
|
||||
log.Fatal("Invalid research database configuration")
|
||||
}
|
||||
researchConfig.RuntimeParams["default_transaction_read_only"] = "on"
|
||||
researchConfig.RuntimeParams["application_name"] = "foundry-research"
|
||||
researchDB := stdlib.OpenDB(*researchConfig)
|
||||
defer researchDB.Close()
|
||||
researchDB.SetMaxOpenConns(2)
|
||||
researchDB.SetMaxIdleConns(2)
|
||||
researchDB.SetConnMaxIdleTime(5 * time.Minute)
|
||||
researchDB.SetConnMaxLifetime(30 * time.Minute)
|
||||
repository, err := research.NewPostgresRepository(researchDB, env("FOUNDRY_RESEARCH_SCHEMA", "research"))
|
||||
if err != nil {
|
||||
log.Fatal("Invalid research database configuration")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
err = repository.Check(ctx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
log.Fatal("Research database is unavailable or not published")
|
||||
}
|
||||
papers = repository
|
||||
}
|
||||
research.Register(mux, papers, callbacks)
|
||||
if err := codex.Register(mux, callbacks, os.Getenv("FOUNDRY_CODEX_SOCKET"), os.Getenv("FOUNDRY_CODEX_GRANTS")); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
+2
-1
@@ -2,12 +2,13 @@
|
||||
"""Apply checked migrations atomically from the trusted deployment runner."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--container", default="endoleap-dev-postgres-1")
|
||||
parser.add_argument("--container", default=os.environ.get("FOUNDRY_POSTGRES_CONTAINER", "foundry-postgres"))
|
||||
parser.add_argument("--database", default="foundry_auth")
|
||||
parser.add_argument("--role", default="foundry")
|
||||
args = parser.parse_args()
|
||||
|
||||
+5
-1
@@ -53,6 +53,10 @@ mail_env=()
|
||||
if [[ -s "$foundry/mail.env" ]]; then
|
||||
mail_env=(--env-file "$foundry/mail.env")
|
||||
fi
|
||||
research_env=()
|
||||
if [[ -s "$foundry/research.env" ]]; then
|
||||
research_env=(--env-file "$foundry/research.env")
|
||||
fi
|
||||
slack_env=()
|
||||
if [[ -n "${SLACK_CLIENT_ID:-}" || -n "${SLACK_CLIENT_SECRET:-}" ]]; then
|
||||
for name in SLACK_CLIENT_ID SLACK_CLIENT_SECRET; do
|
||||
@@ -118,7 +122,7 @@ if [[ -d "$codex_dir" ]]; then
|
||||
)
|
||||
fi
|
||||
|
||||
docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack_env[@]}" --network "$network" \
|
||||
docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack_env[@]}" "${research_env[@]}" --network "$network" \
|
||||
--env-file "$foundry/database.env" -e HOST -e PORT -e PUBLIC_ORIGIN \
|
||||
-e GOOGLE_CLIENT_ID -e GOOGLE_CLIENT_SECRET -e FOUNDRY_PROJECTS -e FOUNDRY_ONBOARDING \
|
||||
--user 1000:1000 --cap-drop ALL --security-opt no-new-privileges \
|
||||
|
||||
+35
-4
@@ -6,12 +6,29 @@ collections remain project scoped.
|
||||
|
||||
## Current implementation boundary
|
||||
|
||||
`internal/research` implements the HTTP contract and repository boundary. It is
|
||||
not registered by `cmd/foundry` yet and does not open any database connection.
|
||||
`internal/research` implements the HTTP contract and a PostgreSQL repository
|
||||
based on the fixed Paper Atlas `a3b677b` schema (TEXT/BIGINT columns, native
|
||||
English title/abstract search index). `cmd/foundry` enables it only when
|
||||
`FOUNDRY_RESEARCH_DATABASE_URL` is configured. No live research connection has
|
||||
been configured during the ongoing migration.
|
||||
The existing SQLite implementation was withdrawn. Wait for the separate Paper
|
||||
Atlas PostgreSQL migration to finish, inspect the published schema and indexes,
|
||||
then implement the repository with a least-privilege read-only connection and
|
||||
register it in the existing server. Do not add `FOUNDRY_RESEARCH_DATABASE`.
|
||||
then verify the queries, indexes and full-corpus performance with a
|
||||
least-privilege read-only connection before enabling it in the existing server.
|
||||
Do not add `FOUNDRY_RESEARCH_DATABASE`.
|
||||
|
||||
The server-side deployment may load a protected
|
||||
`/opt/endoleap-infra/foundry/research.env` containing
|
||||
`FOUNDRY_RESEARCH_DATABASE_URL` and `FOUNDRY_RESEARCH_SCHEMA` (default `research`).
|
||||
Keep the connection string out of Git, Actions variables, browser configuration
|
||||
and command arguments. Use a dedicated role with schema USAGE and table SELECT
|
||||
only. Core uses a separate two-connection pool, read-only connection defaults and
|
||||
read-only transactions. Startup checks the published schema/SELECT permissions;
|
||||
an explicitly enabled but invalid connection fails candidate readiness.
|
||||
|
||||
Deployment migration defaults now follow the shared `foundry-postgres`
|
||||
container name. `FOUNDRY_POSTGRES_CONTAINER` or the existing `--container` option
|
||||
can select a different container without changing database/table names.
|
||||
|
||||
## Routes
|
||||
|
||||
@@ -28,6 +45,12 @@ or repeated parameters return 400. `recent` means descending stable paper ID,
|
||||
not publication date. Search terms must become bound query parameters, never
|
||||
raw SQL or user-supplied full-text syntax.
|
||||
|
||||
Title/abstract search uses the migration's English vector with
|
||||
`plainto_tsquery`; author search uses simple-language terms over the stored
|
||||
author JSON. Both treat input as terms, never full-text operators. Author search
|
||||
and topic filtering still need full-corpus execution-plan/performance checks
|
||||
after publication; the migration's title/abstract index does not index authors.
|
||||
|
||||
List responses contain `items`, `total`, `topics`, `page`, and `pageSize`.
|
||||
Each paper preserves original title/abstract, authors, source dates, topics and
|
||||
source links. Publication decisions need source evidence; absent decisions stay
|
||||
@@ -45,3 +68,11 @@ Unicode, anonymous reads, rejected mutations, project lookup, errors, saturation
|
||||
and timeout slot release. Run compilation/tests only on the user-confirmed ECS
|
||||
runner. These transport tests do not prove a database connection, full-corpus
|
||||
query performance or production availability.
|
||||
|
||||
The PostgreSQL repository batches source/tag/identifier/submission queries in
|
||||
the same read-only repeatable-read transaction. Author tuples are displayed as
|
||||
given name, family name and suffix; null abstracts/dates remain null. Publication
|
||||
status comes from the latest observed decision, using decision date and ID as
|
||||
tie-breakers, and stays unknown when no decision exists. Unit checks cover
|
||||
author shapes, bound user values and schema qualification; real PostgreSQL
|
||||
integration checks remain pending the migration.
|
||||
|
||||
@@ -0,0 +1,345 @@
|
||||
package research
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// The fixed Paper Atlas migration a3b677b preserves business columns as TEXT
|
||||
// and BIGINT and creates the English title/abstract GIN expression below.
|
||||
// No schema creation, indexing or other writes belong to this repository.
|
||||
type PostgresRepository struct {
|
||||
db *sql.DB
|
||||
prefix string
|
||||
}
|
||||
|
||||
var schemaName = regexp.MustCompile(`^[a-z][a-z0-9_]{0,35}$`)
|
||||
|
||||
func NewPostgresRepository(db *sql.DB, schema string) (*PostgresRepository, error) {
|
||||
if db == nil || !schemaName.MatchString(schema) {
|
||||
return nil, errors.New("invalid research database configuration")
|
||||
}
|
||||
return &PostgresRepository{db: db, prefix: `"` + schema + `".`}, nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) query(value string) string {
|
||||
return strings.ReplaceAll(value, "{schema}", r.prefix)
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) begin(ctx context.Context) (*sql.Tx, error) {
|
||||
tx, err := r.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `SELECT set_config('statement_timeout','8000',true),
|
||||
set_config('lock_timeout','1000',true), set_config('work_mem','2MB',true),
|
||||
set_config('max_parallel_workers_per_gather','0',true)`)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return nil, err
|
||||
}
|
||||
return tx, nil
|
||||
}
|
||||
|
||||
// Check resolves required columns and SELECT permissions without scanning the
|
||||
// corpus. The migration publishes the complete schema atomically.
|
||||
func (r *PostgresRepository) Check(ctx context.Context) error {
|
||||
tx, err := r.begin(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
rows, err := tx.QueryContext(ctx, r.query(`SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published,p.title_key,
|
||||
r.external_id,r.url,r.metadata_json,so.name,i.scheme,i.value,t.namespace,t.value,
|
||||
s.id,e.year,e.track,v.acronym,d.status,d.evidence_url,d.observed_at,d.decision_date
|
||||
FROM {schema}papers p LEFT JOIN {schema}source_records r ON r.paper_id=p.id
|
||||
LEFT JOIN {schema}sources so ON so.id=r.source_id LEFT JOIN {schema}identifiers i ON i.paper_id=p.id
|
||||
LEFT JOIN {schema}paper_tags pt ON pt.paper_id=p.id LEFT JOIN {schema}tags t ON t.id=pt.tag_id
|
||||
LEFT JOIN {schema}submissions s ON s.paper_id=p.id LEFT JOIN {schema}editions e ON e.id=s.edition_id
|
||||
LEFT JOIN {schema}venues v ON v.id=e.venue_id LEFT JOIN {schema}decisions d ON d.submission_id=s.id WHERE false`))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = rows.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
func whereClause(filters Filters) (string, []any) {
|
||||
clauses, values := []string{}, []any{}
|
||||
if filters.Query != "" {
|
||||
values = append(values, filters.Query)
|
||||
if filters.SearchIn == "author" {
|
||||
// JSON stores author strings or [family,given,suffix] tuples. Plain
|
||||
// full-text terms search names regardless of tuple display order.
|
||||
clauses = append(clauses, `to_tsvector('simple'::regconfig,p.authors_json) @@ plainto_tsquery('simple'::regconfig,$1)`)
|
||||
} else {
|
||||
clauses = append(clauses, `to_tsvector('english'::regconfig,coalesce(p.title,'') || ' ' || coalesce(p.abstract,'')) @@ plainto_tsquery('english'::regconfig,$1)`)
|
||||
}
|
||||
}
|
||||
if filters.Topic != "" {
|
||||
values = append(values, filters.Topic)
|
||||
clauses = append(clauses, fmt.Sprintf(`EXISTS(SELECT 1 FROM {schema}paper_tags pt
|
||||
JOIN {schema}tags t ON t.id=pt.tag_id
|
||||
WHERE pt.paper_id=p.id AND t.namespace='arxiv:category' AND t.value=$%d)`, len(values)))
|
||||
}
|
||||
if len(clauses) == 0 {
|
||||
return "", values
|
||||
}
|
||||
return " WHERE " + strings.Join(clauses, " AND "), values
|
||||
}
|
||||
|
||||
type scanner interface{ Scan(...any) error }
|
||||
|
||||
func scanPaper(row scanner) (Paper, error) {
|
||||
p := Paper{Authors: []string{}, Topics: []string{}, Sources: []Source{}}
|
||||
var id int64
|
||||
var abstract, published sql.NullString
|
||||
var authors string
|
||||
if err := row.Scan(&id, &p.Title, &abstract, &authors, &published); err != nil {
|
||||
return p, err
|
||||
}
|
||||
p.ID = strconv.FormatInt(id, 10)
|
||||
if abstract.Valid {
|
||||
p.Abstract = &abstract.String
|
||||
}
|
||||
if published.Valid {
|
||||
p.PublishedAt = &published.String
|
||||
}
|
||||
var err error
|
||||
p.Authors, err = decodeAuthors(authors)
|
||||
return p, err
|
||||
}
|
||||
|
||||
func decodeAuthors(raw string) ([]string, error) {
|
||||
var entries []json.RawMessage
|
||||
if err := json.Unmarshal([]byte(raw), &entries); err != nil {
|
||||
return nil, errors.New("invalid stored authors")
|
||||
}
|
||||
names := []string{}
|
||||
for _, entry := range entries {
|
||||
var name string
|
||||
if json.Unmarshal(entry, &name) == nil {
|
||||
if name = strings.TrimSpace(name); name != "" {
|
||||
names = append(names, name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
var tuple []string
|
||||
if json.Unmarshal(entry, &tuple) != nil || len(tuple) == 0 || len(tuple) > 3 {
|
||||
return nil, errors.New("unsupported stored author")
|
||||
}
|
||||
if len(tuple) >= 2 {
|
||||
tuple[0], tuple[1] = tuple[1], tuple[0]
|
||||
}
|
||||
parts := []string{}
|
||||
for _, part := range tuple {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
}
|
||||
if name = strings.Join(parts, " "); name != "" {
|
||||
names = append(names, name)
|
||||
}
|
||||
}
|
||||
return names, nil
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, error) {
|
||||
result := Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}
|
||||
tx, err := r.begin(ctx)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
where, args := whereClause(filters)
|
||||
if err = tx.QueryRowContext(ctx, r.query(`SELECT count(*) FROM {schema}papers p`+where), args...).Scan(&result.Total); err != nil {
|
||||
return result, err
|
||||
}
|
||||
order := "p.id DESC"
|
||||
if filters.Sort == "title" {
|
||||
order = "p.title_key,p.id"
|
||||
}
|
||||
args = append(args, filters.PageSize, (filters.Page-1)*filters.PageSize)
|
||||
query := fmt.Sprintf(`SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published
|
||||
FROM {schema}papers p%s ORDER BY %s LIMIT $%d OFFSET $%d`, where, order, len(args)-1, len(args))
|
||||
rows, err := tx.QueryContext(ctx, r.query(query), args...)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
for rows.Next() {
|
||||
p, err := scanPaper(rows)
|
||||
if err != nil {
|
||||
_ = rows.Close()
|
||||
return result, err
|
||||
}
|
||||
result.Items = append(result.Items, p)
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
if err = r.enrich(ctx, tx, result.Items); err != nil {
|
||||
return result, err
|
||||
}
|
||||
rows, err = tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`))
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
for rows.Next() {
|
||||
var topic string
|
||||
if err = rows.Scan(&topic); err != nil {
|
||||
_ = rows.Close()
|
||||
return result, err
|
||||
}
|
||||
result.Topics = append(result.Topics, topic)
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, tx.Commit()
|
||||
}
|
||||
|
||||
func (r *PostgresRepository) Get(ctx context.Context, id int64) (Paper, error) {
|
||||
tx, err := r.begin(ctx)
|
||||
if err != nil {
|
||||
return Paper{}, err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
p, err := scanPaper(tx.QueryRowContext(ctx, r.query(`SELECT id,title,abstract,authors_json,first_published FROM {schema}papers WHERE id=$1`), id))
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
papers := []Paper{p}
|
||||
if err = r.enrich(ctx, tx, papers); err != nil {
|
||||
return p, err
|
||||
}
|
||||
return papers[0], tx.Commit()
|
||||
}
|
||||
|
||||
// Batch metadata queries avoid per-paper round trips and share the count/page
|
||||
// snapshot. User values are bound; only the validated schema is interpolated.
|
||||
func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Paper) error {
|
||||
if len(papers) == 0 {
|
||||
return nil
|
||||
}
|
||||
ids := make([]int64, len(papers))
|
||||
byID := map[int64]*Paper{}
|
||||
for i := range papers {
|
||||
id, err := strconv.ParseInt(papers[i].ID, 10, 64)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ids[i] = id
|
||||
byID[id] = &papers[i]
|
||||
}
|
||||
rows, err := tx.QueryContext(ctx, r.query(`SELECT DISTINCT pt.paper_id,t.value
|
||||
FROM {schema}paper_tags pt JOIN {schema}tags t ON t.id=pt.tag_id
|
||||
WHERE pt.paper_id=ANY($1) AND t.namespace='arxiv:category' ORDER BY pt.paper_id,t.value`), ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var topic string
|
||||
if err = rows.Scan(&id, &topic); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
byID[id].Topics = append(byID[id].Topics, topic)
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err = tx.QueryContext(ctx, r.query(`SELECT r.paper_id,s.name,r.external_id,r.url,r.metadata_json
|
||||
FROM {schema}source_records r JOIN {schema}sources s ON s.id=r.source_id
|
||||
WHERE r.paper_id=ANY($1) ORDER BY r.paper_id,r.id`), ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var source Source
|
||||
var metadata string
|
||||
if err = rows.Scan(&id, &source.Label, &source.ExternalID, &source.URL, &metadata); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
p := byID[id]
|
||||
switch source.Label {
|
||||
case "arxiv":
|
||||
source.Label = "arXiv"
|
||||
case "openreview":
|
||||
source.Label = "OpenReview"
|
||||
case "pmlr":
|
||||
source.Label = "PMLR"
|
||||
}
|
||||
p.Sources = append(p.Sources, source)
|
||||
var fields map[string]json.RawMessage
|
||||
if json.Unmarshal([]byte(metadata), &fields) == nil {
|
||||
for key, target := range map[string]**string{"doi": &p.DOI, "journal-ref": &p.Journal, "comments": &p.Comments} {
|
||||
var value string
|
||||
if *target == nil && json.Unmarshal(fields[key], &value) == nil && strings.TrimSpace(value) != "" {
|
||||
*target = &value
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err = tx.QueryContext(ctx, r.query(`SELECT paper_id,value FROM {schema}identifiers WHERE paper_id=ANY($1) AND scheme='doi' ORDER BY paper_id,value`), ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var value string
|
||||
if err = rows.Scan(&id, &value); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
if byID[id].DOI == nil && strings.TrimSpace(value) != "" {
|
||||
byID[id].DOI = &value
|
||||
}
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
rows, err = tx.QueryContext(ctx, r.query(`SELECT s.paper_id,v.acronym,e.year,e.track,coalesce(d.status,'unknown'),coalesce(d.evidence_url,'')
|
||||
FROM {schema}submissions s JOIN {schema}editions e ON e.id=s.edition_id JOIN {schema}venues v ON v.id=e.venue_id
|
||||
LEFT JOIN LATERAL (SELECT status,evidence_url FROM {schema}decisions WHERE submission_id=s.id
|
||||
ORDER BY observed_at DESC,coalesce(decision_date,observed_at) DESC,id DESC LIMIT 1) d ON true
|
||||
WHERE s.paper_id=ANY($1) ORDER BY s.paper_id,e.year DESC,v.acronym,e.track,s.id`), ids)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
var publication Publication
|
||||
if err = rows.Scan(&id, &publication.Venue, &publication.Year, &publication.Track, &publication.Status, &publication.EvidenceURL); err != nil {
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
byID[id].Publications = append(byID[id].Publications, publication)
|
||||
}
|
||||
err = rows.Err()
|
||||
_ = rows.Close()
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package research
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestStoredAuthorShapes(t *testing.T) {
|
||||
for _, test := range []struct {
|
||||
raw string
|
||||
want []string
|
||||
}{
|
||||
{`[]`, []string{}},
|
||||
{`["The BABAR Collaboration","B. Aubert"]`, []string{"The BABAR Collaboration", "B. Aubert"}},
|
||||
{`[["Balázs","C.",""],["Berger","E. L.",""],["Smith","Ada","Jr."]]`, []string{"C. Balázs", "E. L. Berger", "Ada Smith Jr."}},
|
||||
{`[" 张三 ",["Only family"],["","Only given",""],""]`, []string{"张三", "Only family", "Only given"}},
|
||||
} {
|
||||
got, err := decodeAuthors(test.raw)
|
||||
if err != nil || !reflect.DeepEqual(got, test.want) {
|
||||
t.Fatalf("%s: %#v, %v", test.raw, got, err)
|
||||
}
|
||||
encoded, _ := json.Marshal(got)
|
||||
if string(encoded) == "null" {
|
||||
t.Fatal("missing authors must remain an array")
|
||||
}
|
||||
}
|
||||
for _, raw := range []string{`broken`, `{"unexpected":"object"}`, `[42]`, `[["a","b","c","d"]]`} {
|
||||
if _, err := decodeAuthors(raw); err == nil {
|
||||
t.Fatalf("accepted unsupported authors %s", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryValuesRemainBound(t *testing.T) {
|
||||
query := `graph'); DROP TABLE papers; -- OR author:*`
|
||||
where, args := whereClause(Filters{Query: query, Topic: "cs.LG", SearchIn: "text"})
|
||||
if strings.Contains(where, query) || !reflect.DeepEqual(args, []any{query, "cs.LG"}) {
|
||||
t.Fatalf("unbound user input: %s %#v", where, args)
|
||||
}
|
||||
if !strings.Contains(where, "plainto_tsquery('english'::regconfig,$1)") || !strings.Contains(where, "t.value=$2") {
|
||||
t.Fatalf("unexpected search contract: %s", where)
|
||||
}
|
||||
where, args = whereClause(Filters{Query: "Ada Smith", SearchIn: "author"})
|
||||
if !strings.Contains(where, "p.authors_json") || strings.Contains(where, "p.abstract") || !reflect.DeepEqual(args, []any{"Ada Smith"}) {
|
||||
t.Fatalf("author search includes unrelated text: %s", where)
|
||||
}
|
||||
where, args = whereClause(Filters{Topic: "cs.AI"})
|
||||
if !strings.Contains(where, "t.value=$1") || !reflect.DeepEqual(args, []any{"cs.AI"}) {
|
||||
t.Fatalf("topic-only binding: %s %#v", where, args)
|
||||
}
|
||||
where, args = whereClause(Filters{})
|
||||
if where != "" || len(args) != 0 {
|
||||
t.Fatal("empty filters changed the corpus")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSchemaCannotEscapeQualification(t *testing.T) {
|
||||
for _, name := range []string{"", `research";DROP SCHEMA public;--`, "a.b", "Public", strings.Repeat("a", 37)} {
|
||||
if _, err := NewPostgresRepository(&sql.DB{}, name); err == nil {
|
||||
t.Fatalf("accepted schema %q", name)
|
||||
}
|
||||
}
|
||||
if _, err := NewPostgresRepository(nil, "research"); err == nil {
|
||||
t.Fatal("accepted nil database")
|
||||
}
|
||||
r, err := NewPostgresRepository(&sql.DB{}, "research_v2")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := r.query(`SELECT p.id FROM {schema}papers p JOIN {schema}source_records s ON s.paper_id=p.id`)
|
||||
if got != `SELECT p.id FROM "research_v2".papers p JOIN "research_v2".source_records s ON s.paper_id=p.id` {
|
||||
t.Fatalf("unqualified corpus tables: %s", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user