Prepare read-only PostgreSQL paper queries and optional Core wiring
This commit is contained in:
@@ -16,3 +16,10 @@ SMTP_FROM=noreply@notice.entroia.com
|
|||||||
# Optional Sign in with Slack.
|
# Optional Sign in with Slack.
|
||||||
SLACK_CLIENT_ID=replace-me
|
SLACK_CLIENT_ID=replace-me
|
||||||
SLACK_CLIENT_SECRET=replace-me
|
SLACK_CLIENT_SECRET=replace-me
|
||||||
|
|
||||||
|
# Optional shared paper corpus; enable only after PostgreSQL publication.
|
||||||
|
# Store the real read-only connection string in the protected server research.env.
|
||||||
|
FOUNDRY_RESEARCH_DATABASE_URL=
|
||||||
|
FOUNDRY_RESEARCH_SCHEMA=research
|
||||||
|
# Deployment migration container; independent of database/schema names.
|
||||||
|
FOUNDRY_POSTGRES_CONTAINER=foundry-postgres
|
||||||
|
|||||||
@@ -74,6 +74,7 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
PUBLIC_ORIGIN: https://api.example.com
|
PUBLIC_ORIGIN: https://api.example.com
|
||||||
DATABASE_URL: postgres://foundry:fixture@127.0.0.1:5432/foundry
|
DATABASE_URL: postgres://foundry:fixture@127.0.0.1:5432/foundry
|
||||||
|
FOUNDRY_RESEARCH_DATABASE_URL: ''
|
||||||
GOOGLE_CLIENT_ID: fixture
|
GOOGLE_CLIENT_ID: fixture
|
||||||
GOOGLE_CLIENT_SECRET: fixture
|
GOOGLE_CLIENT_SECRET: fixture
|
||||||
FOUNDRY_PROJECTS: '{"notes":"http://localhost:3000/auth/callback"}'
|
FOUNDRY_PROJECTS: '{"notes":"http://localhost:3000/auth/callback"}'
|
||||||
|
|||||||
@@ -10,6 +10,10 @@
|
|||||||
4. 由 Gitea Actions 在 `ces-control` 上使用 Go 1.25 构建并检查。部署工作流仅处理 `main` 上影响后端或部署的改动,将二进制启动在服务器 Docker 私网,由 Traefik 提供 HTTPS。通过 `GET /healthz` 检查进程,通过 `GET /readyz` 检查认证库。
|
4. 由 Gitea Actions 在 `ces-control` 上使用 Go 1.25 构建并检查。部署工作流仅处理 `main` 上影响后端或部署的改动,将二进制启动在服务器 Docker 私网,由 Traefik 提供 HTTPS。通过 `GET /healthz` 检查进程,通过 `GET /readyz` 检查认证库。
|
||||||
5. 代理避免记录 Google 回调的查询参数及认证请求头。设置入口限流,并定时删除三张临时表(`oauth_attempts`、`login_grants`、`project_sessions`)中 `expires_at <= now()` 的记录。
|
5. 代理避免记录 Google 回调的查询参数及认证请求头。设置入口限流,并定时删除三张临时表(`oauth_attempts`、`login_grants`、`project_sessions`)中 `expires_at <= now()` 的记录。
|
||||||
|
|
||||||
|
## 共享论文库(Thesisly)
|
||||||
|
|
||||||
|
可选只读接入见[共享论文 API](docs/research-api.md)。Thesisly 使用同一后端的 `/projects/{project}/research/papers` 列表、搜索和详情;只有配置服务器上的 `FOUNDRY_RESEARCH_DATABASE_URL` 时才启用。连接串保存在受保护的 `research.env`,不放入前端或 Actions Variables。当前仍等待 Paper Atlas 全量 PostgreSQL 迁移完成及真实数据验收,尚未启用线上论文接口。
|
||||||
|
|
||||||
## Actions 部署配置
|
## Actions 部署配置
|
||||||
|
|
||||||
在仓库的 Actions Secrets 配置 `GOOGLE_CLIENT_ID` 和 `GOOGLE_CLIENT_SECRET`,在 Actions Variables 配置 `FOUNDRY_PROJECTS`。变量值为 JSON 对象,例如 `{"prompt":"http://localhost:17321/auth/callback"}`。部署读取服务器上预先配置的 `/opt/endoleap-infra/foundry/database.env`,OAuth 值由 Actions 在启动容器时注入。
|
在仓库的 Actions Secrets 配置 `GOOGLE_CLIENT_ID` 和 `GOOGLE_CLIENT_SECRET`,在 Actions Variables 配置 `FOUNDRY_PROJECTS`。变量值为 JSON 对象,例如 `{"prompt":"http://localhost:17321/auth/callback"}`。部署读取服务器上预先配置的 `/opt/endoleap-infra/foundry/database.env`,OAuth 值由 Actions 在启动容器时注入。
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ import (
|
|||||||
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/auth"
|
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/auth"
|
||||||
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/codex"
|
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/codex"
|
||||||
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/platform"
|
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/platform"
|
||||||
|
"gitea.entroia.com/Zayden-Jung/foundry-core/internal/research"
|
||||||
"github.com/jackc/pgx/v5"
|
"github.com/jackc/pgx/v5"
|
||||||
"github.com/jackc/pgx/v5/stdlib"
|
"github.com/jackc/pgx/v5/stdlib"
|
||||||
)
|
)
|
||||||
@@ -61,6 +62,33 @@ func main() {
|
|||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
mux := http.NewServeMux()
|
mux := http.NewServeMux()
|
||||||
|
var papers research.Repository
|
||||||
|
if researchDSN := os.Getenv("FOUNDRY_RESEARCH_DATABASE_URL"); researchDSN != "" {
|
||||||
|
researchConfig, err := pgx.ParseConfig(researchDSN)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Invalid research database configuration")
|
||||||
|
}
|
||||||
|
researchConfig.RuntimeParams["default_transaction_read_only"] = "on"
|
||||||
|
researchConfig.RuntimeParams["application_name"] = "foundry-research"
|
||||||
|
researchDB := stdlib.OpenDB(*researchConfig)
|
||||||
|
defer researchDB.Close()
|
||||||
|
researchDB.SetMaxOpenConns(2)
|
||||||
|
researchDB.SetMaxIdleConns(2)
|
||||||
|
researchDB.SetConnMaxIdleTime(5 * time.Minute)
|
||||||
|
researchDB.SetConnMaxLifetime(30 * time.Minute)
|
||||||
|
repository, err := research.NewPostgresRepository(researchDB, env("FOUNDRY_RESEARCH_SCHEMA", "research"))
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Invalid research database configuration")
|
||||||
|
}
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
err = repository.Check(ctx)
|
||||||
|
cancel()
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("Research database is unavailable or not published")
|
||||||
|
}
|
||||||
|
papers = repository
|
||||||
|
}
|
||||||
|
research.Register(mux, papers, callbacks)
|
||||||
if err := codex.Register(mux, callbacks, os.Getenv("FOUNDRY_CODEX_SOCKET"), os.Getenv("FOUNDRY_CODEX_GRANTS")); err != nil {
|
if err := codex.Register(mux, callbacks, os.Getenv("FOUNDRY_CODEX_SOCKET"), os.Getenv("FOUNDRY_CODEX_GRANTS")); err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-1
@@ -2,12 +2,13 @@
|
|||||||
"""Apply checked migrations atomically from the trusted deployment runner."""
|
"""Apply checked migrations atomically from the trusted deployment runner."""
|
||||||
import argparse
|
import argparse
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
|
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--container", default="endoleap-dev-postgres-1")
|
parser.add_argument("--container", default=os.environ.get("FOUNDRY_POSTGRES_CONTAINER", "foundry-postgres"))
|
||||||
parser.add_argument("--database", default="foundry_auth")
|
parser.add_argument("--database", default="foundry_auth")
|
||||||
parser.add_argument("--role", default="foundry")
|
parser.add_argument("--role", default="foundry")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|||||||
+5
-1
@@ -53,6 +53,10 @@ mail_env=()
|
|||||||
if [[ -s "$foundry/mail.env" ]]; then
|
if [[ -s "$foundry/mail.env" ]]; then
|
||||||
mail_env=(--env-file "$foundry/mail.env")
|
mail_env=(--env-file "$foundry/mail.env")
|
||||||
fi
|
fi
|
||||||
|
research_env=()
|
||||||
|
if [[ -s "$foundry/research.env" ]]; then
|
||||||
|
research_env=(--env-file "$foundry/research.env")
|
||||||
|
fi
|
||||||
slack_env=()
|
slack_env=()
|
||||||
if [[ -n "${SLACK_CLIENT_ID:-}" || -n "${SLACK_CLIENT_SECRET:-}" ]]; then
|
if [[ -n "${SLACK_CLIENT_ID:-}" || -n "${SLACK_CLIENT_SECRET:-}" ]]; then
|
||||||
for name in SLACK_CLIENT_ID SLACK_CLIENT_SECRET; do
|
for name in SLACK_CLIENT_ID SLACK_CLIENT_SECRET; do
|
||||||
@@ -118,7 +122,7 @@ if [[ -d "$codex_dir" ]]; then
|
|||||||
)
|
)
|
||||||
fi
|
fi
|
||||||
|
|
||||||
docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack_env[@]}" --network "$network" \
|
docker run -d --name "$candidate" "${codex_mounts[@]}" "${mail_env[@]}" "${slack_env[@]}" "${research_env[@]}" --network "$network" \
|
||||||
--env-file "$foundry/database.env" -e HOST -e PORT -e PUBLIC_ORIGIN \
|
--env-file "$foundry/database.env" -e HOST -e PORT -e PUBLIC_ORIGIN \
|
||||||
-e GOOGLE_CLIENT_ID -e GOOGLE_CLIENT_SECRET -e FOUNDRY_PROJECTS -e FOUNDRY_ONBOARDING \
|
-e GOOGLE_CLIENT_ID -e GOOGLE_CLIENT_SECRET -e FOUNDRY_PROJECTS -e FOUNDRY_ONBOARDING \
|
||||||
--user 1000:1000 --cap-drop ALL --security-opt no-new-privileges \
|
--user 1000:1000 --cap-drop ALL --security-opt no-new-privileges \
|
||||||
|
|||||||
+35
-4
@@ -6,12 +6,29 @@ collections remain project scoped.
|
|||||||
|
|
||||||
## Current implementation boundary
|
## Current implementation boundary
|
||||||
|
|
||||||
`internal/research` implements the HTTP contract and repository boundary. It is
|
`internal/research` implements the HTTP contract and a PostgreSQL repository
|
||||||
not registered by `cmd/foundry` yet and does not open any database connection.
|
based on the fixed Paper Atlas `a3b677b` schema (TEXT/BIGINT columns, native
|
||||||
|
English title/abstract search index). `cmd/foundry` enables it only when
|
||||||
|
`FOUNDRY_RESEARCH_DATABASE_URL` is configured. No live research connection has
|
||||||
|
been configured during the ongoing migration.
|
||||||
The existing SQLite implementation was withdrawn. Wait for the separate Paper
|
The existing SQLite implementation was withdrawn. Wait for the separate Paper
|
||||||
Atlas PostgreSQL migration to finish, inspect the published schema and indexes,
|
Atlas PostgreSQL migration to finish, inspect the published schema and indexes,
|
||||||
then implement the repository with a least-privilege read-only connection and
|
then verify the queries, indexes and full-corpus performance with a
|
||||||
register it in the existing server. Do not add `FOUNDRY_RESEARCH_DATABASE`.
|
least-privilege read-only connection before enabling it in the existing server.
|
||||||
|
Do not add `FOUNDRY_RESEARCH_DATABASE`.
|
||||||
|
|
||||||
|
The server-side deployment may load a protected
|
||||||
|
`/opt/endoleap-infra/foundry/research.env` containing
|
||||||
|
`FOUNDRY_RESEARCH_DATABASE_URL` and `FOUNDRY_RESEARCH_SCHEMA` (default `research`).
|
||||||
|
Keep the connection string out of Git, Actions variables, browser configuration
|
||||||
|
and command arguments. Use a dedicated role with schema USAGE and table SELECT
|
||||||
|
only. Core uses a separate two-connection pool, read-only connection defaults and
|
||||||
|
read-only transactions. Startup checks the published schema/SELECT permissions;
|
||||||
|
an explicitly enabled but invalid connection fails candidate readiness.
|
||||||
|
|
||||||
|
Deployment migration defaults now follow the shared `foundry-postgres`
|
||||||
|
container name. `FOUNDRY_POSTGRES_CONTAINER` or the existing `--container` option
|
||||||
|
can select a different container without changing database/table names.
|
||||||
|
|
||||||
## Routes
|
## Routes
|
||||||
|
|
||||||
@@ -28,6 +45,12 @@ or repeated parameters return 400. `recent` means descending stable paper ID,
|
|||||||
not publication date. Search terms must become bound query parameters, never
|
not publication date. Search terms must become bound query parameters, never
|
||||||
raw SQL or user-supplied full-text syntax.
|
raw SQL or user-supplied full-text syntax.
|
||||||
|
|
||||||
|
Title/abstract search uses the migration's English vector with
|
||||||
|
`plainto_tsquery`; author search uses simple-language terms over the stored
|
||||||
|
author JSON. Both treat input as terms, never full-text operators. Author search
|
||||||
|
and topic filtering still need full-corpus execution-plan/performance checks
|
||||||
|
after publication; the migration's title/abstract index does not index authors.
|
||||||
|
|
||||||
List responses contain `items`, `total`, `topics`, `page`, and `pageSize`.
|
List responses contain `items`, `total`, `topics`, `page`, and `pageSize`.
|
||||||
Each paper preserves original title/abstract, authors, source dates, topics and
|
Each paper preserves original title/abstract, authors, source dates, topics and
|
||||||
source links. Publication decisions need source evidence; absent decisions stay
|
source links. Publication decisions need source evidence; absent decisions stay
|
||||||
@@ -45,3 +68,11 @@ Unicode, anonymous reads, rejected mutations, project lookup, errors, saturation
|
|||||||
and timeout slot release. Run compilation/tests only on the user-confirmed ECS
|
and timeout slot release. Run compilation/tests only on the user-confirmed ECS
|
||||||
runner. These transport tests do not prove a database connection, full-corpus
|
runner. These transport tests do not prove a database connection, full-corpus
|
||||||
query performance or production availability.
|
query performance or production availability.
|
||||||
|
|
||||||
|
The PostgreSQL repository batches source/tag/identifier/submission queries in
|
||||||
|
the same read-only repeatable-read transaction. Author tuples are displayed as
|
||||||
|
given name, family name and suffix; null abstracts/dates remain null. Publication
|
||||||
|
status comes from the latest observed decision, using decision date and ID as
|
||||||
|
tie-breakers, and stays unknown when no decision exists. Unit checks cover
|
||||||
|
author shapes, bound user values and schema qualification; real PostgreSQL
|
||||||
|
integration checks remain pending the migration.
|
||||||
|
|||||||
@@ -0,0 +1,345 @@
|
|||||||
|
package research
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// The fixed Paper Atlas migration a3b677b preserves business columns as TEXT
|
||||||
|
// and BIGINT and creates the English title/abstract GIN expression below.
|
||||||
|
// No schema creation, indexing or other writes belong to this repository.
|
||||||
|
type PostgresRepository struct {
|
||||||
|
db *sql.DB
|
||||||
|
prefix string
|
||||||
|
}
|
||||||
|
|
||||||
|
var schemaName = regexp.MustCompile(`^[a-z][a-z0-9_]{0,35}$`)
|
||||||
|
|
||||||
|
func NewPostgresRepository(db *sql.DB, schema string) (*PostgresRepository, error) {
|
||||||
|
if db == nil || !schemaName.MatchString(schema) {
|
||||||
|
return nil, errors.New("invalid research database configuration")
|
||||||
|
}
|
||||||
|
return &PostgresRepository{db: db, prefix: `"` + schema + `".`}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *PostgresRepository) query(value string) string {
|
||||||
|
return strings.ReplaceAll(value, "{schema}", r.prefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *PostgresRepository) begin(ctx context.Context) (*sql.Tx, error) {
|
||||||
|
tx, err := r.db.BeginTx(ctx, &sql.TxOptions{Isolation: sql.LevelRepeatableRead, ReadOnly: true})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
_, err = tx.ExecContext(ctx, `SELECT set_config('statement_timeout','8000',true),
|
||||||
|
set_config('lock_timeout','1000',true), set_config('work_mem','2MB',true),
|
||||||
|
set_config('max_parallel_workers_per_gather','0',true)`)
|
||||||
|
if err != nil {
|
||||||
|
_ = tx.Rollback()
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return tx, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Check resolves required columns and SELECT permissions without scanning the
|
||||||
|
// corpus. The migration publishes the complete schema atomically.
|
||||||
|
func (r *PostgresRepository) Check(ctx context.Context) error {
|
||||||
|
tx, err := r.begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
rows, err := tx.QueryContext(ctx, r.query(`SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published,p.title_key,
|
||||||
|
r.external_id,r.url,r.metadata_json,so.name,i.scheme,i.value,t.namespace,t.value,
|
||||||
|
s.id,e.year,e.track,v.acronym,d.status,d.evidence_url,d.observed_at,d.decision_date
|
||||||
|
FROM {schema}papers p LEFT JOIN {schema}source_records r ON r.paper_id=p.id
|
||||||
|
LEFT JOIN {schema}sources so ON so.id=r.source_id LEFT JOIN {schema}identifiers i ON i.paper_id=p.id
|
||||||
|
LEFT JOIN {schema}paper_tags pt ON pt.paper_id=p.id LEFT JOIN {schema}tags t ON t.id=pt.tag_id
|
||||||
|
LEFT JOIN {schema}submissions s ON s.paper_id=p.id LEFT JOIN {schema}editions e ON e.id=s.edition_id
|
||||||
|
LEFT JOIN {schema}venues v ON v.id=e.venue_id LEFT JOIN {schema}decisions d ON d.submission_id=s.id WHERE false`))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err = rows.Close(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
func whereClause(filters Filters) (string, []any) {
|
||||||
|
clauses, values := []string{}, []any{}
|
||||||
|
if filters.Query != "" {
|
||||||
|
values = append(values, filters.Query)
|
||||||
|
if filters.SearchIn == "author" {
|
||||||
|
// JSON stores author strings or [family,given,suffix] tuples. Plain
|
||||||
|
// full-text terms search names regardless of tuple display order.
|
||||||
|
clauses = append(clauses, `to_tsvector('simple'::regconfig,p.authors_json) @@ plainto_tsquery('simple'::regconfig,$1)`)
|
||||||
|
} else {
|
||||||
|
clauses = append(clauses, `to_tsvector('english'::regconfig,coalesce(p.title,'') || ' ' || coalesce(p.abstract,'')) @@ plainto_tsquery('english'::regconfig,$1)`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if filters.Topic != "" {
|
||||||
|
values = append(values, filters.Topic)
|
||||||
|
clauses = append(clauses, fmt.Sprintf(`EXISTS(SELECT 1 FROM {schema}paper_tags pt
|
||||||
|
JOIN {schema}tags t ON t.id=pt.tag_id
|
||||||
|
WHERE pt.paper_id=p.id AND t.namespace='arxiv:category' AND t.value=$%d)`, len(values)))
|
||||||
|
}
|
||||||
|
if len(clauses) == 0 {
|
||||||
|
return "", values
|
||||||
|
}
|
||||||
|
return " WHERE " + strings.Join(clauses, " AND "), values
|
||||||
|
}
|
||||||
|
|
||||||
|
type scanner interface{ Scan(...any) error }
|
||||||
|
|
||||||
|
func scanPaper(row scanner) (Paper, error) {
|
||||||
|
p := Paper{Authors: []string{}, Topics: []string{}, Sources: []Source{}}
|
||||||
|
var id int64
|
||||||
|
var abstract, published sql.NullString
|
||||||
|
var authors string
|
||||||
|
if err := row.Scan(&id, &p.Title, &abstract, &authors, &published); err != nil {
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
p.ID = strconv.FormatInt(id, 10)
|
||||||
|
if abstract.Valid {
|
||||||
|
p.Abstract = &abstract.String
|
||||||
|
}
|
||||||
|
if published.Valid {
|
||||||
|
p.PublishedAt = &published.String
|
||||||
|
}
|
||||||
|
var err error
|
||||||
|
p.Authors, err = decodeAuthors(authors)
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func decodeAuthors(raw string) ([]string, error) {
|
||||||
|
var entries []json.RawMessage
|
||||||
|
if err := json.Unmarshal([]byte(raw), &entries); err != nil {
|
||||||
|
return nil, errors.New("invalid stored authors")
|
||||||
|
}
|
||||||
|
names := []string{}
|
||||||
|
for _, entry := range entries {
|
||||||
|
var name string
|
||||||
|
if json.Unmarshal(entry, &name) == nil {
|
||||||
|
if name = strings.TrimSpace(name); name != "" {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var tuple []string
|
||||||
|
if json.Unmarshal(entry, &tuple) != nil || len(tuple) == 0 || len(tuple) > 3 {
|
||||||
|
return nil, errors.New("unsupported stored author")
|
||||||
|
}
|
||||||
|
if len(tuple) >= 2 {
|
||||||
|
tuple[0], tuple[1] = tuple[1], tuple[0]
|
||||||
|
}
|
||||||
|
parts := []string{}
|
||||||
|
for _, part := range tuple {
|
||||||
|
if part = strings.TrimSpace(part); part != "" {
|
||||||
|
parts = append(parts, part)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if name = strings.Join(parts, " "); name != "" {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *PostgresRepository) List(ctx context.Context, filters Filters) (Page, error) {
|
||||||
|
result := Page{Items: []Paper{}, Topics: []string{}, Page: filters.Page, PageSize: filters.PageSize}
|
||||||
|
tx, err := r.begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
where, args := whereClause(filters)
|
||||||
|
if err = tx.QueryRowContext(ctx, r.query(`SELECT count(*) FROM {schema}papers p`+where), args...).Scan(&result.Total); err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
order := "p.id DESC"
|
||||||
|
if filters.Sort == "title" {
|
||||||
|
order = "p.title_key,p.id"
|
||||||
|
}
|
||||||
|
args = append(args, filters.PageSize, (filters.Page-1)*filters.PageSize)
|
||||||
|
query := fmt.Sprintf(`SELECT p.id,p.title,p.abstract,p.authors_json,p.first_published
|
||||||
|
FROM {schema}papers p%s ORDER BY %s LIMIT $%d OFFSET $%d`, where, order, len(args)-1, len(args))
|
||||||
|
rows, err := tx.QueryContext(ctx, r.query(query), args...)
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
p, err := scanPaper(rows)
|
||||||
|
if err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.Items = append(result.Items, p)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
_ = rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
if err = r.enrich(ctx, tx, result.Items); err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
rows, err = tx.QueryContext(ctx, r.query(`SELECT value FROM {schema}tags WHERE namespace='arxiv:category' ORDER BY value`))
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var topic string
|
||||||
|
if err = rows.Scan(&topic); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
result.Topics = append(result.Topics, topic)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
_ = rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return result, err
|
||||||
|
}
|
||||||
|
return result, tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *PostgresRepository) Get(ctx context.Context, id int64) (Paper, error) {
|
||||||
|
tx, err := r.begin(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return Paper{}, err
|
||||||
|
}
|
||||||
|
defer tx.Rollback()
|
||||||
|
p, err := scanPaper(tx.QueryRowContext(ctx, r.query(`SELECT id,title,abstract,authors_json,first_published FROM {schema}papers WHERE id=$1`), id))
|
||||||
|
if err != nil {
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
papers := []Paper{p}
|
||||||
|
if err = r.enrich(ctx, tx, papers); err != nil {
|
||||||
|
return p, err
|
||||||
|
}
|
||||||
|
return papers[0], tx.Commit()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch metadata queries avoid per-paper round trips and share the count/page
|
||||||
|
// snapshot. User values are bound; only the validated schema is interpolated.
|
||||||
|
func (r *PostgresRepository) enrich(ctx context.Context, tx *sql.Tx, papers []Paper) error {
|
||||||
|
if len(papers) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
ids := make([]int64, len(papers))
|
||||||
|
byID := map[int64]*Paper{}
|
||||||
|
for i := range papers {
|
||||||
|
id, err := strconv.ParseInt(papers[i].ID, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
ids[i] = id
|
||||||
|
byID[id] = &papers[i]
|
||||||
|
}
|
||||||
|
rows, err := tx.QueryContext(ctx, r.query(`SELECT DISTINCT pt.paper_id,t.value
|
||||||
|
FROM {schema}paper_tags pt JOIN {schema}tags t ON t.id=pt.tag_id
|
||||||
|
WHERE pt.paper_id=ANY($1) AND t.namespace='arxiv:category' ORDER BY pt.paper_id,t.value`), ids)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
var topic string
|
||||||
|
if err = rows.Scan(&id, &topic); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
byID[id].Topics = append(byID[id].Topics, topic)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
_ = rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows, err = tx.QueryContext(ctx, r.query(`SELECT r.paper_id,s.name,r.external_id,r.url,r.metadata_json
|
||||||
|
FROM {schema}source_records r JOIN {schema}sources s ON s.id=r.source_id
|
||||||
|
WHERE r.paper_id=ANY($1) ORDER BY r.paper_id,r.id`), ids)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
var source Source
|
||||||
|
var metadata string
|
||||||
|
if err = rows.Scan(&id, &source.Label, &source.ExternalID, &source.URL, &metadata); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
p := byID[id]
|
||||||
|
switch source.Label {
|
||||||
|
case "arxiv":
|
||||||
|
source.Label = "arXiv"
|
||||||
|
case "openreview":
|
||||||
|
source.Label = "OpenReview"
|
||||||
|
case "pmlr":
|
||||||
|
source.Label = "PMLR"
|
||||||
|
}
|
||||||
|
p.Sources = append(p.Sources, source)
|
||||||
|
var fields map[string]json.RawMessage
|
||||||
|
if json.Unmarshal([]byte(metadata), &fields) == nil {
|
||||||
|
for key, target := range map[string]**string{"doi": &p.DOI, "journal-ref": &p.Journal, "comments": &p.Comments} {
|
||||||
|
var value string
|
||||||
|
if *target == nil && json.Unmarshal(fields[key], &value) == nil && strings.TrimSpace(value) != "" {
|
||||||
|
*target = &value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
_ = rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows, err = tx.QueryContext(ctx, r.query(`SELECT paper_id,value FROM {schema}identifiers WHERE paper_id=ANY($1) AND scheme='doi' ORDER BY paper_id,value`), ids)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
var value string
|
||||||
|
if err = rows.Scan(&id, &value); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if byID[id].DOI == nil && strings.TrimSpace(value) != "" {
|
||||||
|
byID[id].DOI = &value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
_ = rows.Close()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
rows, err = tx.QueryContext(ctx, r.query(`SELECT s.paper_id,v.acronym,e.year,e.track,coalesce(d.status,'unknown'),coalesce(d.evidence_url,'')
|
||||||
|
FROM {schema}submissions s JOIN {schema}editions e ON e.id=s.edition_id JOIN {schema}venues v ON v.id=e.venue_id
|
||||||
|
LEFT JOIN LATERAL (SELECT status,evidence_url FROM {schema}decisions WHERE submission_id=s.id
|
||||||
|
ORDER BY observed_at DESC,coalesce(decision_date,observed_at) DESC,id DESC LIMIT 1) d ON true
|
||||||
|
WHERE s.paper_id=ANY($1) ORDER BY s.paper_id,e.year DESC,v.acronym,e.track,s.id`), ids)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
var publication Publication
|
||||||
|
if err = rows.Scan(&id, &publication.Venue, &publication.Year, &publication.Track, &publication.Status, &publication.EvidenceURL); err != nil {
|
||||||
|
_ = rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
byID[id].Publications = append(byID[id].Publications, publication)
|
||||||
|
}
|
||||||
|
err = rows.Err()
|
||||||
|
_ = rows.Close()
|
||||||
|
return err
|
||||||
|
}
|
||||||
@@ -0,0 +1,77 @@
|
|||||||
|
package research
|
||||||
|
|
||||||
|
import (
|
||||||
|
"database/sql"
|
||||||
|
"encoding/json"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestStoredAuthorShapes(t *testing.T) {
|
||||||
|
for _, test := range []struct {
|
||||||
|
raw string
|
||||||
|
want []string
|
||||||
|
}{
|
||||||
|
{`[]`, []string{}},
|
||||||
|
{`["The BABAR Collaboration","B. Aubert"]`, []string{"The BABAR Collaboration", "B. Aubert"}},
|
||||||
|
{`[["Balázs","C.",""],["Berger","E. L.",""],["Smith","Ada","Jr."]]`, []string{"C. Balázs", "E. L. Berger", "Ada Smith Jr."}},
|
||||||
|
{`[" 张三 ",["Only family"],["","Only given",""],""]`, []string{"张三", "Only family", "Only given"}},
|
||||||
|
} {
|
||||||
|
got, err := decodeAuthors(test.raw)
|
||||||
|
if err != nil || !reflect.DeepEqual(got, test.want) {
|
||||||
|
t.Fatalf("%s: %#v, %v", test.raw, got, err)
|
||||||
|
}
|
||||||
|
encoded, _ := json.Marshal(got)
|
||||||
|
if string(encoded) == "null" {
|
||||||
|
t.Fatal("missing authors must remain an array")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, raw := range []string{`broken`, `{"unexpected":"object"}`, `[42]`, `[["a","b","c","d"]]`} {
|
||||||
|
if _, err := decodeAuthors(raw); err == nil {
|
||||||
|
t.Fatalf("accepted unsupported authors %s", raw)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestQueryValuesRemainBound(t *testing.T) {
|
||||||
|
query := `graph'); DROP TABLE papers; -- OR author:*`
|
||||||
|
where, args := whereClause(Filters{Query: query, Topic: "cs.LG", SearchIn: "text"})
|
||||||
|
if strings.Contains(where, query) || !reflect.DeepEqual(args, []any{query, "cs.LG"}) {
|
||||||
|
t.Fatalf("unbound user input: %s %#v", where, args)
|
||||||
|
}
|
||||||
|
if !strings.Contains(where, "plainto_tsquery('english'::regconfig,$1)") || !strings.Contains(where, "t.value=$2") {
|
||||||
|
t.Fatalf("unexpected search contract: %s", where)
|
||||||
|
}
|
||||||
|
where, args = whereClause(Filters{Query: "Ada Smith", SearchIn: "author"})
|
||||||
|
if !strings.Contains(where, "p.authors_json") || strings.Contains(where, "p.abstract") || !reflect.DeepEqual(args, []any{"Ada Smith"}) {
|
||||||
|
t.Fatalf("author search includes unrelated text: %s", where)
|
||||||
|
}
|
||||||
|
where, args = whereClause(Filters{Topic: "cs.AI"})
|
||||||
|
if !strings.Contains(where, "t.value=$1") || !reflect.DeepEqual(args, []any{"cs.AI"}) {
|
||||||
|
t.Fatalf("topic-only binding: %s %#v", where, args)
|
||||||
|
}
|
||||||
|
where, args = whereClause(Filters{})
|
||||||
|
if where != "" || len(args) != 0 {
|
||||||
|
t.Fatal("empty filters changed the corpus")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSchemaCannotEscapeQualification(t *testing.T) {
|
||||||
|
for _, name := range []string{"", `research";DROP SCHEMA public;--`, "a.b", "Public", strings.Repeat("a", 37)} {
|
||||||
|
if _, err := NewPostgresRepository(&sql.DB{}, name); err == nil {
|
||||||
|
t.Fatalf("accepted schema %q", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := NewPostgresRepository(nil, "research"); err == nil {
|
||||||
|
t.Fatal("accepted nil database")
|
||||||
|
}
|
||||||
|
r, err := NewPostgresRepository(&sql.DB{}, "research_v2")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got := r.query(`SELECT p.id FROM {schema}papers p JOIN {schema}source_records s ON s.paper_id=p.id`)
|
||||||
|
if got != `SELECT p.id FROM "research_v2".papers p JOIN "research_v2".source_records s ON s.paper_id=p.id` {
|
||||||
|
t.Fatalf("unqualified corpus tables: %s", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user