refactor: migrate TrumpSignal to TypeScript monorepo and Pages Router
This commit is contained in:
@@ -0,0 +1,20 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
const source = process.env.DATABASE_URL,
|
||||
destination = process.argv[2]
|
||||
if (!source || !destination) throw new Error('DATABASE_URL and backup output path required')
|
||||
const url = new URL(source),
|
||||
child = spawn('pg_dump', ['--format=custom', '--file', destination], {
|
||||
stdio: ['ignore', 'inherit', 'inherit'],
|
||||
env: {
|
||||
...process.env,
|
||||
PGHOST: url.hostname,
|
||||
PGPORT: url.port || '5432',
|
||||
PGUSER: decodeURIComponent(url.username),
|
||||
PGPASSWORD: decodeURIComponent(url.password),
|
||||
PGDATABASE: url.pathname.slice(1),
|
||||
},
|
||||
})
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
child.once('error', reject)
|
||||
child.once('exit', (code) => (code === 0 ? resolve() : reject(new Error('pg_dump failed'))))
|
||||
})
|
||||
@@ -0,0 +1,144 @@
|
||||
import fs from 'node:fs'
|
||||
import path from 'node:path'
|
||||
import ts from 'typescript'
|
||||
const root = path.resolve(import.meta.dirname, '..')
|
||||
const files = []
|
||||
function walk(dir) {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
if (['.git', 'node_modules'].includes(entry.name)) continue
|
||||
const file = path.join(dir, entry.name)
|
||||
if (entry.isDirectory()) walk(file)
|
||||
else if (/\.(ts|tsx|mjs)$/.test(file)) files.push(file)
|
||||
}
|
||||
}
|
||||
walk(root)
|
||||
const errors = [],
|
||||
routes = [],
|
||||
columns = new Map()
|
||||
function value(node, env) {
|
||||
if (!node) return undefined
|
||||
if (ts.isAsExpression(node) || ts.isParenthesizedExpression(node))
|
||||
return value(node.expression, env)
|
||||
if (ts.isStringLiteral(node) || ts.isNoSubstitutionTemplateLiteral(node)) return node.text
|
||||
if (ts.isIdentifier(node)) return env[node.text]
|
||||
if (ts.isArrayLiteralExpression(node)) return node.elements.map((n) => value(n, env))
|
||||
if (ts.isBinaryExpression(node) && node.operatorToken.kind === ts.SyntaxKind.PlusToken) {
|
||||
const a = value(node.left, env),
|
||||
b = value(node.right, env)
|
||||
return typeof a === 'string' && typeof b === 'string' ? a + b : undefined
|
||||
}
|
||||
}
|
||||
function normalize(route) {
|
||||
return route.replace(/\{[^}]+\}|:[^/]+/g, ':param')
|
||||
}
|
||||
for (const file of files) {
|
||||
const source = ts.createSourceFile(
|
||||
file,
|
||||
fs.readFileSync(file, 'utf8'),
|
||||
ts.ScriptTarget.Latest,
|
||||
true,
|
||||
)
|
||||
for (const d of source.parseDiagnostics)
|
||||
errors.push(
|
||||
path.relative(root, file) + ': ' + ts.flattenDiagnosticMessageText(d.messageText, ' '),
|
||||
)
|
||||
function visit(node, env = {}) {
|
||||
if (ts.isForOfStatement(node) && ts.isVariableDeclarationList(node.initializer)) {
|
||||
const name = node.initializer.declarations[0].name,
|
||||
values = value(node.expression, env)
|
||||
if (ts.isIdentifier(name) && Array.isArray(values)) {
|
||||
for (const item of values) visit(node.statement, { ...env, [name.text]: item })
|
||||
return
|
||||
}
|
||||
}
|
||||
if (
|
||||
ts.isCallExpression(node) &&
|
||||
ts.isPropertyAccessExpression(node.expression) &&
|
||||
ts.isIdentifier(node.expression.expression) &&
|
||||
node.expression.expression.text === 'app' &&
|
||||
['get', 'post', 'put', 'patch', 'delete'].includes(node.expression.name.text)
|
||||
) {
|
||||
const route = value(node.arguments[0], env)
|
||||
if (route)
|
||||
routes.push({
|
||||
method: node.expression.name.text.toUpperCase(),
|
||||
path: route,
|
||||
file: path.relative(root, file),
|
||||
})
|
||||
}
|
||||
if (
|
||||
file.endsWith('/packages/database/src/schema.ts') &&
|
||||
ts.isCallExpression(node) &&
|
||||
ts.isIdentifier(node.expression) &&
|
||||
node.expression.text === 'pgTable' &&
|
||||
ts.isStringLiteral(node.arguments[0]) &&
|
||||
ts.isObjectLiteralExpression(node.arguments[1])
|
||||
)
|
||||
columns.set(
|
||||
node.arguments[0].text,
|
||||
node.arguments[1].properties
|
||||
.filter(ts.isPropertyAssignment)
|
||||
.map((p) => p.name.getText(source).replaceAll(/['"]/g, '')),
|
||||
)
|
||||
ts.forEachChild(node, (child) => visit(child, env))
|
||||
}
|
||||
visit(source)
|
||||
}
|
||||
const expected = JSON.parse(fs.readFileSync(path.join(root, 'migration/routes.json'), 'utf8'))
|
||||
for (const route of expected)
|
||||
if (!routes.some((r) => r.method === route.method && normalize(r.path) === normalize(route.path)))
|
||||
errors.push('Missing API route ' + route.method + ' ' + route.path)
|
||||
const legacyColumns = JSON.parse(
|
||||
fs.readFileSync(path.join(root, 'migration/database-columns.json'), 'utf8'),
|
||||
)
|
||||
for (const [table, names] of Object.entries(legacyColumns))
|
||||
for (const name of names)
|
||||
if (!columns.get(table)?.includes(name))
|
||||
errors.push('Missing database column ' + table + '.' + name)
|
||||
const sourcePages = [
|
||||
'/',
|
||||
'/en',
|
||||
'/en/trump',
|
||||
'/en/macro',
|
||||
'/en/kol',
|
||||
'/en/trades',
|
||||
'/en/analytics',
|
||||
'/en/settings',
|
||||
'/en/archive',
|
||||
'/en/posts',
|
||||
'/en/btc',
|
||||
'/en/methodology',
|
||||
'/en/glossary',
|
||||
'/en/case-studies',
|
||||
'/en/privacy',
|
||||
'/en/terms',
|
||||
'/en/contact',
|
||||
]
|
||||
for (const route of sourcePages) {
|
||||
const relative =
|
||||
route === '/'
|
||||
? 'index'
|
||||
: route === '/en'
|
||||
? '[locale]/index'
|
||||
: route.replace('/en/', '[locale]/')
|
||||
if (!fs.existsSync(path.join(root, 'apps/web/pages', relative + '.tsx')))
|
||||
errors.push('Missing page ' + route)
|
||||
}
|
||||
const report = {
|
||||
sourceFilesParsed: files.length,
|
||||
legacyApiHandlers: expected.length,
|
||||
registeredHttpRoutes: routes.length,
|
||||
pages: sourcePages.length,
|
||||
legacyTables: Object.keys(legacyColumns).length,
|
||||
legacyColumns: Object.values(legacyColumns).reduce((n, a) => n + a.length, 0),
|
||||
errors,
|
||||
applicationExecuted: false,
|
||||
buildExecuted: false,
|
||||
testsExecuted: false,
|
||||
}
|
||||
fs.writeFileSync(
|
||||
path.join(root, 'migration/static-review.json'),
|
||||
JSON.stringify(report, null, 2) + '\n',
|
||||
)
|
||||
console.log(JSON.stringify(report, null, 2))
|
||||
if (errors.length) process.exitCode = 1
|
||||
@@ -0,0 +1,62 @@
|
||||
/** Offline, explicit import into an EMPTY migrated PostgreSQL database. */
|
||||
import { DatabaseSync } from 'node:sqlite'
|
||||
import { createDatabase } from '@trumpsignal/database'
|
||||
import columns from '../migration/database-columns.json'
|
||||
const source = process.argv[2],
|
||||
url = process.env.DATABASE_URL
|
||||
if (!source || !url)
|
||||
throw new Error('Usage: DATABASE_URL=... tsx scripts/import-sqlite.ts /absolute/backup.sqlite')
|
||||
const sqlite = new DatabaseSync(source, { readOnly: true }),
|
||||
{ pool, lockPool } = createDatabase(url),
|
||||
client = await pool.connect()
|
||||
const booleanNames = new Set([
|
||||
'relevant',
|
||||
'active',
|
||||
'paper_mode',
|
||||
'auto_trade',
|
||||
'trump_enabled',
|
||||
'macro_enabled',
|
||||
'grow_mode',
|
||||
'talks_vs_trades_flag',
|
||||
'alerts_enabled',
|
||||
'alert_trump',
|
||||
'alert_btc_bottom',
|
||||
'alert_funding',
|
||||
'alert_kol_divergence',
|
||||
'digest_enabled',
|
||||
])
|
||||
try {
|
||||
await client.query('BEGIN')
|
||||
for (const table of Object.keys(columns)) {
|
||||
const count = await client.query<{ n: string }>(`SELECT count(*) AS n FROM "${table}"`)
|
||||
if (Number(count.rows[0].n) > 0) throw new Error('Destination must be empty: ' + table)
|
||||
}
|
||||
for (const [table, names] of Object.entries(columns)) {
|
||||
const rows = sqlite
|
||||
.prepare(`SELECT ${names.map((n) => '"' + n + '"').join(',')} FROM "${table}"`)
|
||||
.all()
|
||||
for (const row of rows) {
|
||||
const values = names.map((n) =>
|
||||
booleanNames.has(n) && row[n] != null ? !!row[n] : (row[n] ?? null),
|
||||
)
|
||||
await client.query(
|
||||
`INSERT INTO "${table}" (${names.map((n) => '"' + n + '"').join(',')}) VALUES (${names.map((_, i) => '$' + (i + 1)).join(',')})`,
|
||||
values,
|
||||
)
|
||||
}
|
||||
await client.query(
|
||||
`SELECT setval(pg_get_serial_sequence($1,'id'),coalesce(max(id),1),max(id) IS NOT NULL) FROM "${table}"`,
|
||||
[table],
|
||||
)
|
||||
}
|
||||
await client.query('COMMIT')
|
||||
process.stdout.write('Import completed. Source opened read-only.\n')
|
||||
} catch (e) {
|
||||
await client.query('ROLLBACK')
|
||||
throw e
|
||||
} finally {
|
||||
sqlite.close()
|
||||
client.release()
|
||||
await pool.end()
|
||||
await lockPool.end()
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
import { readdir, readFile } from 'node:fs/promises'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { createHash } from 'node:crypto'
|
||||
import { createDatabase } from '@trumpsignal/database'
|
||||
const url = process.env.DATABASE_URL
|
||||
if (!url) throw new Error('DATABASE_URL required')
|
||||
const { pool, lockPool } = createDatabase(url),
|
||||
client = await pool.connect()
|
||||
try {
|
||||
await client.query('SELECT pg_advisory_lock(hashtextextended($1,0))', ['trumpsignal-migration'])
|
||||
await client.query(
|
||||
'CREATE TABLE IF NOT EXISTS typescript_migrations (name TEXT PRIMARY KEY, sha256 TEXT NOT NULL, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())',
|
||||
)
|
||||
const folder = fileURLToPath(new URL('../packages/database/migrations/', import.meta.url)),
|
||||
files = (await readdir(folder)).filter((n) => n.endsWith('.sql')).sort()
|
||||
for (const name of files) {
|
||||
const source = await readFile(folder + name, 'utf8'),
|
||||
hash = createHash('sha256').update(source).digest('hex'),
|
||||
old = await client.query<{ sha256: string }>(
|
||||
'SELECT sha256 FROM typescript_migrations WHERE name=$1',
|
||||
[name],
|
||||
)
|
||||
if (old.rows[0]) {
|
||||
if (old.rows[0].sha256 !== hash)
|
||||
throw new Error('Previously applied migration changed: ' + name)
|
||||
continue
|
||||
}
|
||||
if (name.startsWith('0001')) {
|
||||
const legacy = await client.query<{ exists: boolean }>(
|
||||
"SELECT to_regclass('public.alembic_version') IS NOT NULL AS exists",
|
||||
)
|
||||
if (legacy.rows[0].exists) {
|
||||
const head = await client.query<{ version_num: string }>(
|
||||
'SELECT version_num FROM alembic_version',
|
||||
)
|
||||
if (!head.rows.some((r) => r.version_num.startsWith('027')))
|
||||
throw new Error('Existing legacy database must reach Alembic head 027 first')
|
||||
}
|
||||
}
|
||||
await client.query('BEGIN')
|
||||
try {
|
||||
await client.query(source)
|
||||
await client.query('INSERT INTO typescript_migrations(name,sha256) VALUES ($1,$2)', [
|
||||
name,
|
||||
hash,
|
||||
])
|
||||
await client.query('COMMIT')
|
||||
process.stdout.write('Applied ' + name + '\n')
|
||||
} catch (e) {
|
||||
await client.query('ROLLBACK')
|
||||
throw e
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await client.query('SELECT pg_advisory_unlock(hashtextextended($1,0))', ['trumpsignal-migration'])
|
||||
client.release()
|
||||
await pool.end()
|
||||
await lockPool.end()
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
import { eq } from 'drizzle-orm'
|
||||
import { pollBreakout } from '@trumpsignal/backend/breakout'
|
||||
import { createDatabase, subscriptions } from '@trumpsignal/database'
|
||||
import { createContext } from '@trumpsignal/backend/context'
|
||||
import { readConfig } from '@trumpsignal/backend/config'
|
||||
import { createServices } from '@trumpsignal/backend/services'
|
||||
import {
|
||||
reanalyze,
|
||||
backtestBatch,
|
||||
backtestPost,
|
||||
backfillImpact,
|
||||
} from '@trumpsignal/backend/operations'
|
||||
import { decryptApiKey, encryptApiKey } from '@trumpsignal/core/crypto'
|
||||
const config = readConfig(),
|
||||
database = createDatabase(config.database_url),
|
||||
c = createContext(database, config),
|
||||
s = createServices(c),
|
||||
[command, arg] = process.argv.slice(2)
|
||||
try {
|
||||
let result: unknown
|
||||
switch (command) {
|
||||
case 'breakout':
|
||||
result = await pollBreakout(c, s.state)
|
||||
break
|
||||
case 'preflight':
|
||||
await database.pool.query('SELECT 1')
|
||||
result = {
|
||||
database: true,
|
||||
encryption_key_configured: config.encryption_key.length >= 32,
|
||||
ingest_configured: !!config.ingest_api_key,
|
||||
telegram_configured: s.telegram.configured(),
|
||||
ai_configured: !!(config.ai_api_key || config.anthropic_api_key),
|
||||
}
|
||||
break
|
||||
case 'seed-wallets':
|
||||
result = await s.kol.seed()
|
||||
break
|
||||
case 'truth-backfill':
|
||||
result = await s.truth.cnn(true)
|
||||
break
|
||||
case 'kol-feeds':
|
||||
result = await s.kol.pollFeeds()
|
||||
break
|
||||
case 'kol-x':
|
||||
result = await s.kol.pollX()
|
||||
break
|
||||
case 'onchain':
|
||||
result = await s.onchain.poll()
|
||||
break
|
||||
case 'divergence':
|
||||
result = await s.kol.divergence(Number(arg) || 30)
|
||||
break
|
||||
case 'macro':
|
||||
result = await s.macro.poll()
|
||||
break
|
||||
case 'backfill-prices':
|
||||
result = await backfillImpact(c, arg ?? 'BTC')
|
||||
break
|
||||
case 'reanalyze':
|
||||
result = await reanalyze(c, s, {
|
||||
limit: Number(arg) || 500,
|
||||
dry_run: process.env.DRY_RUN !== 'false',
|
||||
delay_secs: 0.5,
|
||||
legacy_signals: true,
|
||||
})
|
||||
break
|
||||
case 'backtest':
|
||||
result = await backtestPost(c, Number(arg))
|
||||
break
|
||||
case 'backtest-batch':
|
||||
result = await backtestBatch(c, Number(arg) || 50)
|
||||
break
|
||||
case 'reencrypt-keys': {
|
||||
const old = process.env.OLD_ENCRYPTION_KEY ?? config.encryption_key,
|
||||
dry = process.env.DRY_RUN !== 'false',
|
||||
rows = await c.db.select().from(subscriptions)
|
||||
let changed = 0
|
||||
for (const row of rows) {
|
||||
if (!row.hl_api_key) continue
|
||||
const key = await decryptApiKey(row.hl_api_key, old, config.environment)
|
||||
if (!dry)
|
||||
await c.db
|
||||
.update(subscriptions)
|
||||
.set({ hl_api_key: await encryptApiKey(key, config.encryption_key) })
|
||||
.where(eq(subscriptions.id, row.id))
|
||||
changed++
|
||||
}
|
||||
result = { dry_run: dry, keys_processed: changed }
|
||||
break
|
||||
}
|
||||
default:
|
||||
throw new Error(
|
||||
'Commands: preflight, seed-wallets, truth-backfill, kol-feeds, kol-x, onchain, divergence, macro, backfill-prices, reanalyze, backtest, backtest-batch, reencrypt-keys',
|
||||
)
|
||||
}
|
||||
process.stdout.write(JSON.stringify(result ?? { ok: true }, null, 2) + '\n')
|
||||
} finally {
|
||||
await database.pool.end()
|
||||
await database.lockPool.end()
|
||||
}
|
||||
Reference in New Issue
Block a user