94 lines
3.7 KiB
TypeScript
94 lines
3.7 KiB
TypeScript
import type { MetadataRoute } from 'next'
|
|
|
|
const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || 'https://trumpsignal.com'
|
|
|
|
// Settings page is user-personalized (wallet address, HL keys, alert prefs).
|
|
// Disallow under every locale prefix in case any /zh URLs survive in the wild
|
|
// — i18n is shelved but the routes still resolve.
|
|
const SETTINGS_DENY = ['/api/', '/_next/', '/en/settings', '/zh/settings']
|
|
|
|
function robots(): MetadataRoute.Robots {
|
|
return {
|
|
rules: [
|
|
// Default: allow everything except private pages
|
|
{
|
|
userAgent: '*',
|
|
allow: '/',
|
|
disallow: SETTINGS_DENY,
|
|
},
|
|
|
|
// ── Standard search crawlers ──────────────────────────────────────────
|
|
{ userAgent: 'Googlebot', allow: '/' },
|
|
{ userAgent: 'Bingbot', allow: '/' },
|
|
{ userAgent: 'Slurp', allow: '/' }, // Yahoo
|
|
{ userAgent: 'DuckDuckBot', allow: '/' },
|
|
{ userAgent: 'YandexBot', allow: '/' },
|
|
|
|
// ── AI / LLM crawlers (GEO) ───────────────────────────────────────────
|
|
// OpenAI — ChatGPT Browse + training
|
|
{ userAgent: 'GPTBot', allow: '/', disallow: SETTINGS_DENY },
|
|
{ userAgent: 'ChatGPT-User', allow: '/', disallow: SETTINGS_DENY },
|
|
{ userAgent: 'OAI-SearchBot', allow: '/', disallow: SETTINGS_DENY },
|
|
// Anthropic — Claude.ai + training (settings are already private so
|
|
// we use a shorter deny list to let anthropic-ai see more content)
|
|
{ userAgent: 'ClaudeBot', allow: '/', disallow: SETTINGS_DENY },
|
|
{ userAgent: 'anthropic-ai', allow: '/', disallow: ['/api/', '/_next/'] },
|
|
// Perplexity AI
|
|
{ userAgent: 'PerplexityBot', allow: '/', disallow: SETTINGS_DENY },
|
|
// Google — Gemini / AI Overviews / Bard training
|
|
{ userAgent: 'Google-Extended', allow: '/', disallow: SETTINGS_DENY },
|
|
{ userAgent: 'Googlebot-Extended', allow: '/', disallow: SETTINGS_DENY },
|
|
// Meta AI (Llama, Meta AI assistant)
|
|
{ userAgent: 'meta-externalagent', allow: '/', disallow: SETTINGS_DENY },
|
|
{ userAgent: 'FacebookBot', allow: '/', disallow: SETTINGS_DENY },
|
|
// Apple (Siri, AI features in Safari)
|
|
{ userAgent: 'Applebot', allow: '/' },
|
|
{ userAgent: 'Applebot-Extended', allow: '/', disallow: SETTINGS_DENY },
|
|
// Cohere — Command R, enterprise LLMs
|
|
{ userAgent: 'cohere-ai', allow: '/', disallow: SETTINGS_DENY },
|
|
// Mistral / Le Chat
|
|
{ userAgent: 'MistralAI-User', allow: '/', disallow: SETTINGS_DENY },
|
|
// You.com AI search
|
|
{ userAgent: 'YouBot', allow: '/', disallow: SETTINGS_DENY },
|
|
// Brave AI (Leo assistant)
|
|
{ userAgent: 'Brave', allow: '/' },
|
|
// Common Crawl — feeds many open-source LLM training datasets
|
|
{ userAgent: 'CCBot', allow: '/', disallow: SETTINGS_DENY },
|
|
],
|
|
sitemap: `${siteUrl}/sitemap.xml`,
|
|
host: siteUrl,
|
|
}
|
|
}
|
|
|
|
export default function handler(
|
|
_req: import('next').NextApiRequest,
|
|
res: import('next').NextApiResponse,
|
|
) {
|
|
res.setHeader('Content-Type', 'text/plain')
|
|
const data = robots()
|
|
const rules = Array.isArray(data.rules) ? data.rules : [data.rules]
|
|
res.send(
|
|
rules
|
|
.map(
|
|
(r) =>
|
|
'User-agent: ' +
|
|
(r.userAgent ?? '*') +
|
|
'\n' +
|
|
[r.allow]
|
|
.flat()
|
|
.filter(Boolean)
|
|
.map((p) => 'Allow: ' + p)
|
|
.join('\n') +
|
|
'\n' +
|
|
[r.disallow]
|
|
.flat()
|
|
.filter(Boolean)
|
|
.map((p) => 'Disallow: ' + p)
|
|
.join('\n'),
|
|
)
|
|
.join('\n') +
|
|
'\nSitemap: ' +
|
|
data.sitemap,
|
|
)
|
|
}
|