import type { MetadataRoute } from 'next' const siteUrl = process.env.NEXT_PUBLIC_SITE_URL || 'https://trumpsignal.com' // Settings page is user-personalized (wallet address, HL keys, alert prefs). // Disallow under every locale prefix in case any /zh URLs survive in the wild // — i18n is shelved but the routes still resolve. const SETTINGS_DENY = ['/api/', '/_next/', '/en/settings', '/zh/settings'] function robots(): MetadataRoute.Robots { return { rules: [ // Default: allow everything except private pages { userAgent: '*', allow: '/', disallow: SETTINGS_DENY, }, // ── Standard search crawlers ────────────────────────────────────────── { userAgent: 'Googlebot', allow: '/' }, { userAgent: 'Bingbot', allow: '/' }, { userAgent: 'Slurp', allow: '/' }, // Yahoo { userAgent: 'DuckDuckBot', allow: '/' }, { userAgent: 'YandexBot', allow: '/' }, // ── AI / LLM crawlers (GEO) ─────────────────────────────────────────── // OpenAI — ChatGPT Browse + training { userAgent: 'GPTBot', allow: '/', disallow: SETTINGS_DENY }, { userAgent: 'ChatGPT-User', allow: '/', disallow: SETTINGS_DENY }, { userAgent: 'OAI-SearchBot', allow: '/', disallow: SETTINGS_DENY }, // Anthropic — Claude.ai + training (settings are already private so // we use a shorter deny list to let anthropic-ai see more content) { userAgent: 'ClaudeBot', allow: '/', disallow: SETTINGS_DENY }, { userAgent: 'anthropic-ai', allow: '/', disallow: ['/api/', '/_next/'] }, // Perplexity AI { userAgent: 'PerplexityBot', allow: '/', disallow: SETTINGS_DENY }, // Google — Gemini / AI Overviews / Bard training { userAgent: 'Google-Extended', allow: '/', disallow: SETTINGS_DENY }, { userAgent: 'Googlebot-Extended', allow: '/', disallow: SETTINGS_DENY }, // Meta AI (Llama, Meta AI assistant) { userAgent: 'meta-externalagent', allow: '/', disallow: SETTINGS_DENY }, { userAgent: 'FacebookBot', allow: '/', disallow: SETTINGS_DENY }, // Apple (Siri, AI features in Safari) { userAgent: 'Applebot', allow: '/' }, { userAgent: 'Applebot-Extended', allow: '/', disallow: SETTINGS_DENY }, // Cohere — Command R, enterprise LLMs { userAgent: 'cohere-ai', allow: '/', disallow: SETTINGS_DENY }, // Mistral / Le Chat { userAgent: 'MistralAI-User', allow: '/', disallow: SETTINGS_DENY }, // You.com AI search { userAgent: 'YouBot', allow: '/', disallow: SETTINGS_DENY }, // Brave AI (Leo assistant) { userAgent: 'Brave', allow: '/' }, // Common Crawl — feeds many open-source LLM training datasets { userAgent: 'CCBot', allow: '/', disallow: SETTINGS_DENY }, ], sitemap: `${siteUrl}/sitemap.xml`, host: siteUrl, } } export default function handler( _req: import('next').NextApiRequest, res: import('next').NextApiResponse, ) { res.setHeader('Content-Type', 'text/plain') const data = robots() const rules = Array.isArray(data.rules) ? data.rules : [data.rules] res.send( rules .map( (r) => 'User-agent: ' + (r.userAgent ?? '*') + '\n' + [r.allow] .flat() .filter(Boolean) .map((p) => 'Allow: ' + p) .join('\n') + '\n' + [r.disallow] .flat() .filter(Boolean) .map((p) => 'Disallow: ' + p) .join('\n'), ) .join('\n') + '\nSitemap: ' + data.sitemap, ) }