feat(seo/geo): WebSite+SearchAction, HowTo, DefinedTermSet, TechArticle, 20+ AI crawlers
SEO structured data (layout.tsx):
- WebSite schema with potentialAction/SearchAction → enables sitelinks
search box in Google results
- SoftwareApplication: added alternateName array, applicationSubCategory,
isAccessibleForFree, availability, screenshot, aggregateRating, expanded
featureList with specifics
- Organization: added logo, contactPoint
- HowTo schema (6 steps, estimatedCost $0, tools list) for "how to set up
Trump Alpha auto-trader" — gets HowTo rich result + cited by AI assistants
- DefinedTermSet in root layout (AHR999, Pi Cycle, Talks-vs-Trades, Funding
Reversal) — LLMs use this for entity definitions
- FAQPage: added 6 search-intent questions ("how do I auto trade trump tweets",
"bot that trades based on Trump posts", "how to know if KOL is dumping",
"best time to buy Bitcoin in bear market", "what is AHR999", "funding rate")
- All uses of siteUrl + today refactored to _base/_today constants
Per-page structured data:
- methodology/page.tsx: TechArticle JSON-LD (headline, about[], author, dateModified)
- glossary/page.tsx: DefinedTermSet with all terms + URL anchors per term;
buildGlossaryJsonLd() generates one DefinedTerm per entry; canonical fixed
to /en/ (was /${locale}/)
Missing metadata added:
- contact: metadata with robots noindex (no SEO value in contact form)
- privacy: metadata + canonical /en/privacy
- terms: metadata + canonical /en/terms
robots.ts — 20+ AI crawler rules:
Added: OAI-SearchBot, Google-Extended, Googlebot-Extended,
meta-externalagent, FacebookBot, Applebot, Applebot-Extended,
cohere-ai, MistralAI-User, YouBot, Brave, CCBot, Slurp, DuckDuckBot,
YandexBot. Now covers Gemini/Google AI Overviews, Meta AI, Apple Siri,
Cohere, Mistral, Brave Leo, and Common Crawl training datasets.
tsc + next build clean.
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
+37
-6
@@ -10,19 +10,50 @@ const SETTINGS_DENY = ['/api/', '/_next/', '/en/settings', '/zh/settings']
|
||||
export default function robots(): MetadataRoute.Robots {
|
||||
return {
|
||||
rules: [
|
||||
// Default: allow everything except private pages
|
||||
{
|
||||
userAgent: '*',
|
||||
allow: '/',
|
||||
disallow: SETTINGS_DENY,
|
||||
},
|
||||
// Explicitly allow AI crawlers (GEO — ensures ChatGPT/Perplexity/Claude index the site)
|
||||
{ userAgent: 'GPTBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'ChatGPT-User', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'PerplexityBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'ClaudeBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'anthropic-ai', allow: '/', disallow: ['/api/', '/_next/'] },
|
||||
|
||||
// ── Standard search crawlers ──────────────────────────────────────────
|
||||
{ userAgent: 'Googlebot', allow: '/' },
|
||||
{ userAgent: 'Bingbot', allow: '/' },
|
||||
{ userAgent: 'Slurp', allow: '/' }, // Yahoo
|
||||
{ userAgent: 'DuckDuckBot', allow: '/' },
|
||||
{ userAgent: 'YandexBot', allow: '/' },
|
||||
|
||||
// ── AI / LLM crawlers (GEO) ───────────────────────────────────────────
|
||||
// OpenAI — ChatGPT Browse + training
|
||||
{ userAgent: 'GPTBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'ChatGPT-User', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'OAI-SearchBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
// Anthropic — Claude.ai + training (settings are already private so
|
||||
// we use a shorter deny list to let anthropic-ai see more content)
|
||||
{ userAgent: 'ClaudeBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'anthropic-ai', allow: '/', disallow: ['/api/', '/_next/'] },
|
||||
// Perplexity AI
|
||||
{ userAgent: 'PerplexityBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
// Google — Gemini / AI Overviews / Bard training
|
||||
{ userAgent: 'Google-Extended', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'Googlebot-Extended', allow: '/', disallow: SETTINGS_DENY },
|
||||
// Meta AI (Llama, Meta AI assistant)
|
||||
{ userAgent: 'meta-externalagent', allow: '/', disallow: SETTINGS_DENY },
|
||||
{ userAgent: 'FacebookBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
// Apple (Siri, AI features in Safari)
|
||||
{ userAgent: 'Applebot', allow: '/' },
|
||||
{ userAgent: 'Applebot-Extended', allow: '/', disallow: SETTINGS_DENY },
|
||||
// Cohere — Command R, enterprise LLMs
|
||||
{ userAgent: 'cohere-ai', allow: '/', disallow: SETTINGS_DENY },
|
||||
// Mistral / Le Chat
|
||||
{ userAgent: 'MistralAI-User', allow: '/', disallow: SETTINGS_DENY },
|
||||
// You.com AI search
|
||||
{ userAgent: 'YouBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
// Brave AI (Leo assistant)
|
||||
{ userAgent: 'Brave', allow: '/' },
|
||||
// Common Crawl — feeds many open-source LLM training datasets
|
||||
{ userAgent: 'CCBot', allow: '/', disallow: SETTINGS_DENY },
|
||||
],
|
||||
sitemap: `${siteUrl}/sitemap.xml`,
|
||||
host: siteUrl,
|
||||
|
||||
Reference in New Issue
Block a user