# https://africatokenization.com/ — The Vanderbilt Portfolio
# Generated 2026-03-28

# ============================================================
# DEFAULT: Allow everything except thin auto-generated pages
# ============================================================
User-agent: *
Allow: /
Disallow: /search/
Disallow: /tags/
Disallow: /categories/
Disallow: /regions/
Disallow: /subscribe/
Disallow: /404.html

Sitemap: https://africatokenization.com/sitemap.xml

# ============================================================
# GOOGLEBOT: Maximum crawl speed — no delay
# Every second of faster indexing = earlier ad revenue
# ============================================================
User-agent: Googlebot
Allow: /
Crawl-delay: 0

User-agent: Googlebot-Image
Allow: /

User-agent: Googlebot-News
Allow: /

# ============================================================
# BING: Polite but comprehensive
# ============================================================
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# ============================================================
# AI CRAWLERS: ALLOW — citation = free authority backlinks
# When GPT/Gemini/Claude cite your content, that's a link signal
# ============================================================
User-agent: GPTBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: anthropic-ai
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot
Allow: /

# ============================================================
# SOCIAL MEDIA CRAWLERS: ALLOW — OG tags need to render
# ============================================================
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# ============================================================
# BLOCK: Scraper bots that add zero value
# ============================================================
User-agent: CCBot
Disallow: /

User-agent: ByteSpider
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: PetalBot
Disallow: /