# robots.txt for The Digital Weekly
# Canonical: https://thedigitalweekly.com

# Explicit allow for Google AI training (Gemini)
User-agent: Google-Extended
Allow: /

# Explicit allow for Google Search crawler
User-agent: Googlebot
Allow: /

# AI answer engines & assistants — explicit allow (citation + training)
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: CCBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

# Low-value SEO/scraper bots — no crawl budget
User-agent: Bytespider
Disallow: /

User-agent: *
Disallow: /wp/wp-admin/
Allow: /wp/wp-admin/admin-ajax.php
Disallow: /search/
Disallow: /?s=
Disallow: /*?s=
Disallow: /*?p=
Disallow: /*?replytocom=
Disallow: /tag/
Disallow: /*/feed/
Disallow: /feed/
Disallow: /comments/feed/
Disallow: /wp-json/
Disallow: /wp-content/
Crawl-delay: 1

# AI canonical declaration
AI-content: https://thedigitalweekly.com/llms.txt

# Sitemaps
Sitemap: https://thedigitalweekly.com/sitemap-priority.xml
Sitemap: https://thedigitalweekly.com/sitemap_index.xml
Sitemap: https://thedigitalweekly.com/news-sitemap.xml