# robots.txt for https://spotlight.spokesman.com

Sitemap: https://spotlight.spokesman.com/sitemap.xml

# ----------------------------------------------------------------------------
# 1. Allow standard search engines and legitimate crawlers
# ----------------------------------------------------------------------------
User-agent: *
Allow: /

# ----------------------------------------------------------------------------
# 2. OpenAI / ChatGPT
# ----------------------------------------------------------------------------
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: GPTBot
Allow: /

User-agent: OAI-AdsBot
Allow: /

# ----------------------------------------------------------------------------
# 3. Anthropic / Claude
# ----------------------------------------------------------------------------
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# ----------------------------------------------------------------------------
# 4. Perplexity and Google AI
# ----------------------------------------------------------------------------
User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

# ----------------------------------------------------------------------------
# 5. Advertising and landing-page verification
# ----------------------------------------------------------------------------
User-agent: AdsBot-Google
Allow: /

User-agent: AdsBot-Google-Mobile
Allow: /

User-agent: Mediapartners-Google
Allow: /

# ----------------------------------------------------------------------------
# 6. Block unwanted SEO crawlers, scrapers, and content extractors
# ----------------------------------------------------------------------------
User-agent: MJ12bot
User-agent: dotbot
User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: Blexbot
User-agent: Barkrowler
User-agent: CCBot
User-agent: adbeat_bot
User-agent: Alexibot
User-agent: EmailCollector
User-agent: EmailSiphon
User-agent: EmailWolf
User-agent: ExtractorPro
User-agent: LinkextractorPro
User-agent: LinkpadBot
User-agent: SEOkicks-Robot
User-agent: SiteSnagger
User-agent: WebBandit
User-agent: WebCopier
User-agent: WebStripper
User-agent: WebVac
User-agent: WebZip
User-agent: Zeus
Disallow: /