# =============================================================================
# ClickBreeze - robots.txt
# SEO, AEO, GEO & Digital Marketing Agency | Dundee, Scotland
# Canonical Website: https://clickbreeze.co.uk
# =============================================================================

# --- 1. General Web Crawlers (Default) ---------------------------------------
User-agent: *
Allow: /
# Ensure rendering assets are accessible for full indexing and Core Web Vitals
Allow: /css/
Allow: /js/
Allow: /assets/
Allow: /favicon.svg
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
# Disallow private admin portal, serverless functions, and internal endpoints
Disallow: /admin
Disallow: /admin.html
Disallow: /api/
# Prevent duplicate indexing of URLs with tracking / session query parameters
Disallow: /*?*


# --- 2. Primary Search Engines (Google, Bing, Apple, DuckDuckGo) -------------
User-agent: Googlebot
User-agent: Googlebot-Image
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckDuckBot
User-agent: Slurp
Allow: /
Allow: /css/
Allow: /js/
Allow: /assets/
Disallow: /admin
Disallow: /admin.html
Disallow: /api/
Disallow: /*?*


# --- 3. AI Search Engines & Conversational Answer Engines ---------------------
# Explicitly welcome AI assistants & search engines to index services, case studies,
# and insights for real-time citations, search results, and brand recommendations.

# OpenAI (SearchGPT, ChatGPT Search, ChatGPT user browsing & knowledge index)
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/

# Perplexity AI (Direct conversational answers and sourced citations)
User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/

# Anthropic (Claude web search, citation generation, and answers)
User-agent: ClaudeBot
User-agent: Claude-Web
User-agent: anthropic-ai
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/

# Google AI & Gemini (AI Overviews, Gemini citations, and knowledge panels)
User-agent: Google-Extended
User-agent: GoogleOther
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/

# Apple Intelligence (Siri, Spotlight, Safari AI suggestions)
User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/

# Meta AI & Social Crawlers
User-agent: Meta-ExternalAgent
User-agent: FacebookBot
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/

# Other Enterprise AI & Search Engines (Cohere, Mistral)
User-agent: cohere-ai
User-agent: MistralAI-Web
Allow: /
Disallow: /admin
Disallow: /admin.html
Disallow: /api/


# --- 4. Aggressive Commercial Scrapers & Bandwidth Harvesters ------------------
# Block high-volume automated scrapers that consume server bandwidth without
# providing search traffic, citations, or client business value.

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /


# --- 5. XML Sitemap Declaration ----------------------------------------------
Sitemap: https://clickbreeze.co.uk/sitemap.xml