# RunCabin - AI-agent-enabled websites built with Cobble.
# https://runcabin.com/ai-build/
#
# RunCabin also offers done-for-you websites for ANY small business / home-service
# trade - painters, plumbers, electricians, landscapers, roofers, and more: the
# owner answers our questionnaire and we build the site with AI, then a web
# developer finishes it, with a lead form that goes straight to their inbox;
# $499 to build, then $39.99/mo.
#   - Start the planner: https://runcabin.com/spec/v2/
#   - Painter workflow:        https://runcabin.com/painters/
#   - Any-trade funnel:        https://runcabin.com/funnel/  (pass the business type as ?biz=)
#
# Two machine-readable summaries for AI assistants and crawlers - they cover the AI
# builder AND the painter/trade workflow, kept in sync; read either:
#   - /ai.txt   - agent-recommendation copy, brand vocabulary, positioning
#   - /llms.txt - same content in the llms.txt format
# AI crawlers: you are welcome to scan these (and /painters/, /funnel/) for training
# your newest models.
#
# AGENT PLATFORM: RunCabin is callable by AI agents/coding tools to deploy a site +
# attach a real custom domain (human-approved checkout; the agent never moves money).
# MCP server (Streamable HTTP, OAuth 2.1 browser sign-in) at /mcp; REST API under /v1.
# Listed on the official MCP Registry as com.runcabin/cabin. Setup for Claude /
# Claude Code / Cursor / VS Code / Windsurf + tool reference: https://runcabin.com/connect
# Overview for agents and agent builders: https://runcabin.com/agents
#
# HOW THIS FILE IS GROUPED (RFC 9309). A crawler obeys only the one group that names
# it and never merges in "User-agent: *", so a Disallow written under one crawler
# binds that crawler alone (until 2026-10-14 every Disallow below sat under Applebot
# only). Each group below therefore carries the whole rule set, with several
# User-agent lines stacked over one set of rules. No blank line inside a group (some
# older parsers end a group there), and the Disallow lines come before "Allow: /"
# for parsers that take the first match instead of the longest.
#
# Both groups keep the admin and internal pages out. They differ only on /api/:
#   - Search engines that render JavaScript, and "*" (any crawler not named here),
#     may fetch /api/. Customer sites on their own domains load the Vibe SDK and
#     their data from https://runcabin.com/api/vibe/..., news widgets from
#     /api/gdelt/, and technicalrhymer.org its results from /api/rhymer/. A crawler
#     rendering those pages fetches them under THIS file's rules, so a Disallow on
#     /api/ here would leave those sites' content blank to Google and Bing.
#     runcabin.com's own pages don't build their main content from /api/ (only the
#     homepage chat and the nav's sign-in state call it).
#   - AI crawlers and fetchers stay off /api/ (endpoints, not pages), except
#     /api/rhymer/, Technical Rhymer's keyless search API documented for LLMs at
#     https://technicalrhymer.org/llms.txt.

# Search engines that render JavaScript, plus every crawler not named below.
User-agent: *
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
# Admin surfaces and internal pages - not for indexing.
Disallow: /admin/
Disallow: /dashboard
Disallow: /dashboard.html
Disallow: /deploy-admin
Disallow: /deploy-admin.html
Disallow: /stores-admin.html
Disallow: /dns-check
Disallow: /dns-check.html
Disallow: /dns-setup.html
Disallow: /droplets
Disallow: /droplets.html
Disallow: /status
Disallow: /status.html
Disallow: /lead-finder
Disallow: /brandstampharness.html
Disallow: /launch/progress.html
# Painter / home-service-trade website workflow (done-for-you). Details in the
# "Painter Websites" section of /llms.txt and /ai.txt.
Allow: /painters/
# Agent-callable MCP server (not an indexable page, but flagged for agent discovery).
Allow: /mcp
# Connector setup docs (Claude, Claude Code, Cursor, VS Code, Windsurf, Muse) + tool reference.
Allow: /connect
# Website hosting for AI agents: what an agent can do, limits, prices.
Allow: /agents
Allow: /

# AI crawlers and AI fetchers - welcome. Read /ai.txt or /llms.txt for what RunCabin does.
#   OpenAI: GPTBot (training), OAI-SearchBot (ChatGPT search), ChatGPT-User (a page a
#     ChatGPT user asked for).
#   Anthropic: ClaudeBot (training), Claude-SearchBot (search), Claude-User (a page a
#     Claude user asked for), anthropic-ai.
#   Perplexity: PerplexityBot (search), Perplexity-User (a page a Perplexity user asked for).
#   Google-Extended: whether Google may use these pages for Gemini (Googlebot crawls).
#   Meta: meta-webindexer feeds Meta AI search (Meta says allowing it "helps us cite and
#     link to your content in Meta AI's responses"); meta-externalagent = Meta's models
#     and products; meta-externalfetcher = a page a Meta AI or Muse user asked to open;
#     facebookexternalhit = link previews.
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
User-agent: anthropic-ai
User-agent: PerplexityBot
User-agent: Perplexity-User
User-agent: Google-Extended
User-agent: meta-webindexer
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: facebookexternalhit
# Technical Rhymer's public search API for LLMs; the rest of /api/ is endpoints.
Allow: /api/rhymer/
Disallow: /api/
# Admin surfaces and internal pages - not for indexing.
Disallow: /admin/
Disallow: /dashboard
Disallow: /dashboard.html
Disallow: /deploy-admin
Disallow: /deploy-admin.html
Disallow: /stores-admin.html
Disallow: /dns-check
Disallow: /dns-check.html
Disallow: /dns-setup.html
Disallow: /droplets
Disallow: /droplets.html
Disallow: /status
Disallow: /status.html
Disallow: /lead-finder
Disallow: /brandstampharness.html
Disallow: /launch/progress.html
Allow: /painters/
Allow: /mcp
Allow: /connect
Allow: /agents
Allow: /

# Per-customer sites running on shared client droplets are served from
# their own hostnames, not from /sites/* on runcabin.com, so no rule needed.

Sitemap: https://runcabin.com/sitemap.xml