# ------------------------------------------------------------
# 1. Standard Search Engines (Google, Bing, Apple, etc.)
# ------------------------------------------------------------
User-agent: *
Allow: /
Disallow: /admin/
Disallow: /login/
Disallow: /client-portal/
Disallow: /checkout/
Disallow: /private/
Disallow: /staging/

# ------------------------------------------------------------
# 2. AI Search & User-Requested Retrieval
# Deliberate policy: allow answer-engine discovery and user-requested fetches.
# ------------------------------------------------------------
# OpenAI search and user-requested browsing
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

# Perplexity answer engine
User-agent: PerplexityBot
Allow: /

# Anthropic search and user-requested browsing
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Google and Apple controls combine AI product use with broader model use;
# they remain allowed because answer-engine visibility is the priority here.
User-agent: Google-Extended
Allow: /

User-agent: Applebot-Extended
Allow: /

# ------------------------------------------------------------
# 3. Training-Only AI Crawlers
# Deliberate policy: opt out when training has a separate crawler control.
# ------------------------------------------------------------
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Bytespider
Disallow: /

# ------------------------------------------------------------
# 4. XML Sitemap Location
# ------------------------------------------------------------
Sitemap: https://phase2labs.net/sitemap.xml