# Robots.txt for Vife AI Agent Platform
# Website: https://vife.ai
# Last updated: 2026-09-09
# Supports Multi-language URLs: /en/*, /zh-CN/*, /ja/*, etc.
#
# NOTE: Cloudflare's "Managed robots.txt" feature PREPENDS a managed block to
# this file at the edge (a `User-agent: *` Content-Signal: search=yes,ai-train=no
# group plus AI-training crawler blocks: GPTBot, Google-Extended, CCBot,
# ClaudeBot, Bytespider, Amazonbot, meta-externalagent, Applebot-Extended, ...).
# To avoid emitting those bot blocks twice, this file intentionally does NOT
# repeat the AI-training crawlers Cloudflare already covers — it only defines
# what Cloudflare does not: private-path disallows, the AI-search allow-list,
# SEO auditors, and aggressive low-value crawlers. If the Cloudflare managed
# block is ever disabled in the dashboard, re-add the AI-training blocks here.

# Match localized private route boundaries, not script filenames such as
# /assets/settings-<hash>.js. Keep query strings and nested routes blocked.

# Default rules for all robots
User-agent: *
# Disallow user settings and private pages
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?
# Disallow API endpoints
Disallow: /api/
# No Crawl-delay here: Google ignores it and Bing honors it, which would
# throttle crawling of 1400+ sitemap URLs. Aggressive bots are blocked below.

# Special rules for specific bots
# ===================================

# Google Bot - public pages indexable, API blocked
User-agent: Googlebot
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

# Google Image Bot - allow public images, block private app/API paths
User-agent: Googlebot-Image
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?
Allow: /

# Bing Bot - public pages indexable, API blocked
User-agent: Bingbot
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

# AI search and user-requested assistant fetchers
# Allow public pages to appear in AI search/answer products while keeping the
# app/API private. (AI *training* crawlers are blocked by the Cloudflare managed
# block prepended above.)
User-agent: OAI-SearchBot
Allow: /
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

User-agent: ChatGPT-User
Allow: /
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

User-agent: Claude-SearchBot
Allow: /
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

User-agent: Claude-User
Allow: /
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

User-agent: PerplexityBot
Allow: /
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

User-agent: Perplexity-User
Allow: /
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

# Anthropic training crawler (not in the Cloudflare managed list) - block
User-agent: anthropic-ai
Disallow: /

# SEO audit crawlers (Ahrefs, Semrush) - allowed so we can audit our own
# rankings and backlinks. Private app/API routes stay blocked.
User-agent: AhrefsBot
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

User-agent: SemrushBot
Disallow: /api/
Disallow: /settings
Disallow: /*/settings$
Disallow: /*/settings/
Disallow: /*/settings?
Disallow: /profile
Disallow: /*/profile$
Disallow: /*/profile/
Disallow: /*/profile?

# Low-value aggressive crawlers - Block completely
User-agent: DotBot
Disallow: /

User-agent: MJ12bot
Disallow: /

# Sitemap location
Sitemap: https://vife.ai/sitemap.xml
# LLMs entry point: https://vife.ai/llms.txt

# Notes:
# ======
# 1. This site supports multiple languages via URL paths (e.g., /en/, /zh-CN/, /ja/)
# 2. Public pages are crawlable by default (no explicit Allow needed)
# 3. Private settings/profile pages and API routes are blocked
# 4. AI search/user-requested fetchers are allowed for public pages
# 5. AI training crawlers are blocked via the Cloudflare managed block above
# 6. No Crawl-delay is emitted; major search crawlers can schedule the 1400+
#    public URLs normally while private/API paths remain disallowed.