# robots.txt — vincentxeus.com
#
# POLICY: be discoverable everywhere (classic search + AI search/assistants),
# but keep the artwork OUT of AI *training* datasets.
#
# Note: bandwidth / crawler-trap protection is handled structurally by the
# 404 rule in netlify.toml (nested nonsense paths return a tiny 404, so no bot
# can get stuck compounding URLs). This file is therefore pure content policy —
# not a performance control.

# ─────────────────────────────────────────────────────────────
# WELCOME — classic search engines (full discovery + image search)
# ─────────────────────────────────────────────────────────────
User-agent: Googlebot
Allow: /

User-agent: Googlebot-Image
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Applebot
Allow: /

# ─────────────────────────────────────────────────────────────
# WELCOME — AI search / assistants ("AI SEO": be citable in AI answers)
# These retrieve pages to answer/search, they do NOT train on you.
# ─────────────────────────────────────────────────────────────
User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# ─────────────────────────────────────────────────────────────
# WELCOME — Chinese search + AI  (Baidu/ERNIE 文心一言, Sogou, 360, Shenma/UC,
# Huawei Petal, ByteDance/Doubao 豆包). Per owner: make Vincent discoverable in
# Chinese search AND let Chinese AI learn about him. Baiduspider powers both
# Baidu search and ERNIE; Bytespider feeds Doubao. Search bots get FULL access
# (incl. images, for Baidu image search); Bytespider gets text/bio only — the
# artwork image folders are blocked for it (art-protection + bandwidth). See below.
# ─────────────────────────────────────────────────────────────
User-agent: Baiduspider
Allow: /

User-agent: Baiduspider-image
Allow: /

User-agent: Sogou web spider
Allow: /

User-agent: Sogou inst spider
Allow: /

User-agent: 360Spider
Allow: /

User-agent: HaoSouSpider
Allow: /

User-agent: YisouSpider
Allow: /

User-agent: PetalBot
Allow: /

User-agent: Bytespider
# Doubao/ByteDance is the aggressive one: let it read the text/bio pages (so the AI
# learns who Vincent is) but NOT download the artwork image folders — keeps the
# paintings out of training AND caps its image bandwidth. Every work is still
# visible to it via each page's text + alt descriptions.
Disallow: /works/
Disallow: /KennedyWomen/
Disallow: /Space/
Disallow: /landing/
Disallow: /fields/
Disallow: /about-band/
Disallow: /thumbnails/
Crawl-delay: 10

# ─────────────────────────────────────────────────────────────
# BLOCKED — AI *training* crawlers + image-dataset scrapers
# (keeps the paintings out of model training sets; sends ~zero traffic)
# ─────────────────────────────────────────────────────────────
User-agent: GPTBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Meta-ExternalAgent
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: AI2Bot
Disallow: /

User-agent: cohere-ai
Disallow: /

# ─────────────────────────────────────────────────────────────
# EVERYONE ELSE — allowed (social-share preview bots, misc crawlers, etc.)
# ─────────────────────────────────────────────────────────────
User-agent: *
Allow: /

Sitemap: https://vincentxeus.com/sitemap.xml