# ============================================================
# Bath Spa University — robots.txt
# Last reviewed: June 2026
# Policy:
#   ALLOW  — traditional search engines
#   ALLOW  — AI search & retrieval crawlers (citation/visibility)
#   ALLOW  — SEO & analytics tools in active use
#   ALLOW  — academic & archival bots
#   ALLOW  — OAI-AdsBot (ad landing page validation only)
#   DISALLOW — AI training crawlers
# ============================================================


# ============================================================
# TRADITIONAL SEARCH ENGINES — ALLOW
# Crawl-delay managed via Google Search Console for Googlebot.
# Delay retained here as a fallback signal only.
# ============================================================

User-agent: Googlebot
Crawl-delay: 0.5
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

User-agent: bingbot
Crawl-delay: 1
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

User-agent: Slurp
Crawl-delay: 1
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

User-agent: DuckDuckBot
Crawl-delay: 1
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/


# ============================================================
# TERMINALFOUR INTERNAL CRAWLER — ALLOW
# ============================================================

User-agent: Terminalfour Nutch Spider
Crawl-delay: 0.5
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/


# ============================================================
# AI SEARCH & RETRIEVAL CRAWLERS — ALLOW
# These surface Bath Spa content in AI-generated answers.
# Blocking these removes us from ChatGPT, Claude, Perplexity etc.
# ============================================================

# OpenAI — ChatGPT Search indexing
User-agent: OAI-SearchBot
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# OpenAI — user-initiated page fetch within ChatGPT
User-agent: ChatGPT-User
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# OpenAI — ad landing page validation only (not used for training)
User-agent: OAI-AdsBot
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# Anthropic — Claude Search indexing
User-agent: Claude-SearchBot
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# Anthropic — user-initiated fetch within Claude
User-agent: Claude-User
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# Perplexity — search retrieval and citations
# Note: Perplexity has a documented history of ignoring robots.txt.
# Enforce at WAF/Cloudflare level if a hard block is needed.
User-agent: PerplexityBot
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

User-agent: Perplexity-User
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# Google AI Overviews — retrieval (distinct from Google-Extended training)
User-agent: GoogleOther
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/


# ============================================================
# SEO & ANALYTICS TOOLS — ALLOW
# Tools actively used by Bath Spa web team.
# Blocking these would break site audits within these platforms.
# Higher crawl-delay to reduce cumulative server load.
# ============================================================

# SEMrush
User-agent: SemrushBot
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# Ahrefs
User-agent: AhrefsBot
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# Moz
User-agent: DotBot
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# Screaming Frog (no declared user-agent — uses desktop browser UA by default,
# so cannot be controlled via robots.txt. Manage via the tool's settings instead.)


# ============================================================
# ACADEMIC & ARCHIVAL BOTS — ALLOW
# ============================================================

# Internet Archive / Wayback Machine
User-agent: ia_archiver
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/

# Turnitin — academic plagiarism detection
User-agent: TurnitinBot
Crawl-delay: 2
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/


# ============================================================
# AI TRAINING CRAWLERS — DISALLOW
# These crawlers harvest content to train AI foundation models.
# Blocking does NOT affect search rankings or AI citation visibility.
# ============================================================

# OpenAI — GPT model training
User-agent: GPTBot
Disallow: /

# Anthropic — Claude model training
User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Google — Gemini AI training (does NOT affect Google Search or AI Overviews)
User-agent: Google-Extended
Disallow: /

# Common Crawl — open dataset ingested by many LLMs, no citation benefit
User-agent: CCBot
Disallow: /

# ByteDance / TikTok
User-agent: Bytespider
Disallow: /

# Apple Intelligence training
User-agent: Applebot-Extended
Disallow: /

# Meta AI training
User-agent: FacebookBot
Disallow: /

# xAI / Grok training
User-agent: Grok
Disallow: /

# Cohere AI training
User-agent: cohere-ai
Disallow: /

# Amazon AI training
User-agent: Amazonbot
Disallow: /

# DataForSeo — commercial data harvesting, no benefit
User-agent: DataForSeoBot
Disallow: /

# Majestic — link intelligence product, no benefit
User-agent: MJ12bot
Disallow: /

# PetalBot — Huawei search, negligible UK university audience value
# Review if international student strategy changes
User-agent: PetalBot
Disallow: /


# ============================================================
# DEFAULT — all other crawlers
# Preserves existing behaviour for anything not listed above.
# ============================================================

User-agent: *
Crawl-delay: 1
Disallow: /search/
Disallow: /blog/tags/
Disallow: /blog/categories/


# ============================================================
Sitemap: https://www.bathspa.ac.uk/sitemap-en.xml
# ============================================================