# robots.txt for remacle.dev
# Prepared by: lab451.org
# Generated: 2026-09-24
# Spec: RFC 9309 (https://www.rfc-editor.org/rfc/rfc9309)

# ====================================================================
# DEFAULT POLICY — applies to any crawler not named below
# ====================================================================

User-agent: *
Allow: /

# Rendering assets must stay crawlable or Google will judge the
# page on a broken, unstyled render.
Allow: /*.css$
Allow: /*.js$
Allow: /*.svg$
Allow: /*.webp$
Allow: /*.woff2$

# System directories
Disallow: /cgi-bin/
Disallow: /vendor/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.svn/
Disallow: /.env
Disallow: /.htaccess
Disallow: /backup/
Disallow: /backups/
Disallow: /tmp/
Disallow: /temp/
Disallow: /cache/
Disallow: /logs/
Disallow: /private/

# Config and backup files
Disallow: /*.sql$
Disallow: /*.log$
Disallow: /*.bak$
Disallow: /*.old$
Disallow: /*.swp$
Disallow: /*.env$
Disallow: /*.ini$
Disallow: /*.conf$
Disallow: /*.yml$
Disallow: /*.yaml$

# Authentication and transactional paths
Disallow: /login
Disallow: /logout
Disallow: /signin
Disallow: /signout
Disallow: /register
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify
Disallow: /callback
Disallow: /admin/
Disallow: /account/
Disallow: /dashboard/
Disallow: /members/
Disallow: /cart
Disallow: /checkout
Disallow: /order-confirmation

# Tracking parameters and duplicate-URL generators
Disallow: /*?*utm_
Disallow: /*?*gclid=
Disallow: /*?*fbclid=
Disallow: /*?*msclkid=
Disallow: /*?*sessionid=
Disallow: /*?*sid=
Disallow: /*?*replytocom=
Disallow: /*?*print=
Disallow: /*?*share=

# Politeness for unnamed crawlers. Named crawlers below are
# trusted to self-throttle and are given no delay.
Crawl-delay: 10

# ====================================================================
# TRUSTED SEARCH ENGINES — full access, no crawl delay
# ====================================================================

User-agent: Googlebot   # Google Search
User-agent: Googlebot-Image   # Google Images
User-agent: Googlebot-News   # Google News
User-agent: Googlebot-Video   # Google Video
User-agent: Google-InspectionTool   # Search Console URL inspection
User-agent: Storebot-Google   # Google Shopping
User-agent: Bingbot   # Microsoft Bing
User-agent: Slurp   # Yahoo Search
User-agent: DuckDuckBot   # DuckDuckGo
User-agent: Applebot   # Apple Spotlight & Siri
User-agent: YandexBot   # Yandex
User-agent: Baiduspider   # Baidu
User-agent: Yeti   # Naver
User-agent: SeznamBot   # Seznam
User-agent: Qwantify   # Qwant
User-agent: MojeekBot   # Mojeek
Allow: /
Disallow: /cgi-bin/
Disallow: /.env
Disallow: /login
Disallow: /logout
Disallow: /signin
Disallow: /signout
Disallow: /register
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify
Disallow: /callback
Disallow: /admin/
Disallow: /account/
Disallow: /dashboard/
Disallow: /members/
Disallow: /cart
Disallow: /checkout
Disallow: /order-confirmation

# ====================================================================
# AI RETRIEVAL — bots that cite you in answers
# ====================================================================

# These fetch pages to answer a live user question and link back.
# Blocking them removes you from AI search results.

User-agent: OAI-SearchBot   # ChatGPT search index
User-agent: ChatGPT-User   # ChatGPT user-initiated browsing
User-agent: Claude-SearchBot   # Claude search index
User-agent: Claude-User   # Claude user-initiated browsing
User-agent: PerplexityBot   # Perplexity index
User-agent: Perplexity-User   # Perplexity user-initiated browsing
User-agent: DuckAssistBot   # DuckDuckGo AI assist
User-agent: MistralAI-User   # Le Chat user-initiated browsing
User-agent: YouBot   # You.com
User-agent: Meta-ExternalFetcher   # Meta AI user-initiated browsing
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /login
Disallow: /logout
Disallow: /signin
Disallow: /signout
Disallow: /register
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify
Disallow: /callback
Disallow: /admin/
Disallow: /account/
Disallow: /dashboard/
Disallow: /members/
Disallow: /cart
Disallow: /checkout
Disallow: /order-confirmation

# ====================================================================
# AI TRAINING — allowed, with system paths excluded
# ====================================================================

User-agent: GPTBot   # OpenAI
User-agent: ClaudeBot   # Anthropic
User-agent: anthropic-ai   # Anthropic (legacy agent)
User-agent: Google-Extended   # Google Gemini
User-agent: Applebot-Extended   # Apple Intelligence
User-agent: Meta-ExternalAgent   # Meta AI
User-agent: FacebookBot   # Meta language models
User-agent: Amazonbot   # Amazon Alexa & Rufus
User-agent: CCBot   # Common Crawl (feeds most open LLMs)
User-agent: cohere-ai   # Cohere
User-agent: Diffbot   # Diffbot knowledge graph
User-agent: AI2Bot   # Allen Institute for AI
User-agent: Timpibot   # Timpi
User-agent: Bytespider   # ByteDance / Doubao
User-agent: omgili   # Webz.io dataset
Allow: /
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /login
Disallow: /logout
Disallow: /signin
Disallow: /signout
Disallow: /register
Disallow: /signup
Disallow: /forgot-password
Disallow: /reset-password
Disallow: /verify
Disallow: /callback
Disallow: /admin/
Disallow: /account/
Disallow: /dashboard/
Disallow: /members/
Disallow: /cart
Disallow: /checkout
Disallow: /order-confirmation
Disallow: /cgi-bin/
Disallow: /vendor/
Disallow: /node_modules/
Disallow: /.git/
Disallow: /.svn/
Disallow: /.env
Disallow: /.htaccess
Disallow: /backup/
Disallow: /backups/
Disallow: /tmp/
Disallow: /temp/
Disallow: /cache/
Disallow: /logs/
Disallow: /private/

# ====================================================================
# LINK PREVIEW BOTS — required for share cards
# ====================================================================

# Blocking these breaks the preview image and title on every
# social and chat platform.

User-agent: Twitterbot
User-agent: facebookexternalhit
User-agent: LinkedInBot
User-agent: Slackbot-LinkExpanding
User-agent: Discordbot
User-agent: WhatsApp
User-agent: TelegramBot
User-agent: Pinterest
User-agent: redditbot
Allow: /

# ====================================================================
# BLOCKED — commercial scrapers, no traffic returned
# ====================================================================

# Backlink and lead-generation crawlers. They consume bandwidth,
# expose your link profile to competitors, and send no visitors.
# Remove any of these if you use that vendor to audit your site.

User-agent: AhrefsBot
User-agent: SemrushBot
User-agent: MJ12bot
User-agent: DotBot
User-agent: DataForSeoBot
User-agent: BLEXBot
User-agent: rogerbot
User-agent: Barkrowler
User-agent: SeekportBot
User-agent: serpstatbot
User-agent: ZoominfoBot
User-agent: magpie-crawler
User-agent: Screaming Frog SEO Spider
User-agent: MegaIndex
User-agent: LinkpadBot
User-agent: Nimbostratus-Bot
User-agent: VelenPublicWebCrawler
User-agent: SiteAuditBot
User-agent: trendictionbot
User-agent: PetalBot
Disallow: /

# ====================================================================
# SITEMAPS
# ====================================================================

Sitemap: https://remacle.dev/sitemap.xml

# AI discoverability files (not part of RFC 9309, listed for humans)
# https://remacle.dev/llms.txt
# https://remacle.dev/llms-full.txt