# Standard crawlers
User-agent: *
Allow: /
Disallow: /*?ref=
Disallow: /*?*&ref=
Disallow: /*?redirect_url=
Disallow: /*?*&redirect_url=
Disallow: /download-gate-form

# Block traditional search engines from LLM markdown mirror pages.
# These are lightweight markdown versions of our docs/blog intended for AI
# crawlers only. Blocking prevents duplicate content issues and crawl budget waste.
# NOTE: Deploy this block only after the X-Robots-Tag noindex headers (in the
# /docs-markdown/ and /blog-markdown/ route handlers) have been live for ~2 weeks.
# If Googlebot is blocked before it can re-crawl and see the noindex directive,
# affected URLs will remain stuck in GSC "Crawled - currently not indexed".
User-agent: Googlebot
Disallow: /docs-markdown/
Disallow: /blog-markdown/
Disallow: /resources/access/

User-agent: Bingbot
Disallow: /docs-markdown/
Disallow: /blog-markdown/
Disallow: /resources/access/

# AI Crawlers
# Inngest welcomes AI crawlers to index our documentation.
# See also: /llms.txt, /llms-full.txt, /ai.txt

User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: PerplexityBot
User-agent: Amazonbot
User-agent: CCBot
Allow: /docs/
Allow: /blog/
Allow: /patterns/
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /docs-markdown/
Allow: /blog-markdown/

User-agent: Google-Extended
Allow: /

User-agent: Bytespider
Disallow: /

# Host
Host: https://www.inngest.com

# Sitemaps
Sitemap: https://www.inngest.com/sitemap.xml