# Search engines and AI answer engines are welcome.
# Ahmedabad home buyers increasingly ask ChatGPT / Gemini / Claude / Perplexity
# for developer recommendations, so those crawlers are explicitly allowed - they
# are a discovery channel, not a threat. Bulk scrapers that send no traffic back
# are blocked further down.

User-agent: *
Allow: /
Disallow: /admin
Disallow: /preview

# --- AI answer engines: allowed ---------------------------------------------
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: OAI-SearchBot
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: ClaudeBot
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: Claude-SearchBot
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: Claude-User
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: Perplexity-User
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /preview

User-agent: meta-externalagent
Allow: /
Disallow: /admin
Disallow: /preview

# --- Bulk scrapers: blocked -------------------------------------------------
# These harvest at volume and send no referral traffic back.

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: Omgilibot
Disallow: /

# llms.txt (llmstxt.org) - a curated, plain-text map of the site for AI
# assistants, alongside this file for crawlers. No robots.txt directive for
# it exists yet; tools that support the convention just fetch it directly.
# https://resset.in/llms.txt

Sitemap: https://resset.in/sitemap.xml

# TEMPORARY - remove this line together with the route that serves it.
# sitemap-legacy.xml lists the previous static site's URLs, all of which now 301.
# It exists only to get those redirects crawled sooner than Google would reach them
# on its own (/about.html had gone 4 days without a recrawl when this was written).
# Listing it here rather than relying on a Search Console submission means it works
# without anyone logging in - but it also means removal is a two-line job: this line
# and src/routes/sitemap-legacy[.]xml.ts. Do it once the Pages report shows the old
# URLs as "Page with redirect" and their impressions have moved to /projects/*.
Sitemap: https://resset.in/sitemap-legacy.xml