# AnySkill — public robots.txt
#
# PD-525 (2026-10-25) + PD-551 (2026-10-27) — search engines and AI search /
# answer engines are WELCOME on the public pages (owner decision "א": AnySkill
# is open to every AI agent). Each token below was checked against its
# operator's own documentation on 2026-10-27 (sources in PD-551). They share
# one group so the intent is explicit and no generic rule below can be read as
# excluding them; they carry the same private-route exclusions.
# Agents that act FOR a user use the API (https://any-skill.com/api/v1) and the
# MCP server (https://mcp.any-skill.com/mcp), not the crawler path.

# Search engines
User-agent: Googlebot
User-agent: Bingbot
User-agent: Applebot
User-agent: DuckAssistBot
User-agent: YouBot
User-agent: Amazonbot
User-agent: Amzn-SearchBot
User-agent: Amzn-User
# Google / Apple AI-use controls (tokens only — they do not crawl)
User-agent: Google-Extended
User-agent: Applebot-Extended
# OpenAI
User-agent: GPTBot
User-agent: OAI-SearchBot
User-agent: ChatGPT-User
# Perplexity
User-agent: PerplexityBot
User-agent: Perplexity-User
# Anthropic
User-agent: ClaudeBot
User-agent: Claude-SearchBot
User-agent: Claude-User
# Meta
User-agent: meta-externalagent
User-agent: meta-externalfetcher
User-agent: meta-webindexer
# Mistral
User-agent: MistralAI-User
User-agent: MistralAI-Index
Allow: /$
Allow: /services
Allow: /cities
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /connect-ai
Allow: /terms
Allow: /privacy
Allow: /support
Allow: /accessibility
Allow: /join
Disallow: /admin
Disallow: /api/
Disallow: /confirm/
Disallow: /chat
Disallow: /me
Disallow: /account
Disallow: /wallet
Disallow: /bookings
Disallow: /jobs
Disallow: /verify
Disallow: /kyc
Disallow: /__/

# Everyone else: public pages yes, admin / financial / private / API no.

User-agent: *

Allow: /$
Allow: /services
Allow: /cities
Allow: /llms.txt
Allow: /llms-full.txt
Allow: /connect-ai
Allow: /support
Allow: /terms
Allow: /privacy
Allow: /accessibility

Disallow: /admin
Disallow: /admin/
Disallow: /api/
Disallow: /confirm/
Disallow: /support/console
Disallow: /escrow
Disallow: /vault
Disallow: /wallet
Disallow: /me
Disallow: /me/
Disallow: /account
Disallow: /account/
Disallow: /chat
Disallow: /chat/
Disallow: /jobs
Disallow: /jobs/
Disallow: /bookings
Disallow: /bookings/
Disallow: /reviews/create
Disallow: /reviews/edit
Disallow: /verify
Disallow: /verify/
Disallow: /kyc
Disallow: /kyc/
Disallow: /login
Disallow: /signup

# Firebase Auth iframe / API artifacts (no user content)
Disallow: /__/
Disallow: /flutter_service_worker.js

# PD-551: no Crawl-delay. Google ignores it, but Bing and Yandex honour it, and
# at 10 s it capped them at ~8,600 URLs a day — too slow for the public pages.

Sitemap: https://any-skill.com/sitemap.xml