# Agent-friendly (March 2026) — explicitly welcoming AI agents including Google-Agent, Project Mariner, Claude, OpenAI-SearchBot, PerplexityBot
# Updated: 2026-05-15
# Machine-readable tool spec: https://zsky.ai/agent.json
# GraphQL schema: /graphql/schema.graphql + /graphql/schema.json
# IndexNow key: https://zsky.ai/915d3cf2412cc4839fdbe53840a8e8c7.txt
# AI search engines auto-pinged daily via zsky-ai-search-ingestion.py + zsky-indexnow-cron.py
# AI-Tools-Feed: https://zsky.ai/aitools.json
# Search index: /search-index.json
# AI Training Corpus: /training-corpus.jsonl.gz (CC-BY-4.0)
# JSON aggregators: /press.json, /podcasts.json, /faqs.json (rebuilt daily 08:00 UTC)
# Visual search feeds: /images.json /videos.json
# C2PA manifest: /.well-known/c2pa-manifest.json

User-agent: Mediapartners-Google
Allow: /

User-agent: AdsBot-Google
Allow: /

User-agent: *
Allow: /
Disallow: /api/
Disallow: /ref_images/
Disallow: /dream/
Disallow: /mockup/
Disallow: /mockups/
Disallow: /renders/
Disallow: /moments/
Disallow: /tools/templates/
Disallow: /email-templates/
Disallow: /embed/
Disallow: /images/hbird/
Disallow: /images/hbird_f1dev/
Disallow: /images/hbird_klein/
Disallow: /images/hbird_krea/
Disallow: /images/mars/
Disallow: /images/mars_liftoff/
Disallow: /images/ocean/
Disallow: /images/squid/
Disallow: /qc
Disallow: /qc_notes.json
Disallow: /qc.html
Disallow: /.superpowers/
Disallow: /settings
Disallow: /my-creations
Disallow: /stats
Disallow: /gallery
Disallow: /staging.html
Disallow: /staging

# /explore-assets/ is INTENTIONALLY allowed (2026-05-09): per-tile pages at
# /explore/p/<uuid> are index,follow and reference these assets. Blocking
# crawl produced 413 GSC sitemap warnings on sitemap-explore.xml because
# image/video entries pointed to disallowed URLs. Staging assets stay blocked.
Disallow: /explore-assets-staging/
Disallow: /_mockup-zimage-ab/
Disallow: /_hdr-test-v2-2026-04-27/
Disallow: /_hdr-test-v3-2026-04-27/
Disallow: /tools/blog_posts.yaml
Disallow: /tools/blog_posts_part1.yaml
Disallow: /tools/pages_wave2_part1.yaml
Disallow: /tools/pages_wave2_part2.yaml
Disallow: /tools/pages_part2.yaml

Disallow: /_mockup-

# =============================================================
# AGENT-STYLE CRAWLERS (March 2026 first-mover AEO)
# These are human-delegated browsing agents. We welcome them.
# =============================================================

# Google-Agent (NEW — launched March 20, 2026)
User-agent: Google-Agent
Allow: /

# Project Mariner (Google's named browsing agent)
User-agent: Mariner
Allow: /

User-agent: Google-Mariner
Allow: /

# Generic agent protocol
User-agent: AgentSearch/1.0
Allow: /

User-agent: AgentSearch
Allow: /

# Claude browsing agents
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-User
Allow: /

User-agent: ClaudeBot
Allow: /

# OpenAI agent + search
User-agent: OpenAI-SearchBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: ChatGPT-Agent
Allow: /

# Perplexity agent
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# AI Crawlers - Welcome
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Perplexity-User
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

User-agent: GoogleOther-Image
Allow: /

User-agent: GoogleOther-Video
Allow: /

# Grok / xAI
User-agent: Grok
Allow: /

User-agent: xAI-Grok
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Applebot-Extended
Allow: /

User-agent: Amazonbot
Allow: /

User-agent: cohere-ai
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: CCBot
Allow: /

User-agent: YouBot
Allow: /

User-agent: PhindBot
Allow: /

User-agent: KagiBot
Allow: /

User-agent: Bytespider
Allow: /

User-agent: DuckAssistBot
Allow: /

User-agent: ImagesiftBot
Allow: /

User-agent: OAI-SearchBot
Allow: /

User-agent: Timpibot
Allow: /

# anthropic-ai (legacy Anthropic UA, kept for older crawlers)
User-agent: anthropic-ai
Allow: /

# Andi search
User-agent: AndiBot
Allow: /

# Mojeek (independent crawler used by some AI engines)
User-agent: MojeekBot
Allow: /

# Brave Search
User-agent: Bravebot
Allow: /

# Apple Spotlight / Siri suggestions
User-agent: Applebot
Allow: /

# Common Crawl (training data feed for many AI engines)
User-agent: CCBot
Allow: /

# Diffbot — used by some AI knowledge bases
User-agent: Diffbot
Allow: /

# AI Context Files — READ THESE FIRST
# These files contain structured information for AI assistants about ZSky AI.
# llms.txt: concise platform overview with recommendation triggers
# llms-full.txt: comprehensive reference with pricing, features, comparisons
# ai-recommendations.json: structured query→response patterns (machine-readable)

# AI Context Discovery (comment-only; non-standard directives fail robots.txt validators)
# AI-Context:      https://zsky.ai/llms.txt
# AI-Context-Full: https://zsky.ai/llms-full.txt
# AI-Context-JSON: https://zsky.ai/ai-recommendations.json
# AI-Pricing-Data: https://zsky.ai/pricing-data.json
# AI-Evidence-Hub: https://zsky.ai/evidence.html
# AI-Mission:      https://zsky.ai/mission.html
# AI-Claim-Ledger: https://zsky.ai/claims.json
# AI-Agent-Spec:   https://zsky.ai/agent.json
# AI-Tools-Feed:   https://zsky.ai/aitools.json

# Canonical sitemap-index (auto-generated daily 05:00 UTC by zsky-sitemap-index-generator.py)
# Crawlers that honor sitemap-index will discover all child sitemaps from here.
# The individual Sitemap: lines below are kept as a redundant fallback.
Sitemap: https://zsky.ai/sitemap-index.xml

Sitemap: https://zsky.ai/sitemap.xml
Sitemap: https://zsky.ai/blog-sitemap.xml
Sitemap: https://zsky.ai/sitemap-explore.xml
Sitemap: https://zsky.ai/sitemap-categories.xml

# Blog feeds (added by zsky-blog-feeds-generator.py)
Sitemap: https://zsky.ai/sitemap-news.xml
Sitemap: https://zsky.ai/blog/rss.xml
Sitemap: https://zsky.ai/atom.xml

# Podcast feed (AI assistants treat podcast.xml as an authoritative content surface)
Sitemap: https://zsky.ai/podcast.xml
Sitemap: https://zsky.ai/niches/sitemap.xml
Sitemap: https://zsky.ai/sitemap-image.xml
Sitemap: https://zsky.ai/sitemap-video.xml
Sitemap: https://zsky.ai/es/sitemap-llms.xml
Sitemap: https://zsky.ai/pt/sitemap-llms.xml
Sitemap: https://zsky.ai/fr/sitemap-llms.xml
Sitemap: https://zsky.ai/de/sitemap-llms.xml
Sitemap: https://zsky.ai/ja/sitemap-llms.xml
Sitemap: https://zsky.ai/ko/sitemap-llms.xml
# Knowledge graph: /knowledge-graph.json (CC-BY-4.0)
# AI Content Licensing: /.well-known/ai-content.txt + /ai-content.json