# Robots.txt for achiya-automation.com
# אחיה אוטומציה - בוטים ואוטומציות לעסקים

# Allow all search engines
User-agent: *
Allow: /
Disallow: /_astro/

# Specific bot configurations
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: Slurp
Allow: /

User-agent: DuckDuckBot
Allow: /

# AI Search Bots — ALLOWED (these help users discover our services)

# OpenAI Search
User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Anthropic Search
User-agent: Claude-SearchBot
Allow: /

User-agent: Claude-User
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Apple (Siri/Spotlight)
User-agent: Applebot
Allow: /

# Other AI Search
User-agent: cohere-ai
Allow: /

User-agent: YouBot
Allow: /

User-agent: AI2Bot
Allow: /

# AI Training Bots — BLOCKED (scrape content for model training, no user benefit)
User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

# Google-Extended — ALLOWED (dual-purpose: controls BOTH Gemini training AND AIO inclusion)
# Blocking this kills AI Overviews + Gemini visibility. Unblocked 2026-04-16 for GEO.
User-agent: Google-Extended
Allow: /

User-agent: CCBot
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Sitemap & AI Resources
Sitemap: https://achiya-automation.com/sitemap-index.xml

# LLM-specific resources (see llmstxt.org)
# llms.txt: https://achiya-automation.com/.well-known/llms.txt
# llms-full.txt: https://achiya-automation.com/.well-known/llms-full.txt
# AI-optimized page: https://achiya-automation.com/for-ai/