# ORCHESTRADER Robots.txt - Optimized for AI/LLM Crawlers & Search Engines
# Last updated: 2025-07-24

# === AI/LLM CRAWLERS (Priority Access) ===
# OpenAI GPT
User-agent: GPTBot
Allow: /
Crawl-delay: 1

# Anthropic Claude  
User-agent: ClaudeBot
Allow: /
Crawl-delay: 1

# Perplexity AI
User-agent: PerplexityBot
Allow: /
Crawl-delay: 1

User-agent: Perplexity-User
Allow: /
Crawl-delay: 1

# Google Gemini/Bard
User-agent: Google-Extended
Allow: /
Crawl-delay: 1

User-agent: GoogleBot
Allow: /
Crawl-delay: 1

# Microsoft Copilot/Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Crawl-delay: 1

User-agent: Meta-ExternalFetcher
Allow: /
Crawl-delay: 1

# === SOCIAL MEDIA CRAWLERS ===
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

User-agent: WhatsApp
Allow: /

User-agent: TelegramBot
Allow: /

# === SEARCH ENGINE CRAWLERS ===
User-agent: Yandex
Allow: /
Crawl-delay: 2

User-agent: Slurp
Allow: /
Crawl-delay: 2

User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

User-agent: Applebot
Allow: /
Crawl-delay: 1

# === AI RESEARCH CRAWLERS ===
User-agent: ChatGPT-User
Allow: /
Crawl-delay: 1

User-agent: CCBot
Allow: /
Crawl-delay: 2

User-agent: anthropic-ai
Allow: /
Crawl-delay: 1

User-agent: Claude-Web
Allow: /
Crawl-delay: 1

# === GENERAL CRAWLERS ===
User-agent: *
Allow: /
Crawl-delay: 3

# === SITEMAPS ===
Sitemap: https://orchestrader.com/sitemap.xml

# === AI-OPTIMIZED CONTENT ===
# LLM-friendly content for AI systems
# See: https://orchestrader.com/llms.txt