# TwoWolves.ai — robots.txt
# AI-first business directory

# Google — blokuojame AI duomenų failus (crawl budget taupymas)
User-agent: Googlebot
Allow: /biz/
Disallow: /biz/*/llms.txt
Disallow: /biz/*/ai.json
Disallow: /biz/*/ai.txt
Disallow: /biz/*/AI-Profile.txt
Disallow: /biz/*/faq-ai.txt
Disallow: /biz/*/products.json
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/

User-agent: *
Disallow: /api/
Disallow: /dashboard/
Disallow: /admin/
Allow: /api/mcp
Allow: /api/public/
Allow: /biz/
Allow: /registry
Allow: /llms.txt
Allow: /

# ── AI Crawlers (explicitly allowed) ──

# OpenAI GPTBot
User-agent: GPTBot
Allow: /
Allow: /biz/
Allow: /llms.txt

# Google Gemini / Bard
User-agent: Google-Extended
Allow: /
Allow: /biz/

# Anthropic ClaudeBot
User-agent: ClaudeBot
Allow: /
Allow: /biz/

# Perplexity
User-agent: PerplexityBot
Allow: /
Allow: /biz/

# Common Crawl (training data)
User-agent: CCBot
Allow: /
Allow: /biz/

# Meta AI
User-agent: Meta-ExternalAgent
Allow: /
Allow: /biz/

# Apple Applebot
User-agent: Applebot
Allow: /

# You.com YouBot
User-agent: YouBot
Allow: /

# DuckDuckGo AI
User-agent: DuckAssistBot
Allow: /

# Diffbot (knowledge graph)
User-agent: Diffbot
Allow: /

# Microsoft Copilot
User-agent: Bingbot
Allow: /
Allow: /biz/

# ── Sitemaps ──
Sitemap: https://www.twowolves.ai/sitemap.xml