# Speakar AI - Robots.txt
# https://www.speakar.ai/robots.txt
# Last Updated: 2026-03-07

# ==============================================
# GENERAL CRAWLERS
# ==============================================
User-agent: *

# Allow all public pages
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt

# Important pages to crawl (high priority)
Allow: /industries
Allow: /industries/*
Allow: /blog
Allow: /blog/*
Allow: /ai-answering-service
Allow: /roi-calculator
Allow: /compare/*
Allow: /call-forwarding
Allow: /call-forwarding/*
Allow: /pricing
Allow: /about

# Disallow admin and dashboard areas
Disallow: /admin
Disallow: /admin/*
Disallow: /dashboard
Disallow: /dashboard/*
Disallow: /login
Disallow: /unauthorized
Disallow: /super-admin

# Disallow API endpoints (except public ones)
Disallow: /api/
Allow: /api/pl/*

# Disallow internal/technical pages
Disallow: /checkout/
Disallow: /callback
Disallow: /oauth
Disallow: /pay/
Disallow: /unsubscribe
Disallow: /pos/
Disallow: /*?page=
Disallow: /*?sort=
Disallow: /*?filter=
Disallow: /*.json
Allow: /manifest.json

# Allow feature pages
Allow: /features/
Allow: /features/*

# Crawl-delay (helps prevent server overload)
Crawl-delay: 1

# ==============================================
# GOOGLE-SPECIFIC RULES
# ==============================================
User-agent: Googlebot
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*

# No crawl delay for Google (they handle it automatically)

User-agent: Google-Extended
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*

User-agent: Googlebot-Image
Allow: /logos/
Allow: /images/
Disallow: /admin

User-agent: Googlebot-Video
Allow: /
Disallow: /admin

User-agent: Googlebot-News
Allow: /blog/
Allow: /blog/*
Disallow: /admin

# ==============================================
# BING-SPECIFIC RULES
# ==============================================
User-agent: Bingbot
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*
Crawl-delay: 1

User-agent: MSNBot
Allow: /
Disallow: /admin
Disallow: /api/
Crawl-delay: 1

# ==============================================
# OPENAI CRAWLERS
# ==============================================
User-agent: OAI-SearchBot
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*

User-agent: GPTBot
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*

User-agent: ChatGPT-User
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*

# ==============================================
# PERPLEXITY AI CRAWLERS
# ==============================================
User-agent: PerplexityBot
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*
Crawl-delay: 1

# ==============================================
# ANTHROPIC / CLAUDE AI CRAWLERS
# ==============================================
User-agent: ClaudeBot
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*
Crawl-delay: 1

User-agent: anthropic-ai
Allow: /
Allow: /robots.txt
Allow: /sitemap.xml
Allow: /llms.txt
Allow: /llms-full.txt
Disallow: /admin
Disallow: /dashboard
Disallow: /api/
Allow: /api/pl/*
Crawl-delay: 1

# ==============================================
# SOCIAL MEDIA CRAWLERS
# ==============================================
User-agent: Twitterbot
Allow: /
Disallow: /admin
Disallow: /api/

User-agent: facebookexternalhit
Allow: /
Disallow: /admin
Disallow: /api/

User-agent: LinkedInBot
Allow: /
Disallow: /admin
Disallow: /api/

# ==============================================
# SEO & ANALYTICS CRAWLERS
# ==============================================
User-agent: Screaming Frog SEO Spider
Allow: /
Disallow: /admin
Disallow: /api/

User-agent: AhrefsBot
Allow: /
Disallow: /admin
Disallow: /api/
Crawl-delay: 2

User-agent: SemrushBot
Allow: /
Disallow: /admin
Disallow: /api/
Crawl-delay: 2

User-agent: DotBot
Allow: /
Disallow: /admin
Disallow: /api/
Crawl-delay: 5

# ==============================================
# BLOCK UNWANTED BOTS
# ==============================================
User-agent: MJ12bot
Disallow: /

User-agent: SurveyBot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: YandexBot
Disallow: /

# ==============================================
# SITEMAPS
# ==============================================
Sitemap: https://www.speakar.ai/sitemap.xml