# Block harmful data scraping bots and competitive intelligence crawlers
User-agent: AwarioSmartBot
Disallow: /

# Block SEO/competitive analysis bots
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: SEMrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

# Block generic scraping frameworks
User-agent: Scrapy
Disallow: /

User-agent: Python-requests
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: curl
Disallow: /

User-agent: wget
Disallow: /

User-agent: HTTPie
Disallow: /

User-agent: axios
Disallow: /

User-agent: Go-http-client
Disallow: /

User-agent: okhttp
Disallow: /

User-agent: Java
Disallow: /

# Block AI training bots

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

# Block content copying bots
User-agent: ia_archiver
Disallow: /

User-agent: Wayback
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: SiteAuditBot
Disallow: /

User-agent: Linkbot
Disallow: /

User-agent: crawler
Disallow: /


User-agent: facebookexternalhit
Disallow: /profiles/
Disallow: /*/chat/
Disallow: /api/

# Block suspicious/generic bots
User-agent: *bot*
Disallow: /api/
Disallow: /*/chat/
Disallow: /profiles/edit/

User-agent: *spider*
Disallow: /api/
Disallow: /*/chat/
Disallow: /profiles/edit/

User-agent: *crawler*
Disallow: /api/
Disallow: /*/chat/
Disallow: /profiles/edit/

User-agent: *
Allow: /
Allow: /static/
Allow: /media/
Allow: /about/
Allow: /faq/
Allow: /privacy-policy/
Allow: /roadmap/
Allow: /blog/
Allow: /sitemap/
Allow: /robots.txt
Allow: /home/
Allow: /posts/
Allow: /biomarkers/
Allow: /supplements/
Allow: /effects/
Allow: /diets/
Allow: /fitness/

# Disallow sensitive areas
Disallow: /admin/
Disallow: /api/
Disallow: /htmx/

# Disallow all chat pages
Disallow: /*/chat/
Disallow: /effects/chat/
Disallow: /supplements/chat/
Disallow: /diets/chat/
Disallow: /fitness/chat/
Disallow: /biomarkers/chat/

# Disallow post creation pages (these are user-specific forms)
Disallow: /posts/create/
Disallow: /*/create/

# Disallow edit/delete pages
Disallow: /*/edit/
Disallow: /*/delete/

# Disallow user-specific profile pages
Disallow: /profiles/edit/
Disallow: /profiles/recommendations/
Disallow: /profiles/timeline/
Disallow: /profiles/onboarding/

# Disallow notifications (user-specific)
Disallow: /notifications/

# Disallow review pages (user-specific content)
Disallow: /review/

# Disallow search pages
Disallow: /search/

# Crawl delay (optional - adjust as needed)
Crawl-delay: 1

# Preferred domain - redirect www to non-www
# This helps search engines understand canonical preference
Host: staqc.com

# Sitemap
Sitemap: https://staqc.com/sitemap.xml