# CryptoBull Insights Hub - Robots.txt
# Last updated: 2025-05-30

User-agent: *
Allow: /

# Allow crawling of CSS and JS for better rendering
Allow: /src/
Allow: /*.css$
Allow: /*.js$

# Block unnecessary paths that don't add SEO value
Disallow: /api/
Disallow: /admin/
Disallow: /*?*
Disallow: /search?
Disallow: /*.json$

# Sitemaps
Sitemap: https://insights.cryptobull.org/sitemap.xml

# Google Bots
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /
Allow: /lovable-uploads/

# Bing Bots
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: BingPreview
Allow: /

# Social Media Bots
User-agent: Twitterbot
Allow: /

User-agent: facebookexternalhit
Allow: /

User-agent: LinkedInBot
Allow: /

# Crawl Delay (in seconds)
# Helps manage server load from crawler requests
Crawl-delay: 1