# robots.txt for Seerror - Advanced SEO Configuration
# Last Updated: 2025-01-27
# Website: https://www.seerror.com

# ============================================
# ALLOWED USER AGENTS
# ============================================

# Google Bot - Full Access
User-agent: Googlebot
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /dashboard.html
Disallow: /profile.html
Disallow: /verify-email.html
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/
Disallow: /__pycache__/
Disallow: /*.log$
Disallow: /*.json$
Disallow: /api/
Crawl-delay: 1

# Google Image Bot
User-agent: Googlebot-Image
Allow: /seerror.png
Allow: /favicon.png
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/

# Google Mobile Bot
User-agent: Googlebot-Mobile
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/

# Bing Bot
User-agent: Bingbot
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/
Disallow: /*.log$
Disallow: /*.json$
Disallow: /api/
Crawl-delay: 2

# Yandex Bot
User-agent: Yandex
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/
Disallow: /*.log$
Disallow: /*.json$
Disallow: /api/
Crawl-delay: 3

# Baidu Bot
User-agent: Baiduspider
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/
Disallow: /*.log$
Disallow: /*.json$
Disallow: /api/
Crawl-delay: 5

# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/
Disallow: /*.log$
Disallow: /*.json$
Disallow: /api/

# Facebook External Hit (for link previews)
User-agent: facebookexternalhit
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/

# Twitter Bot
User-agent: Twitterbot
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/

# LinkedIn Bot
User-agent: LinkedInBot
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/

# WhatsApp Bot
User-agent: WhatsApp
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/

# Slack Bot
User-agent: Slackbot
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/

# ============================================
# DEFAULT RULES (All Other Bots)
# ============================================

User-agent: *
Allow: /
Disallow: /admin.html
Disallow: /admin/
Disallow: /dashboard.html
Disallow: /profile.html
Disallow: /verify-email.html
Disallow: /backend/
Disallow: /analyses/
Disallow: /memory_index/
Disallow: /__pycache__/
Disallow: /*.log$
Disallow: /*.json$
Disallow: /api/
Disallow: /dumy.html
Disallow: /app.log
Crawl-delay: 5

# ============================================
# SPECIFIC PATH RESTRICTIONS
# ============================================

# Block admin dashboard from all crawlers
Disallow: /admin.html
Disallow: /admin/

# Block user-specific pages (should not be indexed)
Disallow: /dashboard.html
Disallow: /profile.html
Disallow: /verify-email.html

# Block backend code and API endpoints
Disallow: /backend/
Disallow: /api/

# Block analysis data files
Disallow: /analyses/
Disallow: /*.json$

# Block memory/index files
Disallow: /memory_index/
Disallow: /__pycache__/

# Block log files
Disallow: /*.log$
Disallow: /app.log

# Block test/dummy pages
Disallow: /dumy.html

# Block sensitive configuration files
Disallow: /*.env$
Disallow: /*.env.local$
Disallow: /firestore.rules
Disallow: /requirements.txt
Disallow: /README.md
Disallow: /*.md$

# ============================================
# ALLOW SPECIFIC IMPORTANT FILES
# ============================================

# Allow important images
Allow: /seerror.png
Allow: /favicon.png
Allow: /*.jpg$
Allow: /*.jpeg$
Allow: /*.png$
Allow: /*.gif$
Allow: /*.webp$
Allow: /*.svg$

# Allow CSS and JS for proper rendering
Allow: /*.css$
Allow: /*.js$

# Allow HTML pages (except blocked ones)
Allow: /*.html$

# ============================================
# SITEMAP LOCATION
# ============================================

# Primary Sitemap
Sitemap: https://www.seerror.com/sitemap.xml

# Additional Sitemaps (if you have multiple)
# Sitemap: https://www.seerror.com/sitemap-images.xml
# Sitemap: https://www.seerror.com/sitemap-videos.xml
# Sitemap: https://www.seerror.com/sitemap-news.xml

# ============================================
# CRAWL DELAY SETTINGS
# ============================================

# Respectful crawling delays to prevent server overload
# Google: 1 second (default)
# Bing: 2 seconds
# Yandex: 3 seconds
# Baidu: 5 seconds
# Others: 5 seconds

# ============================================
# HOST DIRECTIVE (Optional but Recommended)
# ============================================

# Preferred domain (www or non-www)
# Using www version based on canonical URL in index.html
Host: https://www.seerror.com

# ============================================
# NOTES
# ============================================

# This robots.txt file:
# - Allows all major search engines to crawl public pages
# - Blocks admin dashboard and sensitive areas
# - Blocks backend code and API endpoints
# - Blocks log files and temporary data
# - Sets appropriate crawl delays
# - References sitemap.xml for better indexing
# - Allows important assets (images, CSS, JS)
# - Optimized for SEO and security

# For best results:
# 1. Submit sitemap.xml to Google Search Console
# 2. Submit sitemap.xml to Bing Webmaster Tools
# 3. Monitor crawl errors in search console
# 4. Update lastmod dates in sitemap.xml regularly
# 5. Keep robots.txt updated when adding new restricted areas