# MarktAlert.nl Robots.txt
# Generated on 2025-08-20T10:03:55.448Z

# Allow all crawlers by default
User-agent: *
Allow: /

# Disallow admin and user-specific pages
Disallow: /admin
Disallow: /dashboard
Disallow: /manage/
Disallow: /verify/
Disallow: /login/*
Disallow: /api/

# Specific crawler rules (optional)
# Google
User-agent: Googlebot
Allow: /
Crawl-delay: 0

# Bing
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Block bad bots (optional)
User-agent: SemrushBot
Disallow: /

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: DotBot
Disallow: /

# Sitemap location
Sitemap: https://marktalert.nl/sitemap.xml

# Host (optional, helps with canonical domain)
Host: https://marktalert.nl