# -------------------------------------------------------------------
# GROUP 1: Polite Mainstream Search Engines
# Applies a crawl-delay to bots that respect it (like Bing, Yahoo, Yandex).
# Note: Googlebot ignores Crawl-delay. Adjust Google via Search Console.
# -------------------------------------------------------------------
User-agent: *
Crawl-delay: 10
# Disallow: /checkout/


# -------------------------------------------------------------------
# GROUP 2: AI Scrapers & Data Aggregators
# Completely blocks aggressive bots that drain your bandwidth for AI training.
# -------------------------------------------------------------------
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: ClaudeBot
User-agent: Bytespider
User-agent: CCBot
User-agent: PerplexityBot
Disallow: /

User-agent: SiteimproveBot
Allow: /

# -------------------------------------------------------------------
# SITEMAP LOCATION
# Always use the absolute URL with HTTPS to guide good search bots.
# -------------------------------------------------------------------
Sitemap: https://www.sunywcc.edu/sitemap/