# THUGS(red) Gates of Valhalla
# Public pages are open to crawlers. Investigation URLs are not: they
# contain research data, and /lookup/ starts a live OSINT fan-out.

User-agent: *
Allow: /$
Allow: /about
Allow: /privacy
Allow: /sources
Allow: /help
Allow: /css/
Allow: /js/
Allow: /fonts/
Allow: /images/
Disallow: /api/
Disallow: /lookup/
Disallow: /investigation/
Disallow: /timeline
Disallow: /collector
Disallow: /vulnscan
Disallow: /forensics
Crawl-delay: 5

# Preview bots need the OG images and the pages they belong to.
User-agent: Twitterbot
Allow: /images/
Allow: /investigation/
Disallow: /api/
Disallow: /lookup/

User-agent: facebookexternalhit
Allow: /images/
Allow: /investigation/
Disallow: /api/
Disallow: /lookup/

User-agent: LinkedInBot
Allow: /images/
Allow: /investigation/
Disallow: /api/
Disallow: /lookup/

User-agent: Slackbot-LinkExpanding
Allow: /images/
Allow: /investigation/
Disallow: /api/
Disallow: /lookup/

# Training crawlers: this is a security research tool, not a corpus.
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Google-Extended
Disallow: /

Sitemap: https://valhalla.thugs.red/sitemap.xml