# Disallow AI training bots
User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: meta-externalagent
Disallow: /

# Allow search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Allow social media bots to access images for rich embeds
User-agent: Twitterbot
Allow: /
Allow: /img/

User-agent: discordbot
Allow: /
Allow: /img/

User-agent: WhatsApp
Allow: /
Allow: /img/

User-agent: facebookexternalhit
Allow: /
Allow: /img/

User-agent: TelegramBot
Allow: /
Allow: /img/

User-agent: Slackbot
Allow: /
Allow: /img/

# Default rules for all other bots
User-agent: *
Allow: /
Disallow: /shared-data/
Disallow: /documentation/
Disallow: /css/
Disallow: /js/

# Sitemap location
Sitemap: https://midloop.net/sitemap.xml

# Crawl delay
Crawl-delay: 1