User-agent: facebookexternalhit
Disallow: /api/
Allow: /

User-agent: Googlebot
Allow: /
User-agent: Mediapartners-Google
Allow: /
User-agent: Googlebot-Image
Allow: /

User-agent: Yahoo-MMCrawler
Allow: /
User-agent: Yahoo-slurp
Allow: /
User-agent: slurp
Allow: /

User-agent: Msnbot
Allow: /
User-agent: MSNBot-media
Allow: /
User-agent: Bing Preview bot
Allow: /

User-agent: Twitterbot
Allow: /

User-agent: dotbot
Allow: /

User-agent: AhrefsSiteAudit
Allow: /

User-agent: RSiteAuditor
Allow: /
    
User-agent: MediavineMetadataParser
Allow: /    

User-agent: MediavineMetadataParser/*
Allow: /    

User-agent: *
Disallow: /

User-agent: ChatGPT-User
Disallow: /api/
Allow: /

User-agent: OAI-SearchBot
Disallow: /api/
Allow: /

User-agent: Claude-User
Disallow: /api/
Allow: /

User-agent: Claude-SearchBot
Disallow: /api/
Allow: /

User-agent: PerplexityBot
Disallow: /api/
Allow: /

User-agent: Perplexity-User
Disallow: /api/
Allow: /

User-agent: DuckAssistBot
Disallow: /api/
Allow: /

User-agent: Meta-ExternalFetcher
Disallow: /api/
Allow: /

User-agent: MistralAI-User
Disallow: /api/
Allow: /

User-agent: Applebot
Disallow: /api/
Allow: /

User-agent: Applebot-Extended
Disallow: /api/
Allow: /

User-agent: GoogleOther
Disallow: /api/
Allow: /

User-agent: Google-Extended
Disallow: /api/
Allow: /