[E] Add robots.txt to client

ManifoldScholar · Aug 20, 2024 · ceb0371 · ceb0371
1 parent 67fde66
commit ceb0371
Showing 1 changed file with 38 additions and 0 deletions.
diff --git a/client/static/robots.txt b/client/static/robots.txt
@@ -0,0 +1,38 @@
+User-agent: GPTBot
+Disallow: /
+User-agent: ChatGPT-User
+Disallow: /
+User-agent: Google-Extended
+Disallow: /
+User-agent: PerplexityBot
+Disallow: /
+User-agent: Amazonbot
+Disallow: /
+User-agent: ClaudeBot
+Disallow: /
+User-agent: Omgilibot
+Disallow: /
+User-Agent: FacebookBot
+Disallow: /
+User-agent: Applebot
+Disallow: /
+User-agent: anthropic-ai
+Disallow: /
+User-agent: Bytespider
+Disallow: /
+User-agent: Claude-Web
+Disallow: /
+User-agent: Diffbot
+Disallow: /
+User-agent: ImagesiftBot
+Disallow: /
+User-agent: Omgilibot
+Disallow: /
+User-agent: Omgili
+Disallow: /
+User-agent: YouBot
+Disallow: /
+User-agent: CCBot
+Disallow: /
+User-agent: Meta-ExternalAgent
+Disallow: /