fix: add robots.txt

openfoodfacts · Aug 14, 2023 · 10387ff · 10387ff
1 parent 3b8680b
commit 10387ff
Showing 1 changed file with 10 additions and 0 deletions.
diff --git a/robotoff/app/api.py b/robotoff/app/api.py
@@ -1674,6 +1674,15 @@ def on_get(self, req: falcon.Request, resp: falcon.Response):
         resp.media = response
 
 
+class RobotsTxtResource:
+    def on_get(self, req: falcon.Request, resp: falcon.Response):
+        # Disallow completely indexation: otherwise web crawlers send millions
+        # of requests to Robotoff (420k requests/day by Google alone)
+        resp.body = "User-agent: *\nDisallow: /\n"
+        resp.content_type = falcon.MEDIA_TEXT
+        resp.status = falcon.HTTP_200
+
+
 cors = CORS(
     allow_all_origins=True,
     allow_all_headers=True,
@@ -1736,3 +1745,4 @@ def on_get(self, req: falcon.Request, resp: falcon.Response):
 api.add_route("/api/v1/users/statistics/{username}", UserStatisticsResource())
 api.add_route("/api/v1/predictions", PredictionCollection())
 api.add_route("/api/v1/annotation/collection", LogoAnnotationCollection())
+api.add_route("/robots.txt", RobotsTxtResource())