make robots.txt restrictive by default

CHANGELOG.md

@@ -31,6 +31,7 @@ All notable changes to this project will be documented in this file.
 - Add 'browser_versions.csv' to CSV export
 - Add `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES` env var which defaults to `100000` (100KB)
 - Add alternative SMTP adapter plausible/analytics#3654
+- Add restrictive `robots.txt` for self-hosted plausible/analytics#3905
 
 ### Removed
 - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions

lib/plausible_web/endpoint.ex

@@ -33,11 +33,20 @@ defmodule PlausibleWeb.Endpoint do
   plug(PlausibleWeb.Tracker)
   plug(PlausibleWeb.Favicon)
 
+  static_paths = ~w(css js images favicon.ico)
+
+  static_paths =
+    on_full_build do
+      static_paths
+    else
+      static_paths ++ ["robots.txt"]
+    end
+
   plug(Plug.Static,
     at: "/",
     from: :plausible,
     gzip: false,
-    only: ~w(css js images favicon.ico robots.txt)
+    only: static_paths
   )
 
   on_full_build do

priv/static/robots.txt

@@ -1,5 +1,2 @@
-# See http://www.robotstxt.org/robotstxt.html for documentation on how to use the robots.txt file
-#
-# To ban all spiders from the entire site uncomment the next two lines:
-# User-agent: *
-# Disallow: /
+User-agent: *
+Disallow: /