robots.txt

Add default robots.txt that allows bots access to all paths.
Add mix task to generate robots.txt taht allows bots access to no paths.
Document custom emojis, MRF and static_dir
static_dir documentation includes docs for the robots.txt Mix task.

3 files changed, 35 insertions, 2 deletions

diff --git a/lib/mix/tasks/pleroma/robotstxt.ex b/lib/mix/tasks/pleroma/robotstxt.ex
new file mode 100644
index 000000000..2128e1cd6
--- /dev/null
+++ b/lib/mix/tasks/pleroma/robotstxt.ex
@@ -0,0 +1,32 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.RobotsTxt do
+  use Mix.Task
+
+  @shortdoc "Generate robots.txt"
+  @moduledoc """
+  Generates robots.txt
+
+  ## Overwrite robots.txt to disallow all
+
+      mix pleroma.robots_txt disallow_all
+
+  This will write a robots.txt that will hide all paths on your instance
+  from search engines and other robots that obey robots.txt
+
+  """
+  def run(["disallow_all"]) do
+    static_dir = Pleroma.Config.get([:instance, :static_dir], "instance/static/")
+
+    if !File.exists?(static_dir) do
+      File.mkdir_p!(static_dir)
+    end
+
+    robots_txt_path = Path.join(static_dir, "robots.txt")
+    robots_txt_content = "User-Agent: *\nDisallow: /\n"
+
+    File.write!(robots_txt_path, robots_txt_content, [:write])
+  end
+end
diff --git a/lib/pleroma/plugs/instance_static.ex b/lib/pleroma/plugs/instance_static.ex
index 41125921a..a64f1ea80 100644
--- a/lib/pleroma/plugs/instance_static.ex
+++ b/lib/pleroma/plugs/instance_static.ex
@@ -21,7 +21,8 @@ defmodule Pleroma.Plugs.InstanceStatic do
     end
   end
 
-  @only ~w(index.html static emoji packs sounds images instance favicon.png sw.js sw-pleroma.js)
+  @only ~w(index.html robots.txt static emoji packs sounds images instance favicon.png sw.js
+  sw-pleroma.js)
 
   def init(opts) do
     opts
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 697b1bc3a..fa2d1cbe7 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -25,7 +25,7 @@ defmodule Pleroma.Web.Endpoint do
     at: "/",
     from: :pleroma,
     only:
-      ~w(index.html static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
+      ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
     # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
   )