From f161a92cb1abd981e37367fcd5d315ac14510d12 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 23 Jan 2019 18:37:25 +0300
Subject: [#534] Initial implementation of unreachable federation targets
 retirement.

---
 lib/pleroma/instances.ex                     | 12 +++++
 lib/pleroma/instances/instance.ex            | 77 ++++++++++++++++++++++++++++
 lib/pleroma/web/activity_pub/activity_pub.ex | 39 ++++++++++----
 lib/pleroma/web/salmon/salmon.ex             | 13 ++++-
 lib/pleroma/web/websub/websub.ex             | 10 +++-
 5 files changed, 137 insertions(+), 14 deletions(-)
 create mode 100644 lib/pleroma/instances.ex
 create mode 100644 lib/pleroma/instances/instance.ex

(limited to 'lib')

diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
new file mode 100644
index 000000000..25b739520
--- /dev/null
+++ b/lib/pleroma/instances.ex
@@ -0,0 +1,12 @@
+defmodule Pleroma.Instances do
+  @moduledoc "Instances context."
+
+  @adapter Pleroma.Instances.Instance
+
+  defdelegate reachable?(url), to: @adapter
+  defdelegate set_reachable(url), to: @adapter
+  defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter
+
+  def reachability_time_threshold,
+    do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second)
+end
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
new file mode 100644
index 000000000..4507ef6d5
--- /dev/null
+++ b/lib/pleroma/instances/instance.ex
@@ -0,0 +1,77 @@
+defmodule Pleroma.Instances.Instance do
+  @moduledoc "Instance."
+
+  alias Pleroma.Instances
+  alias Pleroma.Instances.Instance
+
+  use Ecto.Schema
+
+  import Ecto.{Query, Changeset}
+
+  alias Pleroma.Repo
+
+  schema "instances" do
+    field(:host, :string)
+    field(:unreachable_since, :naive_datetime)
+    field(:reachability_checked_at, :naive_datetime)
+
+    timestamps()
+  end
+
+  def update_changeset(struct, params \\ %{}) do
+    struct
+    |> cast(params, [:host, :unreachable_since, :reachability_checked_at])
+    |> unique_constraint(:host)
+  end
+
+  def reachable?(url) do
+    !Repo.one(
+      from(i in Instance,
+        where:
+          i.host == ^host(url) and i.unreachable_since <= ^Instances.reachability_time_threshold(),
+        select: true
+      )
+    )
+  end
+
+  def set_reachable(url) do
+    Repo.update_all(
+      from(i in Instance, where: i.host == ^host(url)),
+      set: [
+        unreachable_since: nil,
+        reachability_checked_at: DateTime.utc_now()
+      ]
+    )
+  end
+
+  def set_unreachable(url, unreachable_since \\ nil) do
+    unreachable_since = unreachable_since || DateTime.utc_now()
+    host = host(url)
+    existing_record = Repo.get_by(Instance, %{host: host})
+
+    changes = %{
+      unreachable_since: unreachable_since,
+      reachability_checked_at: NaiveDateTime.utc_now()
+    }
+
+    if existing_record do
+      update_changes =
+        if existing_record.unreachable_since &&
+             NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt,
+           do: Map.delete(changes, :unreachable_since),
+           else: changes
+
+      {:ok, _instance} = Repo.update(update_changeset(existing_record, update_changes))
+    else
+      {:ok, _instance} = Repo.insert(update_changeset(%Instance{}, Map.put(changes, :host, host)))
+    end
+  end
+
+  defp host(url_or_host) do
+    if url_or_host =~ ~r/^http/i do
+      URI.parse(url_or_host).host
+    else
+      url_or_host
+    end
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 82fffd324..b14c91c18 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -3,7 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
-  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
+  alias Pleroma.{Activity, Repo, Object, Upload, User, Notification, Instances}
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
@@ -721,7 +721,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end)
   end
 
-  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
+  def publish_one(%{inbox: inbox} = activity) do
+    if Instances.reachable?(inbox) do
+      do_publish_one(activity)
+    else
+      {:error, :noop}
+    end
+  end
+
+  defp do_publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
     Logger.info("Federating #{id} to #{inbox}")
     host = URI.parse(inbox).host
 
@@ -734,15 +742,24 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         digest: digest
       })
 
-    @httpoison.post(
-      inbox,
-      json,
-      [
-        {"Content-Type", "application/activity+json"},
-        {"signature", signature},
-        {"digest", digest}
-      ]
-    )
+    with {:ok, _} <-
+           result =
+             @httpoison.post(
+               inbox,
+               json,
+               [
+                 {"Content-Type", "application/activity+json"},
+                 {"signature", signature},
+                 {"digest", digest}
+               ]
+             ) do
+      Instances.set_reachable(inbox)
+      result
+    else
+      e ->
+        Instances.set_unreachable(inbox)
+        e
+    end
   end
 
   # TODO:
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index e41657da1..0a0b91433 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.Salmon do
   @httpoison Application.get_env(:pleroma, :httpoison)
 
   use Bitwise
+  alias Pleroma.Instances
   alias Pleroma.Web.XML
   alias Pleroma.Web.OStatus.ActivityRepresenter
   alias Pleroma.User
@@ -167,15 +168,23 @@ defmodule Pleroma.Web.Salmon do
     do: send_to_user(salmon, feed, poster)
 
   defp send_to_user(url, feed, poster) when is_binary(url) do
-    with {:ok, %{status: code}} <-
+    with {:reachable, true} <- {:reachable, Instances.reachable?(url)},
+         {:ok, %{status: code}} <-
            poster.(
              url,
              feed,
              [{"Content-Type", "application/magic-envelope+xml"}]
            ) do
+      Instances.set_reachable(url)
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
     else
-      e -> Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+      {:reachable, false} ->
+        Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end)
+        :noop
+
+      e ->
+        Instances.set_unreachable(url)
+        Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
     end
   end
 
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index 7ca62c83b..a6bbaef37 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.Websub do
   alias Ecto.Changeset
   alias Pleroma.Repo
+  alias Pleroma.Instances
   alias Pleroma.Web.Websub.{WebsubServerSubscription, WebsubClientSubscription}
   alias Pleroma.Web.OStatus.FeedRepresenter
   alias Pleroma.Web.{XML, Endpoint, OStatus}
@@ -267,7 +268,8 @@ defmodule Pleroma.Web.Websub do
     signature = sign(secret || "", xml)
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
-    with {:ok, %{status: code}} <-
+    with {:reachable, true} <- {:reachable, Instances.reachable?(callback)},
+         {:ok, %{status: code}} <-
            @httpoison.post(
              callback,
              xml,
@@ -276,10 +278,16 @@ defmodule Pleroma.Web.Websub do
                {"X-Hub-Signature", "sha1=#{signature}"}
              ]
            ) do
+      Instances.set_reachable(callback)
       Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
       {:ok, code}
     else
+      {:reachable, false} ->
+        Logger.debug(fn -> "Pushing to #{callback} skipped as marked unreachable)" end)
+        {:error, :noop}
+
       e ->
+        Instances.set_unreachable(callback)
         Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
         {:error, e}
     end
-- 
cgit v1.2.3


From 20b54366ee916677b3865acf36baeeb062dd550b Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 24 Jan 2019 11:54:52 +0300
Subject: [#534] Federation publish requests status control (enforced 2xx
 response code check).

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
 lib/pleroma/web/salmon/salmon.ex             | 2 +-
 lib/pleroma/web/websub/websub.ex             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index b14c91c18..10155ff5a 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -742,7 +742,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         digest: digest
       })
 
-    with {:ok, _} <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            result =
              @httpoison.post(
                inbox,
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 0a0b91433..0423ccee0 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -169,7 +169,7 @@ defmodule Pleroma.Web.Salmon do
 
   defp send_to_user(url, feed, poster) when is_binary(url) do
     with {:reachable, true} <- {:reachable, Instances.reachable?(url)},
-         {:ok, %{status: code}} <-
+         {:ok, %{status: code}} when code in 200..299 <-
            poster.(
              url,
              feed,
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index a6bbaef37..9ceb5fbf7 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -269,7 +269,7 @@ defmodule Pleroma.Web.Websub do
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
     with {:reachable, true} <- {:reachable, Instances.reachable?(callback)},
-         {:ok, %{status: code}} <-
+         {:ok, %{status: code}} when code in 200..299 <-
            @httpoison.post(
              callback,
              xml,
-- 
cgit v1.2.3


From 8654a591f08c7d8d5d61f075906f0c6907e877bb Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 24 Jan 2019 17:37:23 +0300
Subject: [#534] Updating external instances reachability on incoming
 federation.

---
 lib/pleroma/instances/instance.ex                       | 14 +++++++++++---
 lib/pleroma/reverse_proxy.ex                            |  3 ++-
 lib/pleroma/web/activity_pub/activity_pub.ex            |  3 ++-
 lib/pleroma/web/activity_pub/activity_pub_controller.ex |  7 +++++++
 lib/pleroma/web/controller_helper.ex                    |  5 +++++
 lib/pleroma/web/ostatus/ostatus_controller.ex           |  7 +++++++
 lib/pleroma/web/salmon/salmon.ex                        |  5 ++++-
 lib/pleroma/web/websub/websub.ex                        |  3 ++-
 lib/pleroma/web/websub/websub_controller.ex             |  9 +++++++++
 9 files changed, 49 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 4507ef6d5..fe52331a3 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -24,7 +24,7 @@ defmodule Pleroma.Instances.Instance do
     |> unique_constraint(:host)
   end
 
-  def reachable?(url) do
+  def reachable?(url) when is_binary(url) do
     !Repo.one(
       from(i in Instance,
         where:
@@ -34,7 +34,9 @@ defmodule Pleroma.Instances.Instance do
     )
   end
 
-  def set_reachable(url) do
+  def reachable?(_), do: true
+
+  def set_reachable(url) when is_binary(url) do
     Repo.update_all(
       from(i in Instance, where: i.host == ^host(url)),
       set: [
@@ -44,7 +46,11 @@ defmodule Pleroma.Instances.Instance do
     )
   end
 
-  def set_unreachable(url, unreachable_since \\ nil) do
+  def set_reachable(_), do: {0, :noop}
+
+  def set_unreachable(url, unreachable_since \\ nil)
+
+  def set_unreachable(url, unreachable_since) when is_binary(url) do
     unreachable_since = unreachable_since || DateTime.utc_now()
     host = host(url)
     existing_record = Repo.get_by(Instance, %{host: host})
@@ -67,6 +73,8 @@ defmodule Pleroma.Instances.Instance do
     end
   end
 
+  def set_unreachable(_, _), do: {0, :noop}
+
   defp host(url_or_host) do
     if url_or_host =~ ~r/^http/i do
       URI.parse(url_or_host).host
diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex
index a25b5ea4e..d8b17212b 100644
--- a/lib/pleroma/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy.ex
@@ -3,7 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
-  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since if-none-match if-range range)
+  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since) ++
+                      ~w(if-none-match if-range range referer)
   @resp_cache_headers ~w(etag date last-modified cache-control)
   @keep_resp_headers @resp_cache_headers ++
                        ~w(content-type content-disposition content-encoding content-range accept-ranges vary)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 10155ff5a..44c295d65 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -750,7 +750,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
                [
                  {"Content-Type", "application/activity+json"},
                  {"signature", signature},
-                 {"digest", digest}
+                 {"digest", digest},
+                 {"referer", Pleroma.Web.Endpoint.url()}
                ]
              ) do
       Instances.set_reachable(inbox)
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 7eed0a600..dc353dff0 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Activity, User, Object}
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -18,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
   plug(:relay_active? when action in [:relay])
+  plug(:set_requester_reachable when action in [:inbox])
 
   def relay_active?(conn, _) do
     if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
@@ -289,4 +291,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> put_status(500)
     |> json("error")
   end
+
+  defp set_requester_reachable(conn, _) do
+    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
+    conn
+  end
 end
diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 14e3d19fd..13cf1877f 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -10,4 +10,9 @@ defmodule Pleroma.Web.ControllerHelper do
     |> put_status(status)
     |> json(json)
   end
+
+  def set_requester_reachable(conn) do
+    with [referer] <- get_req_header(conn, "referer"),
+         do: Pleroma.Instances.set_reachable(referer)
+  end
 end
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index ce022bcc1..a89f16b94 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -15,6 +15,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
+  plug(:set_requester_reachable when action in [:salmon_incoming])
+
   action_fallback(:errors)
 
   def feed_redirect(conn, %{"nickname" => nickname}) do
@@ -201,4 +203,9 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     |> put_status(500)
     |> text("Something went wrong")
   end
+
+  defp set_requester_reachable(conn, _) do
+    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
+    conn
+  end
 end
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 0423ccee0..e4d2d9517 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -173,7 +173,10 @@ defmodule Pleroma.Web.Salmon do
            poster.(
              url,
              feed,
-             [{"Content-Type", "application/magic-envelope+xml"}]
+             [
+               {"Content-Type", "application/magic-envelope+xml"},
+               {"referer", Pleroma.Web.Endpoint.url()}
+             ]
            ) do
       Instances.set_reachable(url)
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index 9ceb5fbf7..ac8903913 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -275,7 +275,8 @@ defmodule Pleroma.Web.Websub do
              xml,
              [
                {"Content-Type", "application/atom+xml"},
-               {"X-Hub-Signature", "sha1=#{signature}"}
+               {"X-Hub-Signature", "sha1=#{signature}"},
+               {"referer", Pleroma.Web.Endpoint.url()}
              ]
            ) do
       Instances.set_reachable(callback)
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
index e58f144e5..02fe075d7 100644
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ b/lib/pleroma/web/websub/websub_controller.ex
@@ -4,9 +4,11 @@
 
 defmodule Pleroma.Web.Websub.WebsubController do
   use Pleroma.Web, :controller
+
   alias Pleroma.{Repo, User}
   alias Pleroma.Web.{Websub, Federator}
   alias Pleroma.Web.Websub.WebsubClientSubscription
+
   require Logger
 
   plug(
@@ -18,6 +20,8 @@ defmodule Pleroma.Web.Websub.WebsubController do
          ]
   )
 
+  plug(:set_requester_reachable when action in [:websub_incoming])
+
   def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
     user = User.get_cached_by_nickname(nickname)
 
@@ -92,4 +96,9 @@ defmodule Pleroma.Web.Websub.WebsubController do
         |> send_resp(500, "Error")
     end
   end
+
+  defp set_requester_reachable(conn, _) do
+    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
+    conn
+  end
 end
-- 
cgit v1.2.3


From 3e9399ec0b498c0c9783ccb0fea9f682c8b9d0ca Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 24 Jan 2019 19:15:23 +0300
Subject: [#534] Optimized bulk publish ops to filter on reachability early.
 `Instance` refactoring.

---
 lib/pleroma/instances.ex                     |  9 +++++
 lib/pleroma/instances/instance.ex            | 57 +++++++++++++++++++---------
 lib/pleroma/web/activity_pub/activity_pub.ex |  5 ++-
 lib/pleroma/web/salmon/salmon.ex             |  8 +++-
 lib/pleroma/web/websub/websub.ex             | 18 +++++----
 5 files changed, 69 insertions(+), 28 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 25b739520..6d445d6b3 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -3,10 +3,19 @@ defmodule Pleroma.Instances do
 
   @adapter Pleroma.Instances.Instance
 
+  defdelegate filter_reachable(urls), to: @adapter
   defdelegate reachable?(url), to: @adapter
   defdelegate set_reachable(url), to: @adapter
   defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter
 
   def reachability_time_threshold,
     do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second)
+
+  def host(url_or_host) when is_binary(url_or_host) do
+    if url_or_host =~ ~r/^http/i do
+      URI.parse(url_or_host).host
+    else
+      url_or_host
+    end
+  end
 end
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index fe52331a3..a17c8dab1 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -18,12 +18,35 @@ defmodule Pleroma.Instances.Instance do
     timestamps()
   end
 
-  def update_changeset(struct, params \\ %{}) do
+  defdelegate host(url), to: Instances
+
+  def changeset(struct, params \\ %{}) do
     struct
     |> cast(params, [:host, :unreachable_since, :reachability_checked_at])
+    |> validate_required([:host])
     |> unique_constraint(:host)
   end
 
+  def filter_reachable([]), do: []
+
+  def filter_reachable(urls) when is_list(urls) do
+    hosts =
+      urls
+      |> Enum.map(&(&1 && host(&1)))
+      |> Enum.filter(&(to_string(&1) != ""))
+
+    unreachable_hosts =
+      Repo.all(
+        from(i in Instance,
+          where:
+            i.host in ^hosts and i.unreachable_since <= ^Instances.reachability_time_threshold(),
+          select: i.host
+        )
+      )
+
+    Enum.filter(urls, &(&1 && host(&1) not in unreachable_hosts))
+  end
+
   def reachable?(url) when is_binary(url) do
     !Repo.one(
       from(i in Instance,
@@ -37,13 +60,13 @@ defmodule Pleroma.Instances.Instance do
   def reachable?(_), do: true
 
   def set_reachable(url) when is_binary(url) do
-    Repo.update_all(
-      from(i in Instance, where: i.host == ^host(url)),
-      set: [
-        unreachable_since: nil,
-        reachability_checked_at: DateTime.utc_now()
-      ]
-    )
+    with host <- host(url),
+         %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
+      {:ok, _instance} =
+        existing_record
+        |> changeset(%{unreachable_since: nil, reachability_checked_at: DateTime.utc_now()})
+        |> Repo.update()
+    end
   end
 
   def set_reachable(_), do: {0, :noop}
@@ -67,19 +90,17 @@ defmodule Pleroma.Instances.Instance do
            do: Map.delete(changes, :unreachable_since),
            else: changes
 
-      {:ok, _instance} = Repo.update(update_changeset(existing_record, update_changes))
+      {:ok, _instance} =
+        existing_record
+        |> changeset(update_changes)
+        |> Repo.update()
     else
-      {:ok, _instance} = Repo.insert(update_changeset(%Instance{}, Map.put(changes, :host, host)))
+      {:ok, _instance} =
+        %Instance{}
+        |> changeset(Map.put(changes, :host, host))
+        |> Repo.insert()
     end
   end
 
   def set_unreachable(_, _), do: {0, :noop}
-
-  defp host(url_or_host) do
-    if url_or_host =~ ~r/^http/i do
-      URI.parse(url_or_host).host
-    else
-      url_or_host
-    end
-  end
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 44c295d65..4b34334a0 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -689,7 +689,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def publish(actor, activity) do
-    followers =
+    remote_followers =
       if actor.follower_address in activity.recipients do
         {:ok, followers} = User.get_followers(actor)
         followers |> Enum.filter(&(!&1.local))
@@ -700,13 +700,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     public = is_public?(activity)
 
     remote_inboxes =
-      (Pleroma.Web.Salmon.remote_users(activity) ++ followers)
+      (Pleroma.Web.Salmon.remote_users(activity) ++ remote_followers)
       |> Enum.filter(fn user -> User.ap_enabled?(user) end)
       |> Enum.map(fn %{info: %{source_data: data}} ->
         (is_map(data["endpoints"]) && Map.get(data["endpoints"], "sharedInbox")) || data["inbox"]
       end)
       |> Enum.uniq()
       |> Enum.filter(fn inbox -> should_federate?(inbox, public) end)
+      |> Instances.filter_reachable()
 
     {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
     json = Jason.encode!(data)
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index e4d2d9517..848131d52 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -221,7 +221,13 @@ defmodule Pleroma.Web.Salmon do
       {:ok, private, _} = keys_from_pem(keys)
       {:ok, feed} = encode(private, feed)
 
-      remote_users(activity)
+      remote_users = remote_users(activity)
+
+      salmon_urls = Enum.map(remote_users, & &1.info.salmon)
+      reachable_salmon_urls = Instances.filter_reachable(salmon_urls)
+
+      remote_users
+      |> Enum.filter(&(&1.info.salmon in reachable_salmon_urls))
       |> Enum.each(fn remote_user ->
         Task.start(fn ->
           Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index ac8903913..bb4442591 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -54,7 +54,12 @@ defmodule Pleroma.Web.Websub do
   ]
   def publish(topic, user, %{data: %{"type" => type}} = activity)
       when type in @supported_activities do
-    # TODO: Only send to still valid subscriptions.
+    response =
+      user
+      |> FeedRepresenter.to_simple_form([activity], [user])
+      |> :xmerl.export_simple(:xmerl_xml)
+      |> to_string
+
     query =
       from(
         sub in WebsubServerSubscription,
@@ -64,13 +69,12 @@ defmodule Pleroma.Web.Websub do
 
     subscriptions = Repo.all(query)
 
-    Enum.each(subscriptions, fn sub ->
-      response =
-        user
-        |> FeedRepresenter.to_simple_form([activity], [user])
-        |> :xmerl.export_simple(:xmerl_xml)
-        |> to_string
+    callbacks = Enum.map(subscriptions, & &1.callback)
+    reachable_callbacks = Instances.filter_reachable(callbacks)
 
+    subscriptions
+    |> Enum.filter(&(&1.callback in reachable_callbacks))
+    |> Enum.each(fn sub ->
       data = %{
         xml: response,
         topic: topic,
-- 
cgit v1.2.3


From 656ed7c84a5d8e423999457f66d8259ec8aa9a44 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 25 Jan 2019 15:10:21 +0300
Subject: [#534] Configurable outgoing federation reachability timeout.

---
 lib/pleroma/instances.ex          | 16 ++++++++++++++--
 lib/pleroma/instances/instance.ex |  6 ++++--
 2 files changed, 18 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 6d445d6b3..0b08f0eb8 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -8,8 +8,20 @@ defmodule Pleroma.Instances do
   defdelegate set_reachable(url), to: @adapter
   defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter
 
-  def reachability_time_threshold,
-    do: NaiveDateTime.add(NaiveDateTime.utc_now(), -30 * 24 * 3600, :second)
+  def reachability_datetime_threshold do
+    federation_reachability_timeout_days =
+      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 90
+
+    if federation_reachability_timeout_days > 0 do
+      NaiveDateTime.add(
+        NaiveDateTime.utc_now(),
+        -federation_reachability_timeout_days * 24 * 3600,
+        :second
+      )
+    else
+      ~N[0000-01-01 00:00:00]
+    end
+  end
 
   def host(url_or_host) when is_binary(url_or_host) do
     if url_or_host =~ ~r/^http/i do
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index a17c8dab1..60e8d0e21 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -39,7 +39,8 @@ defmodule Pleroma.Instances.Instance do
       Repo.all(
         from(i in Instance,
           where:
-            i.host in ^hosts and i.unreachable_since <= ^Instances.reachability_time_threshold(),
+            i.host in ^hosts and
+              i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
           select: i.host
         )
       )
@@ -51,7 +52,8 @@ defmodule Pleroma.Instances.Instance do
     !Repo.one(
       from(i in Instance,
         where:
-          i.host == ^host(url) and i.unreachable_since <= ^Instances.reachability_time_threshold(),
+          i.host == ^host(url) and
+            i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
         select: true
       )
     )
-- 
cgit v1.2.3


From 465adedb7cc457303278444d0f56960f87fde1e9 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 25 Jan 2019 18:29:43 +0300
Subject: [#534] Websub fix: made SQL use UTC time zone when comparing with
 `valid_until` (instead of postgresql-server default time zone).

---
 lib/pleroma/web/websub/websub.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index bb4442591..cbb7a5ac7 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -64,7 +64,7 @@ defmodule Pleroma.Web.Websub do
       from(
         sub in WebsubServerSubscription,
         where: sub.topic == ^topic and sub.state == "active",
-        where: fragment("? > NOW()", sub.valid_until)
+        where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until)
       )
 
     subscriptions = Repo.all(query)
-- 
cgit v1.2.3


From 060d280e64c201d3f8bec5615cc3b02cd460d3e1 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 25 Jan 2019 20:38:13 +0300
Subject: [#534] Made Salmon.send_to_user calls be handled through
 Federator.enqueue.

---
 lib/pleroma/web/federator/federator.ex |  6 +++++-
 lib/pleroma/web/salmon/salmon.ex       | 14 +++++++-------
 2 files changed, 12 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index f3a0e18b8..46f7a4973 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.Federator do
   use GenServer
   alias Pleroma.User
   alias Pleroma.Activity
-  alias Pleroma.Web.{WebFinger, Websub}
+  alias Pleroma.Web.{WebFinger, Websub, Salmon}
   alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -124,6 +124,10 @@ defmodule Pleroma.Web.Federator do
     end
   end
 
+  def handle(:publish_single_salmon, {user_or_url, feed, poster}) do
+    Salmon.send_to_user(user_or_url, feed, poster)
+  end
+
   def handle(:publish_single_ap, params) do
     case ActivityPub.publish_one(params) do
       {:ok, _} ->
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 848131d52..17ca7a6e8 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -164,10 +164,10 @@ defmodule Pleroma.Web.Salmon do
 
   # push an activity to remote accounts
   #
-  defp send_to_user(%{info: %{salmon: salmon}}, feed, poster),
+  def send_to_user(%{info: %{salmon: salmon}}, feed, poster),
     do: send_to_user(salmon, feed, poster)
 
-  defp send_to_user(url, feed, poster) when is_binary(url) do
+  def send_to_user(url, feed, poster) when is_binary(url) do
     with {:reachable, true} <- {:reachable, Instances.reachable?(url)},
          {:ok, %{status: code}} when code in 200..299 <-
            poster.(
@@ -180,6 +180,7 @@ defmodule Pleroma.Web.Salmon do
            ) do
       Instances.set_reachable(url)
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
+      :ok
     else
       {:reachable, false} ->
         Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end)
@@ -188,10 +189,11 @@ defmodule Pleroma.Web.Salmon do
       e ->
         Instances.set_unreachable(url)
         Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
+        :error
     end
   end
 
-  defp send_to_user(_, _, _), do: nil
+  def send_to_user(_, _, _), do: :noop
 
   @supported_activities [
     "Create",
@@ -229,10 +231,8 @@ defmodule Pleroma.Web.Salmon do
       remote_users
       |> Enum.filter(&(&1.info.salmon in reachable_salmon_urls))
       |> Enum.each(fn remote_user ->
-        Task.start(fn ->
-          Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
-          send_to_user(remote_user, feed, poster)
-        end)
+        Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
+        Pleroma.Web.Federator.enqueue(:publish_single_salmon, {remote_user, feed, poster})
       end)
     end
   end
-- 
cgit v1.2.3


From 9560abea102b8cd4927c9350bbd0a1a2f1800ea6 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Mon, 28 Jan 2019 11:03:52 +0300
Subject: [#534] Refactoring / tweaks per MR review.

---
 lib/pleroma/instances/instance.ex                  | 34 +++++++++-------------
 lib/pleroma/plugs/set_requester_reachable_plug.ex  | 16 ++++++++++
 lib/pleroma/web/activity_pub/activity_pub.ex       | 10 +------
 .../web/activity_pub/activity_pub_controller.ex    |  7 +----
 lib/pleroma/web/controller_helper.ex               |  5 ----
 lib/pleroma/web/ostatus/ostatus_controller.ex      |  7 +----
 lib/pleroma/web/salmon/salmon.ex                   |  3 +-
 lib/pleroma/web/websub/websub.ex                   |  3 +-
 lib/pleroma/web/websub/websub_controller.ex        |  7 +----
 9 files changed, 36 insertions(+), 56 deletions(-)
 create mode 100644 lib/pleroma/plugs/set_requester_reachable_plug.ex

(limited to 'lib')

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 60e8d0e21..e3af4a8a7 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -13,7 +13,6 @@ defmodule Pleroma.Instances.Instance do
   schema "instances" do
     field(:host, :string)
     field(:unreachable_since, :naive_datetime)
-    field(:reachability_checked_at, :naive_datetime)
 
     timestamps()
   end
@@ -22,7 +21,7 @@ defmodule Pleroma.Instances.Instance do
 
   def changeset(struct, params \\ %{}) do
     struct
-    |> cast(params, [:host, :unreachable_since, :reachability_checked_at])
+    |> cast(params, [:host, :unreachable_since])
     |> validate_required([:host])
     |> unique_constraint(:host)
   end
@@ -66,7 +65,7 @@ defmodule Pleroma.Instances.Instance do
          %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
       {:ok, _instance} =
         existing_record
-        |> changeset(%{unreachable_since: nil, reachability_checked_at: DateTime.utc_now()})
+        |> changeset(%{unreachable_since: nil})
         |> Repo.update()
     end
   end
@@ -80,27 +79,22 @@ defmodule Pleroma.Instances.Instance do
     host = host(url)
     existing_record = Repo.get_by(Instance, %{host: host})
 
-    changes = %{
-      unreachable_since: unreachable_since,
-      reachability_checked_at: NaiveDateTime.utc_now()
-    }
+    changes = %{unreachable_since: unreachable_since}
 
-    if existing_record do
-      update_changes =
-        if existing_record.unreachable_since &&
-             NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt,
-           do: Map.delete(changes, :unreachable_since),
-           else: changes
-
-      {:ok, _instance} =
-        existing_record
-        |> changeset(update_changes)
-        |> Repo.update()
-    else
-      {:ok, _instance} =
+    cond do
+      is_nil(existing_record) ->
         %Instance{}
         |> changeset(Map.put(changes, :host, host))
         |> Repo.insert()
+
+      existing_record.unreachable_since &&
+          NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
+        {:noop, existing_record}
+
+      true ->
+        existing_record
+        |> changeset(changes)
+        |> Repo.update()
     end
   end
 
diff --git a/lib/pleroma/plugs/set_requester_reachable_plug.ex b/lib/pleroma/plugs/set_requester_reachable_plug.ex
new file mode 100644
index 000000000..88551be70
--- /dev/null
+++ b/lib/pleroma/plugs/set_requester_reachable_plug.ex
@@ -0,0 +1,16 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.Plugs.SetRequesterReachablePlug do
+  import Plug.Conn
+
+  def init(_), do: []
+
+  def call(%Plug.Conn{} = conn, _) do
+    with [referer] <- get_req_header(conn, "referer"),
+         do: Pleroma.Instances.set_reachable(referer)
+
+    conn
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 4b34334a0..4232d6c0a 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -722,15 +722,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end)
   end
 
-  def publish_one(%{inbox: inbox} = activity) do
-    if Instances.reachable?(inbox) do
-      do_publish_one(activity)
-    else
-      {:error, :noop}
-    end
-  end
-
-  defp do_publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
+  def publish_one(%{inbox: inbox, json: json, actor: actor, id: id}) do
     Logger.info("Federating #{id} to #{inbox}")
     host = URI.parse(inbox).host
 
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index dc353dff0..fadb038a2 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -18,8 +18,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   action_fallback(:errors)
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
+  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:inbox])
   plug(:relay_active? when action in [:relay])
-  plug(:set_requester_reachable when action in [:inbox])
 
   def relay_active?(conn, _) do
     if Keyword.get(Application.get_env(:pleroma, :instance), :allow_relay) do
@@ -291,9 +291,4 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> put_status(500)
     |> json("error")
   end
-
-  defp set_requester_reachable(conn, _) do
-    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
-    conn
-  end
 end
diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index 13cf1877f..14e3d19fd 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -10,9 +10,4 @@ defmodule Pleroma.Web.ControllerHelper do
     |> put_status(status)
     |> json(json)
   end
-
-  def set_requester_reachable(conn) do
-    with [referer] <- get_req_header(conn, "referer"),
-         do: Pleroma.Instances.set_reachable(referer)
-  end
 end
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index a89f16b94..e483447ed 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -15,7 +15,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
-  plug(:set_requester_reachable when action in [:salmon_incoming])
+  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:salmon_incoming])
 
   action_fallback(:errors)
 
@@ -203,9 +203,4 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     |> put_status(500)
     |> text("Something went wrong")
   end
-
-  defp set_requester_reachable(conn, _) do
-    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
-    conn
-  end
 end
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 17ca7a6e8..80023127c 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -168,8 +168,7 @@ defmodule Pleroma.Web.Salmon do
     do: send_to_user(salmon, feed, poster)
 
   def send_to_user(url, feed, poster) when is_binary(url) do
-    with {:reachable, true} <- {:reachable, Instances.reachable?(url)},
-         {:ok, %{status: code}} when code in 200..299 <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            poster.(
              url,
              feed,
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index cbb7a5ac7..64eba7221 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -272,8 +272,7 @@ defmodule Pleroma.Web.Websub do
     signature = sign(secret || "", xml)
     Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
 
-    with {:reachable, true} <- {:reachable, Instances.reachable?(callback)},
-         {:ok, %{status: code}} when code in 200..299 <-
+    with {:ok, %{status: code}} when code in 200..299 <-
            @httpoison.post(
              callback,
              xml,
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
index 02fe075d7..9da7e70a1 100644
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ b/lib/pleroma/web/websub/websub_controller.ex
@@ -20,7 +20,7 @@ defmodule Pleroma.Web.Websub.WebsubController do
          ]
   )
 
-  plug(:set_requester_reachable when action in [:websub_incoming])
+  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:websub_incoming])
 
   def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
     user = User.get_cached_by_nickname(nickname)
@@ -96,9 +96,4 @@ defmodule Pleroma.Web.Websub.WebsubController do
         |> send_resp(500, "Error")
     end
   end
-
-  defp set_requester_reachable(conn, _) do
-    Pleroma.Web.ControllerHelper.set_requester_reachable(conn)
-    conn
-  end
 end
-- 
cgit v1.2.3


From 1d2f41642cfec5710055bcf8409778bb362beecb Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Mon, 28 Jan 2019 15:25:06 +0300
Subject: [#534] Various tweaks. Tests for Instances and Instance.

---
 lib/pleroma/instances.ex                     | 13 ++++++++-----
 lib/pleroma/instances/instance.ex            | 28 ++++++++++++++--------------
 lib/pleroma/web/activity_pub/activity_pub.ex |  4 ++--
 lib/pleroma/web/salmon/salmon.ex             |  4 ----
 lib/pleroma/web/websub/websub.ex             | 10 +++-------
 5 files changed, 27 insertions(+), 32 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex
index 0b08f0eb8..5e107f4c9 100644
--- a/lib/pleroma/instances.ex
+++ b/lib/pleroma/instances.ex
@@ -3,14 +3,17 @@ defmodule Pleroma.Instances do
 
   @adapter Pleroma.Instances.Instance
 
-  defdelegate filter_reachable(urls), to: @adapter
-  defdelegate reachable?(url), to: @adapter
-  defdelegate set_reachable(url), to: @adapter
-  defdelegate set_unreachable(url, unreachable_since \\ nil), to: @adapter
+  defdelegate filter_reachable(urls_or_hosts), to: @adapter
+  defdelegate reachable?(url_or_host), to: @adapter
+  defdelegate set_reachable(url_or_host), to: @adapter
+  defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: @adapter
+
+  def set_consistently_unreachable(url_or_host),
+    do: set_unreachable(url_or_host, reachability_datetime_threshold())
 
   def reachability_datetime_threshold do
     federation_reachability_timeout_days =
-      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 90
+      Pleroma.Config.get(:instance)[:federation_reachability_timeout_days] || 0
 
     if federation_reachability_timeout_days > 0 do
       NaiveDateTime.add(
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index e3af4a8a7..a87590d8b 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Instances.Instance do
     timestamps()
   end
 
-  defdelegate host(url), to: Instances
+  defdelegate host(url_or_host), to: Instances
 
   def changeset(struct, params \\ %{}) do
     struct
@@ -28,9 +28,9 @@ defmodule Pleroma.Instances.Instance do
 
   def filter_reachable([]), do: []
 
-  def filter_reachable(urls) when is_list(urls) do
+  def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
     hosts =
-      urls
+      urls_or_hosts
       |> Enum.map(&(&1 && host(&1)))
       |> Enum.filter(&(to_string(&1) != ""))
 
@@ -44,14 +44,14 @@ defmodule Pleroma.Instances.Instance do
         )
       )
 
-    Enum.filter(urls, &(&1 && host(&1) not in unreachable_hosts))
+    Enum.filter(urls_or_hosts, &(&1 && host(&1) not in unreachable_hosts))
   end
 
-  def reachable?(url) when is_binary(url) do
+  def reachable?(url_or_host) when is_binary(url_or_host) do
     !Repo.one(
       from(i in Instance,
         where:
-          i.host == ^host(url) and
+          i.host == ^host(url_or_host) and
             i.unreachable_since <= ^Instances.reachability_datetime_threshold(),
         select: true
       )
@@ -60,8 +60,8 @@ defmodule Pleroma.Instances.Instance do
 
   def reachable?(_), do: true
 
-  def set_reachable(url) when is_binary(url) do
-    with host <- host(url),
+  def set_reachable(url_or_host) when is_binary(url_or_host) do
+    with host <- host(url_or_host),
          %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do
       {:ok, _instance} =
         existing_record
@@ -70,13 +70,13 @@ defmodule Pleroma.Instances.Instance do
     end
   end
 
-  def set_reachable(_), do: {0, :noop}
+  def set_reachable(_), do: {:error, nil}
 
-  def set_unreachable(url, unreachable_since \\ nil)
+  def set_unreachable(url_or_host, unreachable_since \\ nil)
 
-  def set_unreachable(url, unreachable_since) when is_binary(url) do
+  def set_unreachable(url_or_host, unreachable_since) when is_binary(url_or_host) do
     unreachable_since = unreachable_since || DateTime.utc_now()
-    host = host(url)
+    host = host(url_or_host)
     existing_record = Repo.get_by(Instance, %{host: host})
 
     changes = %{unreachable_since: unreachable_since}
@@ -89,7 +89,7 @@ defmodule Pleroma.Instances.Instance do
 
       existing_record.unreachable_since &&
           NaiveDateTime.compare(existing_record.unreachable_since, unreachable_since) != :gt ->
-        {:noop, existing_record}
+        {:ok, existing_record}
 
       true ->
         existing_record
@@ -98,5 +98,5 @@ defmodule Pleroma.Instances.Instance do
     end
   end
 
-  def set_unreachable(_, _), do: {0, :noop}
+  def set_unreachable(_, _), do: {:error, nil}
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 4232d6c0a..6cad02da6 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -750,9 +750,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       Instances.set_reachable(inbox)
       result
     else
-      e ->
+      {_post_result, response} ->
         Instances.set_unreachable(inbox)
-        e
+        {:error, response}
     end
   end
 
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index 80023127c..e96455423 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -181,10 +181,6 @@ defmodule Pleroma.Web.Salmon do
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
       :ok
     else
-      {:reachable, false} ->
-        Logger.debug(fn -> "Pushing Salmon to #{url} skipped as marked unreachable)" end)
-        :noop
-
       e ->
         Instances.set_unreachable(url)
         Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index 64eba7221..abe148270 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -286,14 +286,10 @@ defmodule Pleroma.Web.Websub do
       Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
       {:ok, code}
     else
-      {:reachable, false} ->
-        Logger.debug(fn -> "Pushing to #{callback} skipped as marked unreachable)" end)
-        {:error, :noop}
-
-      e ->
+      {_post_result, response} ->
         Instances.set_unreachable(callback)
-        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(e)}" end)
-        {:error, e}
+        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end)
+        {:error, response}
     end
   end
 end
-- 
cgit v1.2.3


From 92753b0cd9cfcdc5edb64a5e55ad27f73079f9e0 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 29 Jan 2019 13:12:28 +0300
Subject: [#534] Made federation push sender be determined basing on content
 instead of `referer` header. Updated tests.

---
 lib/pleroma/plugs/set_requester_reachable_plug.ex       | 16 ----------------
 lib/pleroma/reverse_proxy.ex                            |  3 +--
 lib/pleroma/web/activity_pub/activity_pub.ex            |  3 +--
 lib/pleroma/web/activity_pub/activity_pub_controller.ex | 11 ++++++++++-
 lib/pleroma/web/ostatus/ostatus.ex                      |  3 +++
 lib/pleroma/web/ostatus/ostatus_controller.ex           |  1 -
 lib/pleroma/web/salmon/salmon.ex                        |  5 +----
 lib/pleroma/web/websub/websub.ex                        |  3 +--
 lib/pleroma/web/websub/websub_controller.ex             |  2 --
 9 files changed, 17 insertions(+), 30 deletions(-)
 delete mode 100644 lib/pleroma/plugs/set_requester_reachable_plug.ex

(limited to 'lib')

diff --git a/lib/pleroma/plugs/set_requester_reachable_plug.ex b/lib/pleroma/plugs/set_requester_reachable_plug.ex
deleted file mode 100644
index 88551be70..000000000
--- a/lib/pleroma/plugs/set_requester_reachable_plug.ex
+++ /dev/null
@@ -1,16 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Plugs.SetRequesterReachablePlug do
-  import Plug.Conn
-
-  def init(_), do: []
-
-  def call(%Plug.Conn{} = conn, _) do
-    with [referer] <- get_req_header(conn, "referer"),
-         do: Pleroma.Instances.set_reachable(referer)
-
-    conn
-  end
-end
diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex
index d8b17212b..a25b5ea4e 100644
--- a/lib/pleroma/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy.ex
@@ -3,8 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.ReverseProxy do
-  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since) ++
-                      ~w(if-none-match if-range range referer)
+  @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since if-unmodified-since if-none-match if-range range)
   @resp_cache_headers ~w(etag date last-modified cache-control)
   @keep_resp_headers @resp_cache_headers ++
                        ~w(content-type content-disposition content-encoding content-range accept-ranges vary)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 22c7824fa..4016808e8 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -784,8 +784,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
                [
                  {"Content-Type", "application/activity+json"},
                  {"signature", signature},
-                 {"digest", digest},
-                 {"referer", Pleroma.Web.Endpoint.url()}
+                 {"digest", digest}
                ]
              ) do
       Instances.set_reachable(inbox)
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index fadb038a2..4dea6ab83 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -18,7 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   action_fallback(:errors)
 
   plug(Pleroma.Web.FederatingPlug when action in [:inbox, :relay])
-  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:inbox])
+  plug(:set_requester_reachable when action in [:inbox])
   plug(:relay_active? when action in [:relay])
 
   def relay_active?(conn, _) do
@@ -291,4 +291,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> put_status(500)
     |> json("error")
   end
+
+  defp set_requester_reachable(%Plug.Conn{} = conn, _) do
+    with actor <- conn.params["actor"],
+         true <- is_binary(actor) do
+      Pleroma.Instances.set_reachable(actor)
+    end
+
+    conn
+  end
 end
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index a3155b79d..a20ca17bb 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -48,6 +48,9 @@ defmodule Pleroma.Web.OStatus do
 
   def handle_incoming(xml_string) do
     with doc when doc != :error <- parse_document(xml_string) do
+      with {:ok, actor_user} <- find_make_or_update_user(doc),
+           do: Pleroma.Instances.set_reachable(actor_user.ap_id)
+
       entries = :xmerl_xpath.string('//entry', doc)
 
       activities =
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index 9392a97f0..302ff38a4 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -14,7 +14,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ActivityPub
 
   plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
-  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:salmon_incoming])
 
   action_fallback(:errors)
 
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
index e96455423..07ca42a5f 100644
--- a/lib/pleroma/web/salmon/salmon.ex
+++ b/lib/pleroma/web/salmon/salmon.ex
@@ -172,10 +172,7 @@ defmodule Pleroma.Web.Salmon do
            poster.(
              url,
              feed,
-             [
-               {"Content-Type", "application/magic-envelope+xml"},
-               {"referer", Pleroma.Web.Endpoint.url()}
-             ]
+             [{"Content-Type", "application/magic-envelope+xml"}]
            ) do
       Instances.set_reachable(url)
       Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
index abe148270..8f7d53b03 100644
--- a/lib/pleroma/web/websub/websub.ex
+++ b/lib/pleroma/web/websub/websub.ex
@@ -278,8 +278,7 @@ defmodule Pleroma.Web.Websub do
              xml,
              [
                {"Content-Type", "application/atom+xml"},
-               {"X-Hub-Signature", "sha1=#{signature}"},
-               {"referer", Pleroma.Web.Endpoint.url()}
+               {"X-Hub-Signature", "sha1=#{signature}"}
              ]
            ) do
       Instances.set_reachable(callback)
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
index 9da7e70a1..a92dfe87b 100644
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ b/lib/pleroma/web/websub/websub_controller.ex
@@ -20,8 +20,6 @@ defmodule Pleroma.Web.Websub.WebsubController do
          ]
   )
 
-  plug(Pleroma.Web.Plugs.SetRequesterReachablePlug when action in [:websub_incoming])
-
   def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
     user = User.get_cached_by_nickname(nickname)
 
-- 
cgit v1.2.3