Merge branch 'release/2.5.2' into 'stable'

Security release 2.5.2

See merge request pleroma/pleroma!3863

16 files changed, 141 insertions, 137 deletions

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index a9a9eeeed..cc3772563 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -8,77 +8,30 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Maps
   alias Pleroma.Object
   alias Pleroma.Object.Containment
-  alias Pleroma.Repo
   alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
+  alias Pleroma.Web.ActivityPub.MRF
   alias Pleroma.Web.ActivityPub.ObjectValidator
+  alias Pleroma.Web.ActivityPub.Pipeline
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
 
   require Logger
   require Pleroma.Constants
 
-  defp touch_changeset(changeset) do
-    updated_at =
-      NaiveDateTime.utc_now()
-      |> NaiveDateTime.truncate(:second)
-
-    Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
-  end
-
-  defp maybe_reinject_internal_fields(%{data: %{} = old_data}, new_data) do
-    has_history? = fn
-      %{"formerRepresentations" => %{"orderedItems" => list}} when is_list(list) -> true
-      _ -> false
-    end
-
-    internal_fields = Map.take(old_data, Pleroma.Constants.object_internal_fields())
-
-    remote_history_exists? = has_history?.(new_data)
-
-    # If the remote history exists, we treat that as the only source of truth.
-    new_data =
-      if has_history?.(old_data) and not remote_history_exists? do
-        Map.put(new_data, "formerRepresentations", old_data["formerRepresentations"])
-      else
-        new_data
-      end
-
-    # If the remote does not have history information, we need to manage it ourselves
-    new_data =
-      if not remote_history_exists? do
-        changed? =
-          Pleroma.Constants.status_updatable_fields()
-          |> Enum.any?(fn field -> Map.get(old_data, field) != Map.get(new_data, field) end)
-
-        %{updated_object: updated_object} =
-          new_data
-          |> Object.Updater.maybe_update_history(old_data,
-            updated: changed?,
-            use_history_in_new_object?: false
-          )
-
-        updated_object
-      else
-        new_data
-      end
-
-    Map.merge(new_data, internal_fields)
-  end
-
-  defp maybe_reinject_internal_fields(_, new_data), do: new_data
-
   @spec reinject_object(struct(), map()) :: {:ok, Object.t()} | {:error, any()}
-  defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do
+  defp reinject_object(%Object{data: %{}} = object, new_data) do
     Logger.debug("Reinjecting object #{new_data["id"]}")
 
-    with data <- maybe_reinject_internal_fields(object, new_data),
-         {:ok, data, _} <- ObjectValidator.validate(data, %{}),
-         changeset <- Object.change(object, %{data: data}),
-         changeset <- touch_changeset(changeset),
-         {:ok, object} <- Repo.insert_or_update(changeset),
-         {:ok, object} <- Object.set_cache(object) do
-      {:ok, object}
+    with {:ok, new_data, _} <- ObjectValidator.validate(new_data, %{}),
+         {:ok, new_data} <- MRF.filter(new_data),
+         {:ok, new_object, _} <-
+           Object.Updater.do_update_and_invalidate_cache(
+             object,
+             new_data,
+             _touch_changeset? = true
+           ) do
+      {:ok, new_object}
     else
       e ->
         Logger.error("Error while processing object: #{inspect(e)}")
@@ -86,20 +39,11 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  defp reinject_object(%Object{} = object, new_data) do
-    Logger.debug("Reinjecting object #{new_data["id"]}")
-
-    with new_data <- Transmogrifier.fix_object(new_data),
-         data <- maybe_reinject_internal_fields(object, new_data),
-         changeset <- Object.change(object, %{data: data}),
-         changeset <- touch_changeset(changeset),
-         {:ok, object} <- Repo.insert_or_update(changeset),
-         {:ok, object} <- Object.set_cache(object) do
+  defp reinject_object(_, new_data) do
+    with {:ok, object, _} <- Pipeline.common_pipeline(new_data, local: false) do
       {:ok, object}
     else
-      e ->
-        Logger.error("Error while processing object: #{inspect(e)}")
-        {:error, e}
+      e -> e
     end
   end
 
diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex
index ab38d3ed2..b1e4870ba 100644
--- a/lib/pleroma/object/updater.ex
+++ b/lib/pleroma/object/updater.ex
@@ -5,6 +5,9 @@
 defmodule Pleroma.Object.Updater do
   require Pleroma.Constants
 
+  alias Pleroma.Object
+  alias Pleroma.Repo
+
   def update_content_fields(orig_object_data, updated_object) do
     Pleroma.Constants.status_updatable_fields()
     |> Enum.reduce(
@@ -97,12 +100,14 @@ defmodule Pleroma.Object.Updater do
   end
 
   defp maybe_update_poll(to_be_updated, updated_object) do
-    choice_key = fn data ->
-      if Map.has_key?(data, "anyOf"), do: "anyOf", else: "oneOf"
+    choice_key = fn
+      %{"anyOf" => [_ | _]} -> "anyOf"
+      %{"oneOf" => [_ | _]} -> "oneOf"
+      _ -> nil
     end
 
     with true <- to_be_updated["type"] == "Question",
-         key <- choice_key.(updated_object),
+         key when not is_nil(key) <- choice_key.(updated_object),
          true <- key == choice_key.(to_be_updated),
          orig_choices <- to_be_updated[key] |> Enum.map(&Map.drop(&1, ["replies"])),
          new_choices <- updated_object[key] |> Enum.map(&Map.drop(&1, ["replies"])),
@@ -237,4 +242,49 @@ defmodule Pleroma.Object.Updater do
       {:history_items, e} -> e
     end
   end
+
+  defp maybe_touch_changeset(changeset, true) do
+    updated_at =
+      NaiveDateTime.utc_now()
+      |> NaiveDateTime.truncate(:second)
+
+    Ecto.Changeset.put_change(changeset, :updated_at, updated_at)
+  end
+
+  defp maybe_touch_changeset(changeset, _), do: changeset
+
+  def do_update_and_invalidate_cache(orig_object, updated_object, touch_changeset? \\ false) do
+    orig_object_ap_id = updated_object["id"]
+    orig_object_data = orig_object.data
+
+    %{
+      updated_data: updated_object_data,
+      updated: updated,
+      used_history_in_new_object?: used_history_in_new_object?
+    } = make_new_object_data_from_update_object(orig_object_data, updated_object)
+
+    changeset =
+      orig_object
+      |> Repo.preload(:hashtags)
+      |> Object.change(%{data: updated_object_data})
+      |> maybe_touch_changeset(touch_changeset?)
+
+    with {:ok, new_object} <- Repo.update(changeset),
+         {:ok, _} <- Object.invalid_object_cache(new_object),
+         {:ok, _} <- Object.set_cache(new_object),
+         # The metadata/utils.ex uses the object id for the cache.
+         {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do
+      if used_history_in_new_object? do
+        with create_activity when not is_nil(create_activity) <-
+               Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id),
+             {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do
+          nil
+        else
+          _ -> nil
+        end
+      end
+
+      {:ok, new_object, updated}
+    end
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index a2152b945..fc5dec362 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -428,37 +428,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
       end
 
     if orig_object_data["type"] in Pleroma.Constants.updatable_object_types() do
-      %{
-        updated_data: updated_object_data,
-        updated: updated,
-        used_history_in_new_object?: used_history_in_new_object?
-      } = Object.Updater.make_new_object_data_from_update_object(orig_object_data, updated_object)
-
-      changeset =
-        orig_object
-        |> Repo.preload(:hashtags)
-        |> Object.change(%{data: updated_object_data})
-
-      with {:ok, new_object} <- Repo.update(changeset),
-           {:ok, _} <- Object.invalid_object_cache(new_object),
-           {:ok, _} <- Object.set_cache(new_object),
-           # The metadata/utils.ex uses the object id for the cache.
-           {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(new_object.id) do
-        if used_history_in_new_object? do
-          with create_activity when not is_nil(create_activity) <-
-                 Pleroma.Activity.get_create_by_object_ap_id(orig_object_ap_id),
-               {:ok, _} <- Pleroma.Activity.HTML.invalidate_cache_for(create_activity.id) do
-            nil
-          else
-            _ -> nil
-          end
-        end
+      {:ok, _, updated} =
+        Object.Updater.do_update_and_invalidate_cache(orig_object, updated_object)
 
-        if updated do
-          object
-          |> Activity.normalize()
-          |> ActivityPub.notify_and_stream()
-        end
+      if updated do
+        object
+        |> Activity.normalize()
+        |> ActivityPub.notify_and_stream()
       end
     end
 
diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex
index 449659f4b..034722eb2 100644
--- a/lib/pleroma/web/feed/feed_view.ex
+++ b/lib/pleroma/web/feed/feed_view.ex
@@ -6,7 +6,6 @@ defmodule Pleroma.Web.Feed.FeedView do
   use Phoenix.HTML
   use Pleroma.Web, :view
 
-  alias Pleroma.Formatter
   alias Pleroma.Object
   alias Pleroma.User
   alias Pleroma.Web.Gettext
@@ -72,7 +71,9 @@ defmodule Pleroma.Web.Feed.FeedView do
 
   def last_activity(activities), do: List.last(activities)
 
-  def activity_title(%{"content" => content, "summary" => summary} = data, opts \\ %{}) do
+  def activity_title(%{"content" => content} = data, opts \\ %{}) do
+    summary = Map.get(data, "summary", "")
+
     title =
       cond do
         summary != "" -> summary
@@ -81,9 +82,8 @@ defmodule Pleroma.Web.Feed.FeedView do
       end
 
     title
-    |> Pleroma.Web.Metadata.Utils.scrub_html()
-    |> Pleroma.Emoji.Formatter.demojify()
-    |> Formatter.truncate(opts[:max_length], opts[:omission])
+    |> Pleroma.Web.Metadata.Utils.scrub_html_and_truncate(opts[:max_length], opts[:omission])
+    |> HtmlEntities.encode()
   end
 
   def activity_description(data) do
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index d2ad62c13..bda5b36ed 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -12,6 +12,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   alias Pleroma.Web.MediaProxy
   alias Plug.Conn
 
+  plug(:sandbox)
+
   def remote(conn, %{"sig" => sig64, "url" => url64}) do
     with {_, true} <- {:enabled, MediaProxy.enabled?()},
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
@@ -202,4 +204,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   defp media_proxy_opts do
     Config.get([:media_proxy, :proxy_opts], [])
   end
+
+  defp sandbox(conn, _params) do
+    conn
+    |> merge_resp_headers([{"content-security-policy", "sandbox;"}])
+  end
 end
diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex
index 15414a988..80a8be9a2 100644
--- a/lib/pleroma/web/metadata/utils.ex
+++ b/lib/pleroma/web/metadata/utils.ex
@@ -30,12 +30,13 @@ defmodule Pleroma.Web.Metadata.Utils do
     |> scrub_html_and_truncate_object_field(object)
   end
 
-  def scrub_html_and_truncate(content, max_length \\ 200) when is_binary(content) do
+  def scrub_html_and_truncate(content, max_length \\ 200, omission \\ "...")
+      when is_binary(content) do
     content
     |> scrub_html
     |> Emoji.Formatter.demojify()
     |> HtmlEntities.decode()
-    |> Formatter.truncate(max_length)
+    |> Formatter.truncate(max_length, omission)
   end
 
   def scrub_html(content) when is_binary(content) do
diff --git a/lib/pleroma/web/plugs/authentication_plug.ex b/lib/pleroma/web/plugs/authentication_plug.ex
index a7fd697b5..f912a1542 100644
--- a/lib/pleroma/web/plugs/authentication_plug.ex
+++ b/lib/pleroma/web/plugs/authentication_plug.ex
@@ -38,10 +38,6 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlug do
 
   def call(conn, _), do: conn
 
-  def checkpw(password, "$6" <> _ = password_hash) do
-    :crypt.crypt(password, password_hash) == password_hash
-  end
-
   def checkpw(password, "$2" <> _ = password_hash) do
     # Handle bcrypt passwords for Mastodon migration
     Bcrypt.verify_pass(password, password_hash)
@@ -60,10 +56,6 @@ defmodule Pleroma.Web.Plugs.AuthenticationPlug do
     do_update_password(user, password)
   end
 
-  def maybe_update_password(%User{password_hash: "$6" <> _} = user, password) do
-    do_update_password(user, password)
-  end
-
   def maybe_update_password(user, _), do: {:ok, user}
 
   defp do_update_password(user, password) do
diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex
index 75318d9c7..0f303176c 100644
--- a/lib/pleroma/web/rich_media/parsers/o_embed.ex
+++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex
@@ -6,8 +6,8 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
   def parse(html, _data) do
     with elements = [_ | _] <- get_discovery_data(html),
          oembed_url when is_binary(oembed_url) <- get_oembed_url(elements),
-         {:ok, oembed_data} <- get_oembed_data(oembed_url) do
-      oembed_data
+         {:ok, oembed_data = %{"html" => html}} <- get_oembed_data(oembed_url) do
+      %{oembed_data | "html" => Pleroma.HTML.filter_tags(html)}
     else
       _e -> %{}
     end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index ba1d64ab2..c1a690e28 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -835,8 +835,7 @@ defmodule Pleroma.Web.Router do
   end
 
   scope "/", Pleroma.Web do
-    # Note: html format is supported only if static FE is enabled
-    pipe_through([:accepts_html_xml, :static_fe])
+    pipe_through([:accepts_html_xml])
 
     get("/users/:nickname/feed", Feed.UserController, :feed, as: :user_feed)
   end
diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex
index 3c0da5c27..b9a04cc76 100644
--- a/lib/pleroma/web/streamer.ex
+++ b/lib/pleroma/web/streamer.ex
@@ -25,6 +25,7 @@ defmodule Pleroma.Web.Streamer do
   def registry, do: @registry
 
   @public_streams ["public", "public:local", "public:media", "public:local:media"]
+  @local_streams ["public:local", "public:local:media"]
   @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"]
 
   @doc "Expands and authorizes a stream, and registers the process for streaming."
@@ -41,14 +42,37 @@ defmodule Pleroma.Web.Streamer do
     end
   end
 
+  defp can_access_stream(user, oauth_token, kind) do
+    with {_, true} <- {:restrict?, Config.restrict_unauthenticated_access?(:timelines, kind)},
+         {_, %User{id: user_id}, %Token{user_id: user_id}} <- {:user, user, oauth_token},
+         {_, true} <-
+           {:scopes,
+            OAuthScopesPlug.filter_descendants(["read:statuses"], oauth_token.scopes) != []} do
+      true
+    else
+      {:restrict?, _} ->
+        true
+
+      _ ->
+        false
+    end
+  end
+
   @doc "Expand and authorizes a stream"
   @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) ::
           {:ok, topic :: String.t()} | {:error, :bad_topic}
   def get_topic(stream, user, oauth_token, params \\ %{})
 
-  # Allow all public steams.
-  def get_topic(stream, _user, _oauth_token, _params) when stream in @public_streams do
-    {:ok, stream}
+  # Allow all public steams if the instance allows unauthenticated access.
+  # Otherwise, only allow users with valid oauth tokens.
+  def get_topic(stream, user, oauth_token, _params) when stream in @public_streams do
+    kind = if stream in @local_streams, do: :local, else: :federated
+
+    if can_access_stream(user, oauth_token, kind) do
+      {:ok, stream}
+    else
+      {:error, :unauthorized}
+    end
   end
 
   # Allow all hashtags streams.
@@ -57,12 +81,20 @@ defmodule Pleroma.Web.Streamer do
   end
 
   # Allow remote instance streams.
-  def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do
-    {:ok, "public:remote:" <> instance}
+  def get_topic("public:remote", user, oauth_token, %{"instance" => instance} = _params) do
+    if can_access_stream(user, oauth_token, :federated) do
+      {:ok, "public:remote:" <> instance}
+    else
+      {:error, :unauthorized}
+    end
   end
 
-  def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do
-    {:ok, "public:remote:media:" <> instance}
+  def get_topic("public:remote:media", user, oauth_token, %{"instance" => instance} = _params) do
+    if can_access_stream(user, oauth_token, :federated) do
+      {:ok, "public:remote:media:" <> instance}
+    else
+      {:error, :unauthorized}
+    end
   end
 
   # Expand user streams.
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
index 260338772..b774f7984 100644
--- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
+++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
@@ -4,8 +4,8 @@
   <id><%= @data["id"] %></id>
   <title><%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %></title>
   <content type="html"><%= activity_description(@data) %></content>
-  <published><%= to_rfc3339(@activity.data["published"]) %></published>
-  <updated><%= to_rfc3339(@activity.data["published"]) %></updated>
+  <published><%= to_rfc3339(@data["published"]) %></published>
+  <updated><%= to_rfc3339(@data["published"]) %></updated>
   <ostatus:conversation ref="<%= activity_context(@activity) %>">
     <%= activity_context(@activity) %>
   </ostatus:conversation>
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
index 5c8f35fe4..7de98f736 100644
--- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
+++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
@@ -4,7 +4,7 @@
   <guid><%= @data["id"] %></guid>
   <title><%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %></title>
   <description><%= activity_description(@data) %></description>
-  <pubDate><%= to_rfc2822(@activity.data["published"]) %></pubDate>
+  <pubDate><%= to_rfc2822(@data["published"]) %></pubDate>
   <ostatus:conversation ref="<%= activity_context(@activity) %>">
     <%= activity_context(@activity) %>
   </ostatus:conversation>
diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
index 25980c1e4..03c222975 100644
--- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
+++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
@@ -7,8 +7,8 @@
   <id><%= @data["id"] %></id>
   <title><%= activity_title(@data, Keyword.get(@feed_config, :post_title, %{})) %></title>
   <content type="html"><%= activity_description(@data) %></content>
-  <published><%= to_rfc3339(@activity.data["published"]) %></published>
-  <updated><%= to_rfc3339(@activity.data["published"]) %></updated>
+  <published><%= to_rfc3339(@data["published"]) %></published>
+  <updated><%= to_rfc3339(@data["published"]) %></updated>
   <ostatus:conversation ref="<%= activity_context(@activity) %>">
     <%= activity_context(@activity) %>
   </ostatus:conversation>
diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex
index d582c83e8..1b8c34b87 100644
--- a/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex
+++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.xml.eex
@@ -4,7 +4,7 @@
 
   <guid isPermalink="true"><%= activity_context(@activity) %></guid>
   <link><%= activity_context(@activity) %></link>
-  <pubDate><%= to_rfc2822(@activity.data["published"]) %></pubDate>
+  <pubDate><%= to_rfc2822(@data["published"]) %></pubDate>
 
   <description><%= activity_description(@data) %></description>
   <%= for attachment <- @data["attachment"] || [] do %>
diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex
index 3805293bc..794417612 100644
--- a/lib/pleroma/workers/background_worker.ex
+++ b/lib/pleroma/workers/background_worker.ex
@@ -45,5 +45,5 @@ defmodule Pleroma.Workers.BackgroundWorker do
   end
 
   @impl Oban.Worker
-  def timeout(_job), do: :timer.seconds(5)
+  def timeout(_job), do: :timer.seconds(900)
 end
diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex
index 4f513b907..cf1bb62b4 100644
--- a/lib/pleroma/workers/receiver_worker.ex
+++ b/lib/pleroma/workers/receiver_worker.ex
@@ -13,6 +13,9 @@ defmodule Pleroma.Workers.ReceiverWorker do
       {:ok, res}
     else
       {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed}
+      {:error, :already_present} -> {:cancel, :already_present}
+      {:error, {:validate_object, reason}} -> {:cancel, reason}
+      {:error, {:error, {:validate, reason}}} -> {:cancel, reason}
       {:error, {:reject, reason}} -> {:cancel, reason}
       e -> e
     end