From 7cf708307644dad42412761c42c75da15dab7cc9 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 25 Feb 2020 16:21:48 +0300
Subject: relay list shows hosts without accepted follow

---
 lib/mix/tasks/pleroma/relay.ex        |  2 +-
 lib/pleroma/activity.ex               |  7 +++++++
 lib/pleroma/web/activity_pub/relay.ex | 19 ++++++++++++++++---
 3 files changed, 24 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex
index c6ca888d4..c3312507e 100644
--- a/lib/mix/tasks/pleroma/relay.ex
+++ b/lib/mix/tasks/pleroma/relay.ex
@@ -35,7 +35,7 @@ defmodule Mix.Tasks.Pleroma.Relay do
   def run(["list"]) do
     start_pleroma()
 
-    with {:ok, list} <- Relay.list() do
+    with {:ok, list} <- Relay.list(true) do
       list |> Enum.each(&shell_info(&1))
     else
       {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 397eb6e3f..6ca05f74e 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -308,6 +308,13 @@ defmodule Pleroma.Activity do
     |> where([a], fragment("? ->> 'state' = 'pending'", a.data))
   end
 
+  def following_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do
+    Queries.by_type("Follow")
+    |> where([a], fragment("?->>'state' = 'pending'", a.data))
+    |> where([a], a.actor == ^ap_id)
+    |> Repo.all()
+  end
+
   def restrict_deactivated_users(query) do
     deactivated_users =
       from(u in User.Query.build(%{deactivated: true}), select: u.ap_id)
diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex
index bb5542c89..729c23af7 100644
--- a/lib/pleroma/web/activity_pub/relay.ex
+++ b/lib/pleroma/web/activity_pub/relay.ex
@@ -60,15 +60,28 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def publish(_), do: {:error, "Not implemented"}
 
-  @spec list() :: {:ok, [String.t()]} | {:error, any()}
-  def list do
+  @spec list(boolean()) :: {:ok, [String.t()]} | {:error, any()}
+  def list(with_not_accepted \\ false) do
     with %User{} = user <- get_actor() do
-      list =
+      accepted =
         user
         |> User.following()
         |> Enum.map(fn entry -> URI.parse(entry).host end)
         |> Enum.uniq()
 
+      list =
+        if with_not_accepted do
+          without_accept =
+            user
+            |> Pleroma.Activity.following_requests_for_actor()
+            |> Enum.map(fn a -> URI.parse(a.data["object"]).host <> " (no Accept received)" end)
+            |> Enum.uniq()
+
+          accepted ++ without_accept
+        else
+          accepted
+        end
+
       {:ok, list}
     else
       error -> format_error(error)
-- 
cgit v1.2.3


From 423cfaf141afb03676aeb5e0f377be67e89092fc Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 11 Mar 2020 15:16:18 +0000
Subject: Merge branch 'fix/1610-release-compilation-config-fix' into 'develop'

Merging default release config on app start

Closes #1610

See merge request pleroma/pleroma!2288
---
 lib/mix/tasks/pleroma/docs.ex                     |  2 +-
 lib/pleroma/application.ex                        |  1 +
 lib/pleroma/config/holder.ex                      | 33 ++++++++++++++++++-----
 lib/pleroma/config/loader.ex                      | 30 +++++++++------------
 lib/pleroma/config/transfer_task.ex               |  2 +-
 lib/pleroma/docs/json.ex                          |  2 +-
 lib/pleroma/web/admin_api/admin_api_controller.ex |  2 +-
 7 files changed, 44 insertions(+), 28 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/docs.ex b/lib/mix/tasks/pleroma/docs.ex
index 3c870f876..6088fc71d 100644
--- a/lib/mix/tasks/pleroma/docs.ex
+++ b/lib/mix/tasks/pleroma/docs.ex
@@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Docs do
   defp do_run(implementation) do
     start_pleroma()
 
-    with descriptions <- Pleroma.Config.Loader.load("config/description.exs"),
+    with descriptions <- Pleroma.Config.Loader.read("config/description.exs"),
          {:ok, file_path} <-
            Pleroma.Docs.Generator.process(
              implementation,
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 18854b850..33f1705df 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -31,6 +31,7 @@ defmodule Pleroma.Application do
   # See http://elixir-lang.org/docs/stable/elixir/Application.html
   # for more information on OTP Applications
   def start(_type, _args) do
+    Pleroma.Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
     Pleroma.Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
diff --git a/lib/pleroma/config/holder.ex b/lib/pleroma/config/holder.ex
index f1a339703..f037d5d48 100644
--- a/lib/pleroma/config/holder.ex
+++ b/lib/pleroma/config/holder.ex
@@ -3,14 +3,33 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Config.Holder do
-  @config Pleroma.Config.Loader.load_and_merge()
+  @config Pleroma.Config.Loader.default_config()
 
-  @spec config() :: keyword()
-  def config, do: @config
+  @spec save_default() :: :ok
+  def save_default do
+    default_config =
+      if System.get_env("RELEASE_NAME") do
+        release_config =
+          [:code.root_dir(), "releases", System.get_env("RELEASE_VSN"), "releases.exs"]
+          |> Path.join()
+          |> Pleroma.Config.Loader.read()
 
-  @spec config(atom()) :: any()
-  def config(group), do: @config[group]
+        Pleroma.Config.Loader.merge(@config, release_config)
+      else
+        @config
+      end
 
-  @spec config(atom(), atom()) :: any()
-  def config(group, key), do: @config[group][key]
+    Pleroma.Config.put(:default_config, default_config)
+  end
+
+  @spec default_config() :: keyword()
+  def default_config, do: get_default()
+
+  @spec default_config(atom()) :: keyword()
+  def default_config(group), do: Keyword.get(get_default(), group)
+
+  @spec default_config(atom(), atom()) :: keyword()
+  def default_config(group, key), do: get_in(get_default(), [group, key])
+
+  defp get_default, do: Pleroma.Config.get(:default_config)
 end
diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex
index df2d18725..6ca6550bd 100644
--- a/lib/pleroma/config/loader.ex
+++ b/lib/pleroma/config/loader.ex
@@ -13,32 +13,28 @@ defmodule Pleroma.Config.Loader do
   ]
 
   if Code.ensure_loaded?(Config.Reader) do
-    @spec load(Path.t()) :: keyword()
-    def load(path), do: Config.Reader.read!(path)
+    @reader Config.Reader
 
-    defp do_merge(conf1, conf2), do: Config.Reader.merge(conf1, conf2)
+    def read(path), do: @reader.read!(path)
   else
     # support for Elixir less than 1.9
-    @spec load(Path.t()) :: keyword()
-    def load(path) do
+    @reader Mix.Config
+    def read(path) do
       path
-      |> Mix.Config.eval!()
+      |> @reader.eval!()
       |> elem(0)
     end
-
-    defp do_merge(conf1, conf2), do: Mix.Config.merge(conf1, conf2)
   end
 
-  @spec load_and_merge() :: keyword()
-  def load_and_merge do
-    all_paths =
-      if Pleroma.Config.get(:release),
-        do: ["config/config.exs", "config/releases.exs"],
-        else: ["config/config.exs"]
+  @spec read(Path.t()) :: keyword()
+
+  @spec merge(keyword(), keyword()) :: keyword()
+  def merge(c1, c2), do: @reader.merge(c1, c2)
 
-    all_paths
-    |> Enum.map(&load(&1))
-    |> Enum.reduce([], &do_merge(&2, &1))
+  @spec default_config() :: keyword()
+  def default_config do
+    "config/config.exs"
+    |> read()
     |> filter()
   end
 
diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex
index 435fc7450..7c3449b5e 100644
--- a/lib/pleroma/config/transfer_task.ex
+++ b/lib/pleroma/config/transfer_task.ex
@@ -83,7 +83,7 @@ defmodule Pleroma.Config.TransferTask do
       key = ConfigDB.from_string(setting.key)
       group = ConfigDB.from_string(setting.group)
 
-      default = Pleroma.Config.Holder.config(group, key)
+      default = Pleroma.Config.Holder.default_config(group, key)
       value = ConfigDB.from_binary(setting.value)
 
       merged_value =
diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex
index 6508a7bdb..74f8b2615 100644
--- a/lib/pleroma/docs/json.ex
+++ b/lib/pleroma/docs/json.ex
@@ -15,7 +15,7 @@ defmodule Pleroma.Docs.JSON do
   end
 
   def compile do
-    with config <- Pleroma.Config.Loader.load("config/description.exs") do
+    with config <- Pleroma.Config.Loader.read("config/description.exs") do
       config[:pleroma][:config_description]
       |> Pleroma.Docs.Generator.convert_to_strings()
       |> Jason.encode!()
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index de0755ee5..47b7d2da3 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -834,7 +834,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
       configs = ConfigDB.get_all_as_keyword()
 
       merged =
-        Config.Holder.config()
+        Config.Holder.default_config()
         |> ConfigDB.merge(configs)
         |> Enum.map(fn {group, values} ->
           Enum.map(values, fn {key, value} ->
-- 
cgit v1.2.3


From f8dc597c51ef294311687e5f3ff468635779638f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 10 Mar 2020 13:08:00 -0500
Subject: Fix enforcement of character limits

---
 lib/pleroma/web/common_api/utils.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 8746273c4..348fdedf1 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -591,7 +591,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     limit = Pleroma.Config.get([:instance, :limit])
     length = String.length(full_payload)
 
-    if length < limit do
+    if length <= limit do
       :ok
     else
       {:error, dgettext("errors", "The status is over the character limit")}
-- 
cgit v1.2.3


From e7837bc14e3fd539837802bca0c2ed05e2178ea5 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Wed, 11 Mar 2020 16:53:05 +0000
Subject: Merge branch 'fix/signup-without-email' into 'develop'

Allow account registration without an email

See merge request pleroma/pleroma!2246
---
 lib/pleroma/user.ex                                        |  9 ++++++++-
 .../web/mastodon_api/controllers/account_controller.ex     | 14 ++++++++++++--
 2 files changed, 20 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 5fe79333e..7531757f5 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -530,7 +530,14 @@ defmodule Pleroma.User do
   end
 
   def maybe_validate_required_email(changeset, true), do: changeset
-  def maybe_validate_required_email(changeset, _), do: validate_required(changeset, [:email])
+
+  def maybe_validate_required_email(changeset, _) do
+    if Pleroma.Config.get([:instance, :account_activation_required]) do
+      validate_required(changeset, [:email])
+    else
+      changeset
+    end
+  end
 
   defp put_ap_id(changeset) do
     ap_id = ap_id(%User{nickname: get_field(changeset, :nickname)})
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index dc3b47415..88c997b9f 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -76,7 +76,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   @doc "POST /api/v1/accounts"
   def create(
         %{assigns: %{app: app}} = conn,
-        %{"username" => nickname, "email" => _, "password" => _, "agreement" => true} = params
+        %{"username" => nickname, "password" => _, "agreement" => true} = params
       ) do
     params =
       params
@@ -93,7 +93,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
       |> Map.put("bio", params["bio"] || "")
       |> Map.put("confirm", params["password"])
 
-    with {:ok, user} <- TwitterAPI.register_user(params, need_confirmation: true),
+    with :ok <- validate_email_param(params),
+         {:ok, user} <- TwitterAPI.register_user(params, need_confirmation: true),
          {:ok, token} <- Token.create_token(app, user, %{scopes: app.scopes}) do
       json(conn, %{
         token_type: "Bearer",
@@ -114,6 +115,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     render_error(conn, :forbidden, "Invalid credentials")
   end
 
+  defp validate_email_param(%{"email" => _}), do: :ok
+
+  defp validate_email_param(_) do
+    case Pleroma.Config.get([:instance, :account_activation_required]) do
+      true -> {:error, %{"error" => "Missing parameters"}}
+      _ -> :ok
+    end
+  end
+
   @doc "GET /api/v1/accounts/verify_credentials"
   def verify_credentials(%{assigns: %{user: user}} = conn, _) do
     chat_token = Phoenix.Token.sign(conn, "user socket", user.id)
-- 
cgit v1.2.3


From 80bc8c2cc980b5e3270110313514a5bad2d3c9fb Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 11 Mar 2020 17:58:25 +0300
Subject: Revert "Set better Cache-Control header for static content"

On furher investigation it seems like all that did was cause unintuitive
behavior. The emoji request flood that was the reason for introducing it
isn't really that big of a deal either, since Plug.Static only needs to
read file modification time and size to determine the ETag.

Closes #1613
---
 lib/pleroma/web/endpoint.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 118c3ac6f..72cb3ee27 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Web.Endpoint do
   plug(Pleroma.Plugs.HTTPSecurityPlug)
   plug(Pleroma.Plugs.UploadedMedia)
 
-  @static_cache_control "public max-age=86400 must-revalidate"
+  @static_cache_control "public, no-cache"
 
   # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files
   # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well
-- 
cgit v1.2.3


From 5f9fbd7d336d2ffed6cd8f2640d9399f78ed7c2f Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 13 Feb 2020 03:39:47 +0100
Subject: Formatting: Do not use \n and prefer <br> instead

It moves bbcode to bbcode_pleroma as the former is owned by kaniini
and transfering ownership wasn't done in a timely manner.

Closes: https://git.pleroma.social/pleroma/pleroma/issues/1374
Closes: https://git.pleroma.social/pleroma/pleroma/issues/1375
---
 lib/pleroma/earmark_renderer.ex     | 256 ++++++++++++++++++++++++++++++++++++
 lib/pleroma/web/common_api/utils.ex |   2 +-
 2 files changed, 257 insertions(+), 1 deletion(-)
 create mode 100644 lib/pleroma/earmark_renderer.ex

(limited to 'lib')

diff --git a/lib/pleroma/earmark_renderer.ex b/lib/pleroma/earmark_renderer.ex
new file mode 100644
index 000000000..6211a3b4a
--- /dev/null
+++ b/lib/pleroma/earmark_renderer.ex
@@ -0,0 +1,256 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+#
+# This file is derived from Earmark, under the following copyright:
+# Copyright © 2014 Dave Thomas, The Pragmatic Programmers
+# SPDX-License-Identifier: Apache-2.0
+# Upstream: https://github.com/pragdave/earmark/blob/master/lib/earmark/html_renderer.ex
+defmodule Pleroma.EarmarkRenderer do
+  @moduledoc false
+
+  alias Earmark.Block
+  alias Earmark.Context
+  alias Earmark.HtmlRenderer
+  alias Earmark.Options
+
+  import Earmark.Inline, only: [convert: 3]
+  import Earmark.Helpers.HtmlHelpers
+  import Earmark.Message, only: [add_messages_from: 2, get_messages: 1, set_messages: 2]
+  import Earmark.Context, only: [append: 2, set_value: 2]
+  import Earmark.Options, only: [get_mapper: 1]
+
+  @doc false
+  def render(blocks, %Context{options: %Options{}} = context) do
+    messages = get_messages(context)
+
+    {contexts, html} =
+      get_mapper(context.options).(
+        blocks,
+        &render_block(&1, put_in(context.options.messages, []))
+      )
+      |> Enum.unzip()
+
+    all_messages =
+      contexts
+      |> Enum.reduce(messages, fn ctx, messages1 -> messages1 ++ get_messages(ctx) end)
+
+    {put_in(context.options.messages, all_messages), html |> IO.iodata_to_binary()}
+  end
+
+  #############
+  # Paragraph #
+  #############
+  defp render_block(%Block.Para{lnb: lnb, lines: lines, attrs: attrs}, context) do
+    lines = convert(lines, lnb, context)
+    add_attrs(lines, "<p>#{lines.value}</p>", attrs, [], lnb)
+  end
+
+  ########
+  # Html #
+  ########
+  defp render_block(%Block.Html{html: html}, context) do
+    {context, html}
+  end
+
+  defp render_block(%Block.HtmlComment{lines: lines}, context) do
+    {context, lines}
+  end
+
+  defp render_block(%Block.HtmlOneline{html: html}, context) do
+    {context, html}
+  end
+
+  #########
+  # Ruler #
+  #########
+  defp render_block(%Block.Ruler{lnb: lnb, attrs: attrs}, context) do
+    add_attrs(context, "<hr />", attrs, [], lnb)
+  end
+
+  ###########
+  # Heading #
+  ###########
+  defp render_block(
+         %Block.Heading{lnb: lnb, level: level, content: content, attrs: attrs},
+         context
+       ) do
+    converted = convert(content, lnb, context)
+    html = "<h#{level}>#{converted.value}</h#{level}>"
+    add_attrs(converted, html, attrs, [], lnb)
+  end
+
+  ##############
+  # Blockquote #
+  ##############
+
+  defp render_block(%Block.BlockQuote{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
+    {context1, body} = render(blocks, context)
+    html = "<blockquote>#{body}</blockquote>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  #########
+  # Table #
+  #########
+
+  defp render_block(
+         %Block.Table{lnb: lnb, header: header, rows: rows, alignments: aligns, attrs: attrs},
+         context
+       ) do
+    {context1, html} = add_attrs(context, "<table>", attrs, [], lnb)
+    context2 = set_value(context1, html)
+
+    context3 =
+      if header do
+        append(add_trs(append(context2, "<thead>"), [header], "th", aligns, lnb), "</thead>")
+      else
+        # Maybe an error, needed append(context, html)
+        context2
+      end
+
+    context4 = append(add_trs(append(context3, "<tbody>"), rows, "td", aligns, lnb), "</tbody>")
+
+    {context4, [context4.value, "</table>"]}
+  end
+
+  ########
+  # Code #
+  ########
+
+  defp render_block(
+         %Block.Code{lnb: lnb, language: language, attrs: attrs} = block,
+         %Context{options: options} = context
+       ) do
+    class =
+      if language, do: ~s{ class="#{code_classes(language, options.code_class_prefix)}"}, else: ""
+
+    tag = ~s[<pre><code#{class}>]
+    lines = options.render_code.(block)
+    html = ~s[#{tag}#{lines}</code></pre>]
+    add_attrs(context, html, attrs, [], lnb)
+  end
+
+  #########
+  # Lists #
+  #########
+
+  defp render_block(
+         %Block.List{lnb: lnb, type: type, blocks: items, attrs: attrs, start: start},
+         context
+       ) do
+    {context1, content} = render(items, context)
+    html = "<#{type}#{start}>#{content}</#{type}>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  # format a single paragraph list item, and remove the para tags
+  defp render_block(
+         %Block.ListItem{lnb: lnb, blocks: blocks, spaced: false, attrs: attrs},
+         context
+       )
+       when length(blocks) == 1 do
+    {context1, content} = render(blocks, context)
+    content = Regex.replace(~r{</?p>}, content, "")
+    html = "<li>#{content}</li>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  # format a spaced list item
+  defp render_block(%Block.ListItem{lnb: lnb, blocks: blocks, attrs: attrs}, context) do
+    {context1, content} = render(blocks, context)
+    html = "<li>#{content}</li>"
+    add_attrs(context1, html, attrs, [], lnb)
+  end
+
+  ##################
+  # Footnote Block #
+  ##################
+
+  defp render_block(%Block.FnList{blocks: footnotes}, context) do
+    items =
+      Enum.map(footnotes, fn note ->
+        blocks = append_footnote_link(note)
+        %Block.ListItem{attrs: "#fn:#{note.number}", type: :ol, blocks: blocks}
+      end)
+
+    {context1, html} = render_block(%Block.List{type: :ol, blocks: items}, context)
+    {context1, Enum.join([~s[<div class="footnotes">], "<hr />", html, "</div>"])}
+  end
+
+  #######################################
+  # Isolated IALs are rendered as paras #
+  #######################################
+
+  defp render_block(%Block.Ial{verbatim: verbatim}, context) do
+    {context, "<p>{:#{verbatim}}</p>"}
+  end
+
+  ####################
+  # IDDef is ignored #
+  ####################
+
+  defp render_block(%Block.IdDef{}, context), do: {context, ""}
+
+  #####################################
+  # And here are the inline renderers #
+  #####################################
+
+  defdelegate br, to: HtmlRenderer
+  defdelegate codespan(text), to: HtmlRenderer
+  defdelegate em(text), to: HtmlRenderer
+  defdelegate strong(text), to: HtmlRenderer
+  defdelegate strikethrough(text), to: HtmlRenderer
+
+  defdelegate link(url, text), to: HtmlRenderer
+  defdelegate link(url, text, title), to: HtmlRenderer
+
+  defdelegate image(path, alt, title), to: HtmlRenderer
+
+  defdelegate footnote_link(ref, backref, number), to: HtmlRenderer
+
+  # Table rows
+  defp add_trs(context, rows, tag, aligns, lnb) do
+    numbered_rows =
+      rows
+      |> Enum.zip(Stream.iterate(lnb, &(&1 + 1)))
+
+    numbered_rows
+    |> Enum.reduce(context, fn {row, lnb}, ctx ->
+      append(add_tds(append(ctx, "<tr>"), row, tag, aligns, lnb), "</tr>")
+    end)
+  end
+
+  defp add_tds(context, row, tag, aligns, lnb) do
+    Enum.reduce(1..length(row), context, add_td_fn(row, tag, aligns, lnb))
+  end
+
+  defp add_td_fn(row, tag, aligns, lnb) do
+    fn n, ctx ->
+      style =
+        case Enum.at(aligns, n - 1, :default) do
+          :default -> ""
+          align -> " style=\"text-align: #{align}\""
+        end
+
+      col = Enum.at(row, n - 1)
+      converted = convert(col, lnb, set_messages(ctx, []))
+      append(add_messages_from(ctx, converted), "<#{tag}#{style}>#{converted.value}</#{tag}>")
+    end
+  end
+
+  ###############################
+  # Append Footnote Return Link #
+  ###############################
+
+  defdelegate append_footnote_link(note), to: HtmlRenderer
+  defdelegate append_footnote_link(note, fnlink), to: HtmlRenderer
+
+  defdelegate render_code(lines), to: HtmlRenderer
+
+  defp code_classes(language, prefix) do
+    ["" | String.split(prefix || "")]
+    |> Enum.map(fn pfx -> "#{pfx}#{language}" end)
+    |> Enum.join(" ")
+  end
+end
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 348fdedf1..635e7cd38 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -331,7 +331,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   def format_input(text, "text/markdown", options) do
     text
     |> Formatter.mentions_escape(options)
-    |> Earmark.as_html!()
+    |> Earmark.as_html!(%Earmark.Options{renderer: Pleroma.EarmarkRenderer})
     |> Formatter.linkify(options)
     |> Formatter.html_escape("text/html")
   end
-- 
cgit v1.2.3


From bd80ff9a6c9f825aba9fadfc1fea6f05c3226590 Mon Sep 17 00:00:00 2001
From: Phil Hagelberg <phil@hagelb.org>
Date: Sat, 29 Feb 2020 18:53:49 -0800
Subject: Fix static FE plug to handle missing Accept header.

---
 lib/pleroma/plugs/static_fe_plug.ex | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex
index deebe4879..156e6788e 100644
--- a/lib/pleroma/plugs/static_fe_plug.ex
+++ b/lib/pleroma/plugs/static_fe_plug.ex
@@ -21,6 +21,9 @@ defmodule Pleroma.Plugs.StaticFEPlug do
   defp enabled?, do: Pleroma.Config.get([:static_fe, :enabled], false)
 
   defp accepts_html?(conn) do
-    conn |> get_req_header("accept") |> List.first() |> String.contains?("text/html")
+    case get_req_header(conn, "accept") do
+      [accept | _] -> String.contains?(accept, "text/html")
+      _ -> false
+    end
   end
 end
-- 
cgit v1.2.3


From ffd636f109d437ff49f38d5fabeb1109566f117b Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Fri, 13 Mar 2020 19:30:42 +0400
Subject: Fix hashtags WebSocket streaming

---
 lib/pleroma/activity/ir/topics.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex
index 4acc1a3e0..9e65bedad 100644
--- a/lib/pleroma/activity/ir/topics.ex
+++ b/lib/pleroma/activity/ir/topics.ex
@@ -39,7 +39,7 @@ defmodule Pleroma.Activity.Ir.Topics do
     end
   end
 
-  defp item_creation_tags(tags, %{data: %{"type" => "Create"}} = object, activity) do
+  defp item_creation_tags(tags, object, %{data: %{"type" => "Create"}} = activity) do
     tags ++ hashtags_to_topics(object) ++ attachment_topics(object, activity)
   end
 
-- 
cgit v1.2.3


From 3f54215219a03367d29e47527a2a6eefe59a89d7 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 29 Feb 2020 01:23:36 +0100
Subject: auth_controller.ex: Add admin scope to MastoFE

Related: https://git.pleroma.social/pleroma/pleroma/issues/1265
---
 lib/pleroma/web/mastodon_api/controllers/auth_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
index f165c9965..37b389382 100644
--- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
@@ -86,6 +86,6 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do
   @spec get_or_make_app() :: {:ok, App.t()} | {:error, Ecto.Changeset.t()}
   defp get_or_make_app do
     %{client_name: @local_mastodon_name, redirect_uris: "."}
-    |> App.get_or_make(["read", "write", "follow", "push"])
+    |> App.get_or_make(["read", "write", "follow", "push", "admin"])
   end
 end
-- 
cgit v1.2.3


From 0b823755a21646e83959cbf74fc7dff404b92187 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 14 Mar 2020 11:19:42 +0000
Subject: Merge branch 'fix/cache-control-headers' into 'develop'

Fix Cache Control headers on media

See merge request pleroma/pleroma!2295
---
 lib/pleroma/plugs/uploaded_media.ex        |  7 ++++++-
 lib/pleroma/reverse_proxy/reverse_proxy.ex | 20 +++++++++-----------
 2 files changed, 15 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex
index f372829a2..36ff024a7 100644
--- a/lib/pleroma/plugs/uploaded_media.ex
+++ b/lib/pleroma/plugs/uploaded_media.ex
@@ -14,9 +14,14 @@ defmodule Pleroma.Plugs.UploadedMedia do
   # no slashes
   @path "media"
 
+  @default_cache_control_header "public, max-age=1209600"
+
   def init(_opts) do
     static_plug_opts =
-      []
+      [
+        headers: %{"cache-control" => @default_cache_control_header},
+        cache_control_for_etags: @default_cache_control_header
+      ]
       |> Keyword.put(:from, "__unconfigured_media_plug")
       |> Keyword.put(:at, "/__unconfigured_media_plug")
       |> Plug.Static.init()
diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex
index a281a00dc..8b713b8f4 100644
--- a/lib/pleroma/reverse_proxy/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.ReverseProxy do
 
   @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++
                       ~w(if-unmodified-since if-none-match if-range range)
-  @resp_cache_headers ~w(etag date last-modified cache-control)
+  @resp_cache_headers ~w(etag date last-modified)
   @keep_resp_headers @resp_cache_headers ++
                        ~w(content-type content-disposition content-encoding content-range) ++
                        ~w(accept-ranges vary)
@@ -34,9 +34,6 @@ defmodule Pleroma.ReverseProxy do
   * request: `#{inspect(@keep_req_headers)}`
   * response: `#{inspect(@keep_resp_headers)}`
 
-  If no caching headers (`#{inspect(@resp_cache_headers)}`) are returned by upstream, `cache-control` will be
-  set to `#{inspect(@default_cache_control_header)}`.
-
   Options:
 
   * `redirect_on_failure` (default `false`). Redirects the client to the real remote URL if there's any HTTP
@@ -297,16 +294,17 @@ defmodule Pleroma.ReverseProxy do
 
   defp build_resp_cache_headers(headers, _opts) do
     has_cache? = Enum.any?(headers, fn {k, _} -> k in @resp_cache_headers end)
-    has_cache_control? = List.keymember?(headers, "cache-control", 0)
 
     cond do
-      has_cache? && has_cache_control? ->
-        headers
-
       has_cache? ->
-        # There's caching header present but no cache-control -- we need to explicitely override it
-        # to public as Plug defaults to "max-age=0, private, must-revalidate"
-        List.keystore(headers, "cache-control", 0, {"cache-control", "public"})
+        # There's caching header present but no cache-control -- we need to set our own
+        # as Plug defaults to "max-age=0, private, must-revalidate"
+        List.keystore(
+          headers,
+          "cache-control",
+          0,
+          {"cache-control", @default_cache_control_header}
+        )
 
       true ->
         List.keystore(
-- 
cgit v1.2.3


From 306d633b40534fa37908e76d362da73acb16bd1e Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 2 Mar 2020 04:23:29 +0100
Subject: pleroma_api_controller.ex: Improve conversations error reporting

Related: https://git.pleroma.social/pleroma/pleroma/issues/1594
---
 .../controllers/pleroma_api_controller.ex          | 33 ++++++++++++++++------
 1 file changed, 25 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
index 0e160bbfc..dae7f0f2f 100644
--- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
@@ -101,6 +101,11 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
       conn
       |> put_view(ConversationView)
       |> render("participation.json", %{participation: participation, for: user})
+    else
+      _error ->
+        conn
+        |> put_status(404)
+        |> json(%{"error" => "Unknown conversation id"})
     end
   end
 
@@ -108,9 +113,9 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
         %{assigns: %{user: user}} = conn,
         %{"id" => participation_id} = params
       ) do
-    participation = Participation.get(participation_id, preload: [:conversation])
-
-    if user.id == participation.user_id do
+    with %Participation{} = participation <-
+           Participation.get(participation_id, preload: [:conversation]),
+         true <- user.id == participation.user_id do
       params =
         params
         |> Map.put("blocking_user", user)
@@ -126,6 +131,11 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
       |> add_link_headers(activities)
       |> put_view(StatusView)
       |> render("index.json", %{activities: activities, for: user, as: :activity})
+    else
+      _error ->
+        conn
+        |> put_status(404)
+        |> json(%{"error" => "Unknown conversation id"})
     end
   end
 
@@ -133,15 +143,22 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
         %{assigns: %{user: user}} = conn,
         %{"id" => participation_id, "recipients" => recipients}
       ) do
-    participation =
-      participation_id
-      |> Participation.get()
-
-    with true <- user.id == participation.user_id,
+    with %Participation{} = participation <- Participation.get(participation_id),
+         true <- user.id == participation.user_id,
          {:ok, participation} <- Participation.set_recipients(participation, recipients) do
       conn
       |> put_view(ConversationView)
       |> render("participation.json", %{participation: participation, for: user})
+    else
+      {:error, message} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{"error" => message})
+
+      _error ->
+        conn
+        |> put_status(404)
+        |> json(%{"error" => "Unknown conversation id"})
     end
   end
 
-- 
cgit v1.2.3


From fcf51a77baa4661d4b20d55b87577d5de0a8c422 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sun, 15 Mar 2020 19:53:52 +0000
Subject: Merge branch 'features/staticfe-sanitization' into 'develop'

static_fe: Sanitize HTML

Closes #1614

See merge request pleroma/pleroma!2299
---
 lib/pleroma/user.ex                                | 24 ++++++++++++++++++++++
 lib/pleroma/web/activity_pub/views/user_view.ex    |  7 +------
 lib/pleroma/web/admin_api/views/account_view.ex    |  4 ++--
 lib/pleroma/web/mastodon_api/views/account_view.ex | 19 ++++-------------
 lib/pleroma/web/static_fe/static_fe_controller.ex  | 13 +++++++++---
 5 files changed, 41 insertions(+), 26 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 7531757f5..c68616385 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.User do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Delivery
   alias Pleroma.FollowingRelationship
+  alias Pleroma.HTML
   alias Pleroma.Keys
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -2062,4 +2063,27 @@ defmodule Pleroma.User do
     |> validate_required([:invisible])
     |> update_and_set_cache()
   end
+
+  def sanitize_html(%User{} = user) do
+    sanitize_html(user, nil)
+  end
+
+  # User data that mastodon isn't filtering (treated as plaintext):
+  # - field name
+  # - display name
+  def sanitize_html(%User{} = user, filter) do
+    fields =
+      user
+      |> User.fields()
+      |> Enum.map(fn %{"name" => name, "value" => value} ->
+        %{
+          "name" => name,
+          "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)
+        }
+      end)
+
+    user
+    |> Map.put(:bio, HTML.filter_tags(user.bio, filter))
+    |> Map.put(:fields, fields)
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index c0358b678..bc21ac6c7 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -73,6 +73,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     {:ok, _, public_key} = Keys.keys_from_pem(user.keys)
     public_key = :public_key.pem_entry_encode(:SubjectPublicKeyInfo, public_key)
     public_key = :public_key.pem_encode([public_key])
+    user = User.sanitize_html(user)
 
     endpoints = render("endpoints.json", %{user: user})
 
@@ -81,12 +82,6 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     fields =
       user
       |> User.fields()
-      |> Enum.map(fn %{"name" => name, "value" => value} ->
-        %{
-          "name" => Pleroma.HTML.strip_tags(name),
-          "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)
-        }
-      end)
       |> Enum.map(&Map.put(&1, "type", "PropertyValue"))
 
     %{
diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index 619390ef4..1e03849de 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.AdminAPI.AccountView do
   use Pleroma.Web, :view
 
-  alias Pleroma.HTML
   alias Pleroma.User
   alias Pleroma.Web.AdminAPI.AccountView
   alias Pleroma.Web.MediaProxy
@@ -26,7 +25,8 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
 
   def render("show.json", %{user: user}) do
     avatar = User.avatar_url(user) |> MediaProxy.url()
-    display_name = HTML.strip_tags(user.name || user.nickname)
+    display_name = Pleroma.HTML.strip_tags(user.name || user.nickname)
+    user = User.sanitize_html(user, FastSanitize.Sanitizer.StripTags)
 
     %{
       "id" => user.id,
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 6dc191250..341dc2c91 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.MastodonAPI.AccountView do
   use Pleroma.Web, :view
 
-  alias Pleroma.HTML
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
@@ -67,6 +66,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
   end
 
   defp do_render("show.json", %{user: user} = opts) do
+    user = User.sanitize_html(user, User.html_filter_policy(opts[:for]))
     display_name = user.name || user.nickname
 
     image = User.avatar_url(user) |> MediaProxy.url()
@@ -100,17 +100,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
         }
       end)
 
-    fields =
-      user
-      |> User.fields()
-      |> Enum.map(fn %{"name" => name, "value" => value} ->
-        %{
-          "name" => name,
-          "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)
-        }
-      end)
-
-    bio = HTML.filter_tags(user.bio, User.html_filter_policy(opts[:for]))
     relationship = render("relationship.json", %{user: opts[:for], target: user})
 
     %{
@@ -123,17 +112,17 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       followers_count: followers_count,
       following_count: following_count,
       statuses_count: user.note_count,
-      note: bio || "",
+      note: user.bio || "",
       url: User.profile_url(user),
       avatar: image,
       avatar_static: image,
       header: header,
       header_static: header,
       emojis: emojis,
-      fields: fields,
+      fields: user.fields,
       bot: bot,
       source: %{
-        note: HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
+        note: Pleroma.HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
         sensitive: false,
         fields: user.raw_fields,
         pleroma: %{
diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex
index 5ac75f1c4..98977bc19 100644
--- a/lib/pleroma/web/static_fe/static_fe_controller.ex
+++ b/lib/pleroma/web/static_fe/static_fe_controller.ex
@@ -54,10 +54,17 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do
         _ -> data["url"] || data["external_url"] || data["id"]
       end
 
+    content =
+      if data["content"] do
+        Pleroma.HTML.filter_tags(data["content"])
+      else
+        nil
+      end
+
     %{
-      user: user,
+      user: User.sanitize_html(user),
       title: get_title(activity.object),
-      content: data["content"] || nil,
+      content: content,
       attachment: data["attachment"],
       link: link,
       published: data["published"],
@@ -109,7 +116,7 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do
         next_page_id = List.last(timeline) && List.last(timeline).id
 
         render(conn, "profile.html", %{
-          user: user,
+          user: User.sanitize_html(user),
           timeline: timeline,
           prev_page_id: prev_page_id,
           next_page_id: next_page_id,
-- 
cgit v1.2.3


From 9d0975529182a1f9002ed8ccd7aa94d058ef83f7 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 14 Mar 2020 15:39:58 +0300
Subject: rip out fetch_initial_posts

Every time someone tries to use it, it goes mad and tries to scrape the
entire fediverse for no visible reason, it's better to just remove it
than continue shipping it in it's current state.

idea acked by lain and feld on irc

Closes #1595 #1422
---
 lib/pleroma/user.ex                      | 32 +-------------------------
 lib/pleroma/web/activity_pub/utils.ex    | 39 --------------------------------
 lib/pleroma/workers/background_worker.ex |  4 ----
 3 files changed, 1 insertion(+), 74 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index c68616385..911dde6e2 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -840,10 +840,6 @@ defmodule Pleroma.User do
       _e ->
         with [_nick, _domain] <- String.split(nickname, "@"),
              {:ok, user} <- fetch_by_nickname(nickname) do
-          if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do
-            fetch_initial_posts(user)
-          end
-
           {:ok, user}
         else
           _e -> {:error, "not found " <> nickname}
@@ -851,11 +847,6 @@ defmodule Pleroma.User do
     end
   end
 
-  @doc "Fetch some posts when the user has just been federated with"
-  def fetch_initial_posts(user) do
-    BackgroundWorker.enqueue("fetch_initial_posts", %{"user_id" => user.id})
-  end
-
   @spec get_followers_query(User.t(), pos_integer() | nil) :: Ecto.Query.t()
   def get_followers_query(%User{} = user, nil) do
     User.Query.build(%{followers: user, deactivated: false})
@@ -1321,16 +1312,6 @@ defmodule Pleroma.User do
     Repo.delete(user)
   end
 
-  def perform(:fetch_initial_posts, %User{} = user) do
-    pages = Pleroma.Config.get!([:fetch_initial_posts, :pages])
-
-    # Insert all the posts in reverse order, so they're in the right order on the timeline
-    user.source_data["outbox"]
-    |> Utils.fetch_ordered_collection(pages)
-    |> Enum.reverse()
-    |> Enum.each(&Pleroma.Web.Federator.incoming_ap_doc/1)
-  end
-
   def perform(:deactivate_async, user, status), do: deactivate(user, status)
 
   @spec perform(atom(), User.t(), list()) :: list() | {:error, any()}
@@ -1459,18 +1440,7 @@ defmodule Pleroma.User do
     if !is_nil(user) and !needs_update?(user) do
       {:ok, user}
     else
-      # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled)
-      should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled])
-
-      resp = fetch_by_ap_id(ap_id)
-
-      if should_fetch_initial do
-        with {:ok, %User{} = user} <- resp do
-          fetch_initial_posts(user)
-        end
-      end
-
-      resp
+      fetch_by_ap_id(ap_id)
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index 2bc958670..15dd2ed45 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -784,45 +784,6 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
   defp build_flag_object(_), do: []
 
-  @doc """
-  Fetches the OrderedCollection/OrderedCollectionPage from `from`, limiting the amount of pages fetched after
-  the first one to `pages_left` pages.
-  If the amount of pages is higher than the collection has, it returns whatever was there.
-  """
-  def fetch_ordered_collection(from, pages_left, acc \\ []) do
-    with {:ok, response} <- Tesla.get(from),
-         {:ok, collection} <- Jason.decode(response.body) do
-      case collection["type"] do
-        "OrderedCollection" ->
-          # If we've encountered the OrderedCollection and not the page,
-          # just call the same function on the page address
-          fetch_ordered_collection(collection["first"], pages_left)
-
-        "OrderedCollectionPage" ->
-          if pages_left > 0 do
-            # There are still more pages
-            if Map.has_key?(collection, "next") do
-              # There are still more pages, go deeper saving what we have into the accumulator
-              fetch_ordered_collection(
-                collection["next"],
-                pages_left - 1,
-                acc ++ collection["orderedItems"]
-              )
-            else
-              # No more pages left, just return whatever we already have
-              acc ++ collection["orderedItems"]
-            end
-          else
-            # Got the amount of pages needed, add them all to the accumulator
-            acc ++ collection["orderedItems"]
-          end
-
-        _ ->
-          {:error, "Not an OrderedCollection or OrderedCollectionPage"}
-      end
-    end
-  end
-
   #### Report-related helpers
   def get_reports(params, page, page_size) do
     params =
diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex
index 598df6580..0f8ece2c4 100644
--- a/lib/pleroma/workers/background_worker.ex
+++ b/lib/pleroma/workers/background_worker.ex
@@ -10,10 +10,6 @@ defmodule Pleroma.Workers.BackgroundWorker do
   use Pleroma.Workers.WorkerHelper, queue: "background"
 
   @impl Oban.Worker
-  def perform(%{"op" => "fetch_initial_posts", "user_id" => user_id}, _job) do
-    user = User.get_cached_by_id(user_id)
-    User.perform(:fetch_initial_posts, user)
-  end
 
   def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}, _job) do
     user = User.get_cached_by_id(user_id)
-- 
cgit v1.2.3


From c46d035f7bc79f451c8b2356f3b809c29684cfe4 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 13 Mar 2020 21:15:42 +0300
Subject: rate limiter: disable based on if remote ip was found, not on if the
 plug was enabled

The current rate limiter disable logic won't trigger when the remote ip
is not forwarded, only when the remoteip plug is not enabled, which is
not the case on most instances since it's enabled by default. This
changes the behavior to warn and disable  when the remote ip was not forwarded,
even if the RemoteIP plug is enabled.

Also closes #1620
---
 lib/pleroma/plugs/rate_limiter/rate_limiter.ex | 27 +++++++++++++++-----------
 lib/pleroma/plugs/remote_ip.ex                 |  7 +++++--
 2 files changed, 21 insertions(+), 13 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/rate_limiter/rate_limiter.ex b/lib/pleroma/plugs/rate_limiter/rate_limiter.ex
index c3f6351c8..1529da717 100644
--- a/lib/pleroma/plugs/rate_limiter/rate_limiter.ex
+++ b/lib/pleroma/plugs/rate_limiter/rate_limiter.ex
@@ -78,7 +78,7 @@ defmodule Pleroma.Plugs.RateLimiter do
   end
 
   def call(conn, plug_opts) do
-    if disabled?() do
+    if disabled?(conn) do
       handle_disabled(conn)
     else
       action_settings = action_settings(plug_opts)
@@ -87,9 +87,9 @@ defmodule Pleroma.Plugs.RateLimiter do
   end
 
   defp handle_disabled(conn) do
-    if Config.get(:env) == :prod do
-      Logger.warn("Rate limiter is disabled for localhost/socket")
-    end
+    Logger.warn(
+      "Rate limiter disabled due to forwarded IP not being found. Please ensure your reverse proxy is providing the X-Forwarded-For header or disable the RemoteIP plug/rate limiter."
+    )
 
     conn
   end
@@ -109,16 +109,21 @@ defmodule Pleroma.Plugs.RateLimiter do
     end
   end
 
-  def disabled? do
+  def disabled?(conn) do
     localhost_or_socket =
-      Config.get([Pleroma.Web.Endpoint, :http, :ip])
-      |> Tuple.to_list()
-      |> Enum.join(".")
-      |> String.match?(~r/^local|^127.0.0.1/)
+      case Config.get([Pleroma.Web.Endpoint, :http, :ip]) do
+        {127, 0, 0, 1} -> true
+        {0, 0, 0, 0, 0, 0, 0, 1} -> true
+        {:local, _} -> true
+        _ -> false
+      end
 
-    remote_ip_disabled = not Config.get([Pleroma.Plugs.RemoteIp, :enabled])
+    remote_ip_not_found =
+      if Map.has_key?(conn.assigns, :remote_ip_found),
+        do: !conn.assigns.remote_ip_found,
+        else: false
 
-    localhost_or_socket and remote_ip_disabled
+    localhost_or_socket and remote_ip_not_found
   end
 
   @inspect_bucket_not_found {:error, :not_found}
diff --git a/lib/pleroma/plugs/remote_ip.ex b/lib/pleroma/plugs/remote_ip.ex
index 2eca4f8f6..0ac9050d0 100644
--- a/lib/pleroma/plugs/remote_ip.ex
+++ b/lib/pleroma/plugs/remote_ip.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Plugs.RemoteIp do
   This is a shim to call [`RemoteIp`](https://git.pleroma.social/pleroma/remote_ip) but with runtime configuration.
   """
 
+  import Plug.Conn
+
   @behaviour Plug
 
   @headers ~w[
@@ -26,11 +28,12 @@ defmodule Pleroma.Plugs.RemoteIp do
 
   def init(_), do: nil
 
-  def call(conn, _) do
+  def call(%{remote_ip: original_remote_ip} = conn, _) do
     config = Pleroma.Config.get(__MODULE__, [])
 
     if Keyword.get(config, :enabled, false) do
-      RemoteIp.call(conn, remote_ip_opts(config))
+      %{remote_ip: new_remote_ip} = conn = RemoteIp.call(conn, remote_ip_opts(config))
+      assign(conn, :remote_ip_found, original_remote_ip != new_remote_ip)
     else
       conn
     end
-- 
cgit v1.2.3


From 0e92aa0025fbdb268735502cae0a434558e290c5 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 25 Mar 2020 18:00:24 +0000
Subject: Merge branch '1364-notifications-sending-control' into 'develop'

[#1364] Ability to opt-out of notifications (in all clients)

Closes #1364

See merge request pleroma/pleroma!2301
---
 lib/pleroma/activity.ex                            |  11 ++
 lib/pleroma/notification.ex                        | 134 +++++++++++++++------
 lib/pleroma/thread_mute.ex                         |  38 +++++-
 lib/pleroma/user.ex                                |  58 +++++++--
 lib/pleroma/user_relationship.ex                   |   9 +-
 lib/pleroma/web/activity_pub/activity_pub.ex       |   8 +-
 lib/pleroma/web/activity_pub/transmogrifier.ex     |   8 +-
 .../mastodon_api/controllers/account_controller.ex |  10 +-
 lib/pleroma/web/streamer/worker.ex                 |   2 +-
 9 files changed, 208 insertions(+), 70 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 6ca05f74e..5a8329e69 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -95,6 +95,17 @@ defmodule Pleroma.Activity do
     |> preload([activity, object: object], object: object)
   end
 
+  # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
+  def user_actor(%Activity{actor: nil}), do: nil
+
+  def user_actor(%Activity{} = activity) do
+    with %User{} <- activity.user_actor do
+      activity.user_actor
+    else
+      _ -> User.get_cached_by_ap_id(activity.actor)
+    end
+  end
+
   def with_joined_user_actor(query, join_type \\ :inner) do
     join(query, join_type, [activity], u in User,
       on: u.ap_id == activity.actor,
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 60dba3434..824ba5ecb 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Notification do
   alias Pleroma.Object
   alias Pleroma.Pagination
   alias Pleroma.Repo
+  alias Pleroma.ThreadMute
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.Push
@@ -17,6 +18,7 @@ defmodule Pleroma.Notification do
 
   import Ecto.Query
   import Ecto.Changeset
+
   require Logger
 
   @type t :: %__MODULE__{}
@@ -37,11 +39,11 @@ defmodule Pleroma.Notification do
   end
 
   defp for_user_query_ap_id_opts(user, opts) do
-    ap_id_relations =
+    ap_id_relationships =
       [:block] ++
         if opts[@include_muted_option], do: [], else: [:notification_mute]
 
-    preloaded_ap_ids = User.outgoing_relations_ap_ids(user, ap_id_relations)
+    preloaded_ap_ids = User.outgoing_relationships_ap_ids(user, ap_id_relationships)
 
     exclude_blocked_opts = Map.merge(%{blocked_users_ap_ids: preloaded_ap_ids[:block]}, opts)
 
@@ -101,7 +103,7 @@ defmodule Pleroma.Notification do
 
     query
     |> where([n, a], a.actor not in ^notification_muted_ap_ids)
-    |> join(:left, [n, a], tm in Pleroma.ThreadMute,
+    |> join(:left, [n, a], tm in ThreadMute,
       on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
     )
     |> where([n, a, o, tm], is_nil(tm.user_id))
@@ -284,58 +286,111 @@ defmodule Pleroma.Notification do
   def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do
     object = Object.normalize(activity)
 
-    unless object && object.data["type"] == "Answer" do
-      users = get_notified_from_activity(activity)
-      notifications = Enum.map(users, fn user -> create_notification(activity, user) end)
-      {:ok, notifications}
-    else
+    if object && object.data["type"] == "Answer" do
       {:ok, []}
+    else
+      do_create_notifications(activity)
     end
   end
 
   def create_notifications(%Activity{data: %{"type" => type}} = activity)
       when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do
+    do_create_notifications(activity)
+  end
+
+  def create_notifications(_), do: {:ok, []}
+
+  defp do_create_notifications(%Activity{} = activity) do
+    {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity)
+    potential_receivers = enabled_receivers ++ disabled_receivers
+
     notifications =
-      activity
-      |> get_notified_from_activity()
-      |> Enum.map(&create_notification(activity, &1))
+      Enum.map(potential_receivers, fn user ->
+        do_send = user in enabled_receivers
+        create_notification(activity, user, do_send)
+      end)
 
     {:ok, notifications}
   end
 
-  def create_notifications(_), do: {:ok, []}
-
   # TODO move to sql, too.
-  def create_notification(%Activity{} = activity, %User{} = user) do
+  def create_notification(%Activity{} = activity, %User{} = user, do_send \\ true) do
     unless skip?(activity, user) do
       notification = %Notification{user_id: user.id, activity: activity}
       {:ok, notification} = Repo.insert(notification)
 
-      ["user", "user:notification"]
-      |> Streamer.stream(notification)
+      if do_send do
+        Streamer.stream(["user", "user:notification"], notification)
+        Push.send(notification)
+      end
 
-      Push.send(notification)
       notification
     end
   end
 
+  @doc """
+  Returns a tuple with 2 elements:
+    {enabled notification receivers, currently disabled receivers (blocking / [thread] muting)}
+
+  NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1
+  """
   def get_notified_from_activity(activity, local_only \\ true)
 
   def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only)
       when type in ["Create", "Like", "Announce", "Follow", "Move", "EmojiReact"] do
-    []
-    |> Utils.maybe_notify_to_recipients(activity)
-    |> Utils.maybe_notify_mentioned_recipients(activity)
-    |> Utils.maybe_notify_subscribers(activity)
-    |> Utils.maybe_notify_followers(activity)
-    |> Enum.uniq()
-    |> User.get_users_from_set(local_only)
+    potential_receiver_ap_ids =
+      []
+      |> Utils.maybe_notify_to_recipients(activity)
+      |> Utils.maybe_notify_mentioned_recipients(activity)
+      |> Utils.maybe_notify_subscribers(activity)
+      |> Utils.maybe_notify_followers(activity)
+      |> Enum.uniq()
+
+    # Since even subscribers and followers can mute / thread-mute, filtering all above AP IDs
+    notification_enabled_ap_ids =
+      potential_receiver_ap_ids
+      |> exclude_relationship_restricted_ap_ids(activity)
+      |> exclude_thread_muter_ap_ids(activity)
+
+    potential_receivers =
+      potential_receiver_ap_ids
+      |> Enum.uniq()
+      |> User.get_users_from_set(local_only)
+
+    notification_enabled_users =
+      Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end)
+
+    {notification_enabled_users, potential_receivers -- notification_enabled_users}
+  end
+
+  def get_notified_from_activity(_, _local_only), do: {[], []}
+
+  @doc "Filters out AP IDs of users basing on their relationships with activity actor user"
+  def exclude_relationship_restricted_ap_ids([], _activity), do: []
+
+  def exclude_relationship_restricted_ap_ids(ap_ids, %Activity{} = activity) do
+    relationship_restricted_ap_ids =
+      activity
+      |> Activity.user_actor()
+      |> User.incoming_relationships_ungrouped_ap_ids([
+        :block,
+        :notification_mute
+      ])
+
+    Enum.uniq(ap_ids) -- relationship_restricted_ap_ids
   end
 
-  def get_notified_from_activity(_, _local_only), do: []
+  @doc "Filters out AP IDs of users who mute activity thread"
+  def exclude_thread_muter_ap_ids([], _activity), do: []
+
+  def exclude_thread_muter_ap_ids(ap_ids, %Activity{} = activity) do
+    thread_muter_ap_ids = ThreadMute.muter_ap_ids(activity.data["context"])
+
+    Enum.uniq(ap_ids) -- thread_muter_ap_ids
+  end
 
   @spec skip?(Activity.t(), User.t()) :: boolean()
-  def skip?(activity, user) do
+  def skip?(%Activity{} = activity, %User{} = user) do
     [
       :self,
       :followers,
@@ -344,18 +399,20 @@ defmodule Pleroma.Notification do
       :non_follows,
       :recently_followed
     ]
-    |> Enum.any?(&skip?(&1, activity, user))
+    |> Enum.find(&skip?(&1, activity, user))
   end
 
+  def skip?(_, _), do: false
+
   @spec skip?(atom(), Activity.t(), User.t()) :: boolean()
-  def skip?(:self, activity, user) do
+  def skip?(:self, %Activity{} = activity, %User{} = user) do
     activity.data["actor"] == user.ap_id
   end
 
   def skip?(
         :followers,
-        activity,
-        %{notification_settings: %{followers: false}} = user
+        %Activity{} = activity,
+        %User{notification_settings: %{followers: false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
@@ -364,15 +421,19 @@ defmodule Pleroma.Notification do
 
   def skip?(
         :non_followers,
-        activity,
-        %{notification_settings: %{non_followers: false}} = user
+        %Activity{} = activity,
+        %User{notification_settings: %{non_followers: false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
     !User.following?(follower, user)
   end
 
-  def skip?(:follows, activity, %{notification_settings: %{follows: false}} = user) do
+  def skip?(
+        :follows,
+        %Activity{} = activity,
+        %User{notification_settings: %{follows: false}} = user
+      ) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)
     User.following?(user, followed)
@@ -380,15 +441,16 @@ defmodule Pleroma.Notification do
 
   def skip?(
         :non_follows,
-        activity,
-        %{notification_settings: %{non_follows: false}} = user
+        %Activity{} = activity,
+        %User{notification_settings: %{non_follows: false}} = user
       ) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)
     !User.following?(user, followed)
   end
 
-  def skip?(:recently_followed, %{data: %{"type" => "Follow"}} = activity, user) do
+  # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL
+  def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do
     actor = activity.data["actor"]
 
     Notification.for_user(user)
diff --git a/lib/pleroma/thread_mute.ex b/lib/pleroma/thread_mute.ex
index cc815430a..a7ea13891 100644
--- a/lib/pleroma/thread_mute.ex
+++ b/lib/pleroma/thread_mute.ex
@@ -9,7 +9,8 @@ defmodule Pleroma.ThreadMute do
   alias Pleroma.ThreadMute
   alias Pleroma.User
 
-  require Ecto.Query
+  import Ecto.Changeset
+  import Ecto.Query
 
   schema "thread_mutes" do
     belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
@@ -18,19 +19,44 @@ defmodule Pleroma.ThreadMute do
 
   def changeset(mute, params \\ %{}) do
     mute
-    |> Ecto.Changeset.cast(params, [:user_id, :context])
-    |> Ecto.Changeset.foreign_key_constraint(:user_id)
-    |> Ecto.Changeset.unique_constraint(:user_id, name: :unique_index)
+    |> cast(params, [:user_id, :context])
+    |> foreign_key_constraint(:user_id)
+    |> unique_constraint(:user_id, name: :unique_index)
   end
 
   def query(user_id, context) do
     {:ok, user_id} = FlakeId.Ecto.CompatType.dump(user_id)
 
     ThreadMute
-    |> Ecto.Query.where(user_id: ^user_id)
-    |> Ecto.Query.where(context: ^context)
+    |> where(user_id: ^user_id)
+    |> where(context: ^context)
   end
 
+  def muters_query(context) do
+    ThreadMute
+    |> join(:inner, [tm], u in assoc(tm, :user))
+    |> where([tm], tm.context == ^context)
+    |> select([tm, u], u.ap_id)
+  end
+
+  def muter_ap_ids(context, ap_ids \\ nil)
+
+  # Note: applies to fake activities (ActivityPub.Utils.get_notified_from_object/1 etc.)
+  def muter_ap_ids(context, _ap_ids) when is_nil(context), do: []
+
+  def muter_ap_ids(context, ap_ids) do
+    context
+    |> muters_query()
+    |> maybe_filter_on_ap_id(ap_ids)
+    |> Repo.all()
+  end
+
+  defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do
+    where(query, [tm, u], u.ap_id in ^ap_ids)
+  end
+
+  defp maybe_filter_on_ap_id(query, _ap_ids), do: query
+
   def add_mute(user_id, context) do
     %ThreadMute{}
     |> changeset(%{user_id: user_id, context: context})
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 911dde6e2..4919c8e58 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -150,22 +150,26 @@ defmodule Pleroma.User do
            {outgoing_relation, outgoing_relation_target},
            {incoming_relation, incoming_relation_source}
          ]} <- @user_relationships_config do
-      # Definitions of `has_many :blocker_blocks`, `has_many :muter_mutes` etc.
+      # Definitions of `has_many` relations: :blocker_blocks, :muter_mutes, :reblog_muter_mutes,
+      #   :notification_muter_mutes, :subscribee_subscriptions
       has_many(outgoing_relation, UserRelationship,
         foreign_key: :source_id,
         where: [relationship_type: relationship_type]
       )
 
-      # Definitions of `has_many :blockee_blocks`, `has_many :mutee_mutes` etc.
+      # Definitions of `has_many` relations: :blockee_blocks, :mutee_mutes, :reblog_mutee_mutes,
+      #   :notification_mutee_mutes, :subscriber_subscriptions
       has_many(incoming_relation, UserRelationship,
         foreign_key: :target_id,
         where: [relationship_type: relationship_type]
       )
 
-      # Definitions of `has_many :blocked_users`, `has_many :muted_users` etc.
+      # Definitions of `has_many` relations: :blocked_users, :muted_users, :reblog_muted_users,
+      #   :notification_muted_users, :subscriber_users
       has_many(outgoing_relation_target, through: [outgoing_relation, :target])
 
-      # Definitions of `has_many :blocker_users`, `has_many :muter_users` etc.
+      # Definitions of `has_many` relations: :blocker_users, :muter_users, :reblog_muter_users,
+      #   :notification_muter_users, :subscribee_users
       has_many(incoming_relation_source, through: [incoming_relation, :source])
     end
 
@@ -185,7 +189,9 @@ defmodule Pleroma.User do
 
   for {_relationship_type, [{_outgoing_relation, outgoing_relation_target}, _]} <-
         @user_relationships_config do
-    # Definitions of `blocked_users_relation/1`, `muted_users_relation/1`, etc.
+    # `def blocked_users_relation/2`, `def muted_users_relation/2`,
+    #   `def reblog_muted_users_relation/2`, `def notification_muted_users/2`,
+    #   `def subscriber_users/2`
     def unquote(:"#{outgoing_relation_target}_relation")(user, restrict_deactivated? \\ false) do
       target_users_query = assoc(user, unquote(outgoing_relation_target))
 
@@ -196,7 +202,8 @@ defmodule Pleroma.User do
       end
     end
 
-    # Definitions of `blocked_users/1`, `muted_users/1`, etc.
+    # `def blocked_users/2`, `def muted_users/2`, `def reblog_muted_users/2`,
+    #   `def notification_muted_users/2`, `def subscriber_users/2`
     def unquote(outgoing_relation_target)(user, restrict_deactivated? \\ false) do
       __MODULE__
       |> apply(unquote(:"#{outgoing_relation_target}_relation"), [
@@ -206,7 +213,8 @@ defmodule Pleroma.User do
       |> Repo.all()
     end
 
-    # Definitions of `blocked_users_ap_ids/1`, `muted_users_ap_ids/1`, etc.
+    # `def blocked_users_ap_ids/2`, `def muted_users_ap_ids/2`, `def reblog_muted_users_ap_ids/2`,
+    #   `def notification_muted_users_ap_ids/2`, `def subscriber_users_ap_ids/2`
     def unquote(:"#{outgoing_relation_target}_ap_ids")(user, restrict_deactivated? \\ false) do
       __MODULE__
       |> apply(unquote(:"#{outgoing_relation_target}_relation"), [
@@ -1214,13 +1222,15 @@ defmodule Pleroma.User do
   end
 
   @doc """
-  Returns map of outgoing (blocked, muted etc.) relations' user AP IDs by relation type.
-  E.g. `outgoing_relations_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}`
+  Returns map of outgoing (blocked, muted etc.) relationships' user AP IDs by relation type.
+  E.g. `outgoing_relationships_ap_ids(user, [:block])` -> `%{block: ["https://some.site/users/userapid"]}`
   """
-  @spec outgoing_relations_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())}
-  def outgoing_relations_ap_ids(_, []), do: %{}
+  @spec outgoing_relationships_ap_ids(User.t(), list(atom())) :: %{atom() => list(String.t())}
+  def outgoing_relationships_ap_ids(_user, []), do: %{}
 
-  def outgoing_relations_ap_ids(%User{} = user, relationship_types)
+  def outgoing_relationships_ap_ids(nil, _relationship_types), do: %{}
+
+  def outgoing_relationships_ap_ids(%User{} = user, relationship_types)
       when is_list(relationship_types) do
     db_result =
       user
@@ -1239,6 +1249,30 @@ defmodule Pleroma.User do
     )
   end
 
+  def incoming_relationships_ungrouped_ap_ids(user, relationship_types, ap_ids \\ nil)
+
+  def incoming_relationships_ungrouped_ap_ids(_user, [], _ap_ids), do: []
+
+  def incoming_relationships_ungrouped_ap_ids(nil, _relationship_types, _ap_ids), do: []
+
+  def incoming_relationships_ungrouped_ap_ids(%User{} = user, relationship_types, ap_ids)
+      when is_list(relationship_types) do
+    user
+    |> assoc(:incoming_relationships)
+    |> join(:inner, [user_rel], u in assoc(user_rel, :source))
+    |> where([user_rel, u], user_rel.relationship_type in ^relationship_types)
+    |> maybe_filter_on_ap_id(ap_ids)
+    |> select([user_rel, u], u.ap_id)
+    |> distinct(true)
+    |> Repo.all()
+  end
+
+  defp maybe_filter_on_ap_id(query, ap_ids) when is_list(ap_ids) do
+    where(query, [user_rel, u], u.ap_id in ^ap_ids)
+  end
+
+  defp maybe_filter_on_ap_id(query, _ap_ids), do: query
+
   def deactivate_async(user, status \\ true) do
     BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status})
   end
diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex
index 393947942..01b6ace9d 100644
--- a/lib/pleroma/user_relationship.ex
+++ b/lib/pleroma/user_relationship.ex
@@ -21,15 +21,18 @@ defmodule Pleroma.UserRelationship do
   end
 
   for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do
-    # Definitions of `create_block/2`, `create_mute/2` etc.
+    # `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`,
+    #   `def create_notification_mute/2`, `def create_inverse_subscription/2`
     def unquote(:"create_#{relationship_type}")(source, target),
       do: create(unquote(relationship_type), source, target)
 
-    # Definitions of `delete_block/2`, `delete_mute/2` etc.
+    # `def delete_block/2`, `def delete_mute/2`, `def delete_reblog_mute/2`,
+    #   `def delete_notification_mute/2`, `def delete_inverse_subscription/2`
     def unquote(:"delete_#{relationship_type}")(source, target),
       do: delete(unquote(relationship_type), source, target)
 
-    # Definitions of `block_exists?/2`, `mute_exists?/2` etc.
+    # `def block_exists?/2`, `def mute_exists?/2`, `def reblog_mute_exists?/2`,
+    #   `def notification_mute_exists?/2`, `def inverse_subscription_exists?/2`
     def unquote(:"#{relationship_type}_exists?")(source, target),
       do: exists?(unquote(relationship_type), source, target)
   end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index d9f74b6a4..60e74758f 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1230,17 +1230,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp fetch_activities_query_ap_ids_ops(opts) do
     source_user = opts["muting_user"]
-    ap_id_relations = if source_user, do: [:mute, :reblog_mute], else: []
+    ap_id_relationships = if source_user, do: [:mute, :reblog_mute], else: []
 
-    ap_id_relations =
-      ap_id_relations ++
+    ap_id_relationships =
+      ap_id_relationships ++
         if opts["blocking_user"] && opts["blocking_user"] == source_user do
           [:block]
         else
           []
         end
 
-    preloaded_ap_ids = User.outgoing_relations_ap_ids(source_user, ap_id_relations)
+    preloaded_ap_ids = User.outgoing_relationships_ap_ids(source_user, ap_id_relationships)
 
     restrict_blocked_opts = Map.merge(%{"blocked_users_ap_ids" => preloaded_ap_ids[:block]}, opts)
     restrict_muted_opts = Map.merge(%{"muted_users_ap_ids" => preloaded_ap_ids[:mute]}, opts)
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 9cd3de705..d6549a932 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -1108,13 +1108,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def add_mention_tags(object) do
-    mentions =
-      object
-      |> Utils.get_notified_from_object()
-      |> Enum.map(&build_mention_tag/1)
+    {enabled_receivers, disabled_receivers} = Utils.get_notified_from_object(object)
+    potential_receivers = enabled_receivers ++ disabled_receivers
+    mentions = Enum.map(potential_receivers, &build_mention_tag/1)
 
     tags = object["tag"] || []
-
     Map.put(object, "tag", tags ++ mentions)
   end
 
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 88c997b9f..9d83a9fc1 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -63,11 +63,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
     when action != :create
   )
 
-  @relations [:follow, :unfollow]
+  @relationship_actions [:follow, :unfollow]
   @needs_account ~W(followers following lists follow unfollow mute unmute block unblock)a
 
-  plug(RateLimiter, [name: :relations_id_action, params: ["id", "uri"]] when action in @relations)
-  plug(RateLimiter, [name: :relations_actions] when action in @relations)
+  plug(
+    RateLimiter,
+    [name: :relation_id_action, params: ["id", "uri"]] when action in @relationship_actions
+  )
+
+  plug(RateLimiter, [name: :relations_actions] when action in @relationship_actions)
   plug(RateLimiter, [name: :app_account_creation] when action == :create)
   plug(:assign_account_by_id when action in @needs_account)
 
diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex
index 29f992a67..abfed21c8 100644
--- a/lib/pleroma/web/streamer/worker.ex
+++ b/lib/pleroma/web/streamer/worker.ex
@@ -130,7 +130,7 @@ defmodule Pleroma.Web.Streamer.Worker do
 
   defp should_send?(%User{} = user, %Activity{} = item) do
     %{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} =
-      User.outgoing_relations_ap_ids(user, [:block, :mute, :reblog_mute])
+      User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute])
 
     recipient_blocks = MapSet.new(blocked_ap_ids ++ muted_ap_ids)
     recipients = MapSet.new(item.recipients)
-- 
cgit v1.2.3


From 10b7b2b4a42773c5ab0f69379418728da68b5ce0 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Tue, 24 Mar 2020 19:10:32 +0000
Subject: Merge branch 'fix/activity-deletion' into 'develop'

Fix activity deletion

Closes #1640

See merge request pleroma/pleroma!2328
---
 lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++++++
 1 file changed, 10 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 60e74758f..eaaaba775 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -584,6 +584,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  defp do_delete(%Object{data: %{"type" => "Tombstone", "id" => ap_id}}, _) do
+    activity =
+      ap_id
+      |> Activity.Queries.by_object_id()
+      |> Activity.Queries.by_type("Delete")
+      |> Repo.one()
+
+    {:ok, activity}
+  end
+
   @spec block(User.t(), User.t(), String.t() | nil, boolean()) ::
           {:ok, Activity.t()} | {:error, any()}
   def block(blocker, blocked, activity_id \\ nil, local \\ true) do
-- 
cgit v1.2.3


From c4d3ccc7b952445069ed3fc8586da14af6db937d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 24 Mar 2020 17:34:13 +0000
Subject: Merge branch 'admin-api-change-password' into 'develop'

Admin API: `PATCH /api/pleroma/admin/users/:nickname/update_credentials`

See merge request pleroma/pleroma!2149
---
 lib/pleroma/moderation_log.ex                      | 11 +++
 lib/pleroma/user.ex                                | 86 +++++++++++++++++++++-
 lib/pleroma/web/admin_api/admin_api_controller.ex  | 51 ++++++++++++-
 lib/pleroma/web/admin_api/views/account_view.ex    | 40 ++++++++++
 .../mastodon_api/controllers/account_controller.ex | 60 +++------------
 lib/pleroma/web/router.ex                          |  2 +
 6 files changed, 197 insertions(+), 53 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex
index e32895f70..7aacd9d80 100644
--- a/lib/pleroma/moderation_log.ex
+++ b/lib/pleroma/moderation_log.ex
@@ -605,6 +605,17 @@ defmodule Pleroma.ModerationLog do
     }"
   end
 
+  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "updated_users",
+          "subject" => subjects
+        }
+      }) do
+    "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 4919c8e58..343bc27f5 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -425,9 +425,55 @@ defmodule Pleroma.User do
     |> validate_format(:nickname, local_nickname_regex())
     |> validate_length(:bio, max: bio_limit)
     |> validate_length(:name, min: 1, max: name_limit)
+    |> put_fields()
+    |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)})
+    |> put_change_if_present(:avatar, &put_upload(&1, :avatar))
+    |> put_change_if_present(:banner, &put_upload(&1, :banner))
+    |> put_change_if_present(:background, &put_upload(&1, :background))
+    |> put_change_if_present(
+      :pleroma_settings_store,
+      &{:ok, Map.merge(struct.pleroma_settings_store, &1)}
+    )
     |> validate_fields(false)
   end
 
+  defp put_fields(changeset) do
+    if raw_fields = get_change(changeset, :raw_fields) do
+      raw_fields =
+        raw_fields
+        |> Enum.filter(fn %{"name" => n} -> n != "" end)
+
+      fields =
+        raw_fields
+        |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end)
+
+      changeset
+      |> put_change(:raw_fields, raw_fields)
+      |> put_change(:fields, fields)
+    else
+      changeset
+    end
+  end
+
+  defp put_change_if_present(changeset, map_field, value_function) do
+    if value = get_change(changeset, map_field) do
+      with {:ok, new_value} <- value_function.(value) do
+        put_change(changeset, map_field, new_value)
+      else
+        _ -> changeset
+      end
+    else
+      changeset
+    end
+  end
+
+  defp put_upload(value, type) do
+    with %Plug.Upload{} <- value,
+         {:ok, object} <- ActivityPub.upload(value, type: type) do
+      {:ok, object.data}
+    end
+  end
+
   def upgrade_changeset(struct, params \\ %{}, remote? \\ false) do
     bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000)
     name_limit = Pleroma.Config.get([:instance, :user_name_length], 100)
@@ -471,6 +517,27 @@ defmodule Pleroma.User do
     |> validate_fields(remote?)
   end
 
+  def update_as_admin_changeset(struct, params) do
+    struct
+    |> update_changeset(params)
+    |> cast(params, [:email])
+    |> delete_change(:also_known_as)
+    |> unique_constraint(:email)
+    |> validate_format(:email, @email_regex)
+  end
+
+  @spec update_as_admin(%User{}, map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
+  def update_as_admin(user, params) do
+    params = Map.put(params, "password_confirmation", params["password"])
+    changeset = update_as_admin_changeset(user, params)
+
+    if params["password"] do
+      reset_password(user, changeset, params)
+    else
+      User.update_and_set_cache(changeset)
+    end
+  end
+
   def password_update_changeset(struct, params) do
     struct
     |> cast(params, [:password, :password_confirmation])
@@ -481,10 +548,14 @@ defmodule Pleroma.User do
   end
 
   @spec reset_password(User.t(), map) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()}
-  def reset_password(%User{id: user_id} = user, data) do
+  def reset_password(%User{} = user, params) do
+    reset_password(user, user, params)
+  end
+
+  def reset_password(%User{id: user_id} = user, struct, params) do
     multi =
       Multi.new()
-      |> Multi.update(:user, password_update_changeset(user, data))
+      |> Multi.update(:user, password_update_changeset(struct, params))
       |> Multi.delete_all(:tokens, OAuth.Token.Query.get_by_user(user_id))
       |> Multi.delete_all(:auth, OAuth.Authorization.delete_by_user_query(user))
 
@@ -1890,6 +1961,17 @@ defmodule Pleroma.User do
 
   def fields(%{fields: fields}), do: fields
 
+  def sanitized_fields(%User{} = user) do
+    user
+    |> User.fields()
+    |> Enum.map(fn %{"name" => name, "value" => value} ->
+      %{
+        "name" => name,
+        "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)
+      }
+    end)
+  end
+
   def validate_fields(changeset, remote? \\ false) do
     limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields
     limit = Pleroma.Config.get([:instance, limit_name], 0)
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 47b7d2da3..6c88549f5 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -38,7 +38,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   plug(
     OAuthScopesPlug,
     %{scopes: ["read:accounts"], admin: true}
-    when action in [:list_users, :user_show, :right_get]
+    when action in [:list_users, :user_show, :right_get, :show_user_credentials]
   )
 
   plug(
@@ -54,7 +54,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            :tag_users,
            :untag_users,
            :right_add,
-           :right_delete
+           :right_delete,
+           :update_user_credentials
          ]
   )
 
@@ -658,6 +659,52 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     json_response(conn, :no_content, "")
   end
 
+  @doc "Show a given user's credentials"
+  def show_user_credentials(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do
+      conn
+      |> put_view(AccountView)
+      |> render("credentials.json", %{user: user, for: admin})
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
+  @doc "Updates a given user"
+  def update_user_credentials(
+        %{assigns: %{user: admin}} = conn,
+        %{"nickname" => nickname} = params
+      ) do
+    with {_, user} <- {:user, User.get_cached_by_nickname(nickname)},
+         {:ok, _user} <-
+           User.update_as_admin(user, params) do
+      ModerationLog.insert_log(%{
+        actor: admin,
+        subject: [user],
+        action: "updated_users"
+      })
+
+      if params["password"] do
+        User.force_password_reset_async(user)
+      end
+
+      ModerationLog.insert_log(%{
+        actor: admin,
+        subject: [user],
+        action: "force_password_reset"
+      })
+
+      json(conn, %{status: "success"})
+    else
+      {:error, changeset} ->
+        {_, {error, _}} = Enum.at(changeset.errors, 0)
+        json(conn, %{error: "New password #{error}."})
+
+      _ ->
+        json(conn, %{error: "Unable to change password."})
+    end
+  end
+
   def list_reports(conn, params) do
     {page, page_size} = page_params(params)
 
diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index 1e03849de..a16a3ebf0 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -23,6 +23,43 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
     }
   end
 
+  def render("credentials.json", %{user: user, for: for_user}) do
+    user = User.sanitize_html(user, User.html_filter_policy(for_user))
+    avatar = User.avatar_url(user) |> MediaProxy.url()
+    banner = User.banner_url(user) |> MediaProxy.url()
+    background = image_url(user.background) |> MediaProxy.url()
+
+    user
+    |> Map.take([
+      :id,
+      :bio,
+      :email,
+      :fields,
+      :name,
+      :nickname,
+      :locked,
+      :no_rich_text,
+      :default_scope,
+      :hide_follows,
+      :hide_followers_count,
+      :hide_follows_count,
+      :hide_followers,
+      :hide_favorites,
+      :allow_following_move,
+      :show_role,
+      :skip_thread_containment,
+      :pleroma_settings_store,
+      :raw_fields,
+      :discoverable,
+      :actor_type
+    ])
+    |> Map.merge(%{
+      "avatar" => avatar,
+      "banner" => banner,
+      "background" => background
+    })
+  end
+
   def render("show.json", %{user: user}) do
     avatar = User.avatar_url(user) |> MediaProxy.url()
     display_name = Pleroma.HTML.strip_tags(user.name || user.nickname)
@@ -104,4 +141,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
         ""
     end
   end
+
+  defp image_url(%{"url" => [%{"href" => href} | _]}), do: href
+  defp image_url(_), do: nil
 end
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 9d83a9fc1..73853c1e4 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   import Pleroma.Web.ControllerHelper,
     only: [add_link_headers: 2, truthy_param?: 1, assign_account_by_id: 2, json_response: 3]
 
-  alias Pleroma.Emoji
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Plugs.RateLimiter
   alias Pleroma.User
@@ -144,17 +143,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   def update_credentials(%{assigns: %{user: original_user}} = conn, params) do
     user = original_user
 
-    params =
-      if Map.has_key?(params, "fields_attributes") do
-        Map.update!(params, "fields_attributes", fn fields ->
-          fields
-          |> normalize_fields_attributes()
-          |> Enum.filter(fn %{"name" => n} -> n != "" end)
-        end)
-      else
-        params
-      end
-
     user_params =
       [
         :no_rich_text,
@@ -173,46 +161,20 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
         add_if_present(acc, params, to_string(key), key, &{:ok, truthy_param?(&1)})
       end)
       |> add_if_present(params, "display_name", :name)
-      |> add_if_present(params, "note", :bio, fn value -> {:ok, User.parse_bio(value, user)} end)
-      |> add_if_present(params, "avatar", :avatar, fn value ->
-        with %Plug.Upload{} <- value,
-             {:ok, object} <- ActivityPub.upload(value, type: :avatar) do
-          {:ok, object.data}
-        end
-      end)
-      |> add_if_present(params, "header", :banner, fn value ->
-        with %Plug.Upload{} <- value,
-             {:ok, object} <- ActivityPub.upload(value, type: :banner) do
-          {:ok, object.data}
-        end
-      end)
-      |> add_if_present(params, "pleroma_background_image", :background, fn value ->
-        with %Plug.Upload{} <- value,
-             {:ok, object} <- ActivityPub.upload(value, type: :background) do
-          {:ok, object.data}
-        end
-      end)
-      |> add_if_present(params, "fields_attributes", :fields, fn fields ->
-        fields = Enum.map(fields, fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end)
-
-        {:ok, fields}
-      end)
-      |> add_if_present(params, "fields_attributes", :raw_fields)
-      |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store, fn value ->
-        {:ok, Map.merge(user.pleroma_settings_store, value)}
-      end)
+      |> add_if_present(params, "note", :bio)
+      |> add_if_present(params, "avatar", :avatar)
+      |> add_if_present(params, "header", :banner)
+      |> add_if_present(params, "pleroma_background_image", :background)
+      |> add_if_present(
+        params,
+        "fields_attributes",
+        :raw_fields,
+        &{:ok, normalize_fields_attributes(&1)}
+      )
+      |> add_if_present(params, "pleroma_settings_store", :pleroma_settings_store)
       |> add_if_present(params, "default_scope", :default_scope)
       |> add_if_present(params, "actor_type", :actor_type)
 
-    emojis_text = (user_params["display_name"] || "") <> (user_params["note"] || "")
-
-    user_emojis =
-      user
-      |> Map.get(:emoji, [])
-      |> Enum.concat(Emoji.Formatter.get_emoji_map(emojis_text))
-      |> Enum.dedup()
-
-    user_params = Map.put(user_params, :emoji, user_emojis)
     changeset = User.update_changeset(user, user_params)
 
     with {:ok, user} <- User.update_and_set_cache(changeset) do
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 980242c68..cb590acfb 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -173,6 +173,8 @@ defmodule Pleroma.Web.Router do
 
     get("/users/:nickname/password_reset", AdminAPIController, :get_password_reset)
     patch("/users/force_password_reset", AdminAPIController, :force_password_reset)
+    get("/users/:nickname/credentials", AdminAPIController, :show_user_credentials)
+    patch("/users/:nickname/credentials", AdminAPIController, :update_user_credentials)
 
     get("/users", AdminAPIController, :list_users)
     get("/users/:nickname", AdminAPIController, :user_show)
-- 
cgit v1.2.3


From e99e2a86afd50c8a7d0f243e1e4017f06ec75441 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 23 Mar 2020 19:32:01 +0000
Subject: Merge branch 'bugfix/profile-bio-newline' into 'develop'

AccountView: fix for other forms of <br> in bio

Closes #1643

See merge request pleroma/pleroma!2322
---
 lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 341dc2c91..a234e7028 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -122,7 +122,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       fields: user.fields,
       bot: bot,
       source: %{
-        note: Pleroma.HTML.strip_tags((user.bio || "") |> String.replace("<br>", "\n")),
+        note: (user.bio || "") |> String.replace(~r(<br */?>), "\n") |> Pleroma.HTML.strip_tags(),
         sensitive: false,
         fields: user.raw_fields,
         pleroma: %{
-- 
cgit v1.2.3


From 01a3f145d51fe7aeb9050c89e03bebf10793aba3 Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Tue, 31 Mar 2020 22:04:02 +0000
Subject: Merge branch 'bugfix/funkwhale-channel' into 'develop'

Fix profile url for funkwhale channels, removes one source_data use

Closes #1653

See merge request pleroma/pleroma!2333
---
 lib/pleroma/user.ex                                         |  6 +-----
 lib/pleroma/web/activity_pub/activity_pub.ex                | 13 +++++++++++++
 lib/pleroma/web/mastodon_api/views/account_view.ex          |  4 ++--
 lib/pleroma/web/metadata/opengraph.ex                       |  2 +-
 .../web/templates/static_fe/static_fe/_user_card.html.eex   |  2 +-
 .../web/templates/static_fe/static_fe/profile.html.eex      |  2 +-
 6 files changed, 19 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 343bc27f5..faba3eb52 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -276,16 +276,12 @@ defmodule Pleroma.User do
     end
   end
 
-  def profile_url(%User{source_data: %{"url" => url}}), do: url
-  def profile_url(%User{ap_id: ap_id}), do: ap_id
-  def profile_url(_), do: nil
-
   def ap_id(%User{nickname: nickname}), do: "#{Web.base_url()}/users/#{nickname}"
 
   def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa
   def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers"
 
-  @spec ap_following(User.t()) :: Sring.t()
+  @spec ap_following(User.t()) :: String.t()
   def ap_following(%User{following_address: fa}) when is_binary(fa), do: fa
   def ap_following(%User{} = user), do: "#{ap_id(user)}/following"
 
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index eaaaba775..5f895406d 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1380,6 +1380,18 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
+  @spec get_actor_url(any()) :: binary() | nil
+  defp get_actor_url(url) when is_binary(url), do: url
+  defp get_actor_url(%{"href" => href}) when is_binary(href), do: href
+
+  defp get_actor_url(url) when is_list(url) do
+    url
+    |> List.first()
+    |> get_actor_url()
+  end
+
+  defp get_actor_url(_url), do: nil
+
   defp object_to_user_data(data) do
     avatar =
       data["icon"]["url"] &&
@@ -1409,6 +1421,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
     user_data = %{
       ap_id: data["id"],
+      uri: get_actor_url(data["url"]),
       ap_enabled: true,
       source_data: data,
       banner: banner,
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index a234e7028..6ff84c957 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       id: to_string(user.id),
       acct: user.nickname,
       username: username_from_nickname(user.nickname),
-      url: User.profile_url(user)
+      url: user.uri || user.ap_id
     }
   end
 
@@ -113,7 +113,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       following_count: following_count,
       statuses_count: user.note_count,
       note: user.bio || "",
-      url: User.profile_url(user),
+      url: user.uri || user.ap_id,
       avatar: image,
       avatar_static: image,
       header: header,
diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex
index 21446ac77..68c871e71 100644
--- a/lib/pleroma/web/metadata/opengraph.ex
+++ b/lib/pleroma/web/metadata/opengraph.ex
@@ -68,7 +68,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do
            property: "og:title",
            content: Utils.user_name_string(user)
          ], []},
-        {:meta, [property: "og:url", content: User.profile_url(user)], []},
+        {:meta, [property: "og:url", content: user.uri || user.ap_id], []},
         {:meta, [property: "og:description", content: truncated_bio], []},
         {:meta, [property: "og:type", content: "website"], []},
         {:meta, [property: "og:image", content: Utils.attachment_url(User.avatar_url(user))], []},
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex
index c7789f9ac..2a7582d45 100644
--- a/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex
+++ b/lib/pleroma/web/templates/static_fe/static_fe/_user_card.html.eex
@@ -1,5 +1,5 @@
 <div class="p-author h-card">
-  <a class="u-url" rel="author noopener" href="<%= User.profile_url(@user) %>">
+  <a class="u-url" rel="author noopener" href="<%= (@user.uri || @user.ap_id) %>">
     <div class="avatar">
       <img src="<%= User.avatar_url(@user) |> MediaProxy.url %>" width="48" height="48" alt="">
     </div>
diff --git a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex
index 94063c92d..e7d2aecad 100644
--- a/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex
+++ b/lib/pleroma/web/templates/static_fe/static_fe/profile.html.eex
@@ -8,7 +8,7 @@
       <button type="submit" class="collapse">Remote follow</button>
     </form>
     <%= raw Formatter.emojify(@user.name, emoji_for_user(@user)) %> |
-    <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: User.profile_url(@user) %>
+    <%= link "@#{@user.nickname}@#{Endpoint.host()}", to: (@user.uri || @user.ap_id) %>
   </h3>
   <p><%= raw @user.bio %></p>
 </header>
-- 
cgit v1.2.3


From b0a9a02af3edb28002b633030b3f6d63ae2309bf Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Sun, 29 Mar 2020 19:18:22 +0000
Subject: Merge branch 'feature/funkwhale-audio' into 'develop'

Add support for funkwhale Audio activity

Closes #764 and #1624

See merge request pleroma/pleroma!2287
---
 lib/pleroma/web/activity_pub/transmogrifier.ex    | 5 +++--
 lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index d6549a932..09bd9a442 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -229,7 +229,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     Map.put(object, "url", url["href"])
   end
 
-  def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
+  def fix_url(%{"type" => object_type, "url" => url} = object)
+      when object_type in ["Video", "Audio"] and is_list(url) do
     first_element = Enum.at(url, 0)
 
     link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end)
@@ -398,7 +399,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer"] do
+      when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do
     actor = Containment.get_actor(data)
 
     data =
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index f7469cdff..a042075f5 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -421,7 +421,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   end
 
   def render_content(%{data: %{"type" => object_type}} = object)
-      when object_type in ["Video", "Event"] do
+      when object_type in ["Video", "Event", "Audio"] do
     with name when not is_nil(name) and name != "" <- object.data["name"] do
       "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}"
     else
-- 
cgit v1.2.3


From 94240cac4eb229538020a055cf6fec2b7cb0761f Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 7 Apr 2020 11:41:10 +0000
Subject: Merge branch 'fix-auto-link-for-profile-fields' into 'develop'

Use Pleroma.Formatter.linkify/2 instead

See merge request pleroma/pleroma!2352
---
 lib/pleroma/user.ex | 20 ++++++++------------
 1 file changed, 8 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index faba3eb52..0115abed5 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.User do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Delivery
   alias Pleroma.FollowingRelationship
+  alias Pleroma.Formatter
   alias Pleroma.HTML
   alias Pleroma.Keys
   alias Pleroma.Notification
@@ -441,7 +442,7 @@ defmodule Pleroma.User do
 
       fields =
         raw_fields
-        |> Enum.map(fn f -> Map.update!(f, "value", &AutoLinker.link(&1)) end)
+        |> Enum.map(fn f -> Map.update!(f, "value", &parse_fields(&1)) end)
 
       changeset
       |> put_change(:raw_fields, raw_fields)
@@ -451,6 +452,12 @@ defmodule Pleroma.User do
     end
   end
 
+  defp parse_fields(value) do
+    value
+    |> Formatter.linkify(mentions_format: :full)
+    |> elem(0)
+  end
+
   defp put_change_if_present(changeset, map_field, value_function) do
     if value = get_change(changeset, map_field) do
       with {:ok, new_value} <- value_function.(value) do
@@ -1957,17 +1964,6 @@ defmodule Pleroma.User do
 
   def fields(%{fields: fields}), do: fields
 
-  def sanitized_fields(%User{} = user) do
-    user
-    |> User.fields()
-    |> Enum.map(fn %{"name" => name, "value" => value} ->
-      %{
-        "name" => name,
-        "value" => Pleroma.HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly)
-      }
-    end)
-  end
-
   def validate_fields(changeset, remote? \\ false) do
     limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields
     limit = Pleroma.Config.get([:instance, limit_name], 0)
-- 
cgit v1.2.3


From 61889e00fc4a77e92ed7af3b6a270d10d5b2f34b Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 29 Apr 2020 14:26:31 +0300
Subject: Deactivate local users on deletion instead of deleting the record

Prevents the possibility of re-registration, which allowed to read
DMs of the deleted account.

Also includes a migration that tries to find any already deleted
accounts and insert skeletons for them.

Closes pleroma/pleroma#1687
---
 lib/pleroma/user.ex                                           | 11 +++++++++--
 .../web/pleroma_api/controllers/pleroma_api_controller.ex     |  5 ++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 0115abed5..0e5121694 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1416,8 +1416,15 @@ defmodule Pleroma.User do
     end)
 
     delete_user_activities(user)
-    invalidate_cache(user)
-    Repo.delete(user)
+
+    if user.local do
+      user
+      |> change(%{deactivated: true, email: nil})
+      |> update_and_set_cache()
+    else
+      invalidate_cache(user)
+      Repo.delete(user)
+    end
   end
 
   def perform(:deactivate_async, user, status), do: deactivate(user, status)
diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
index dae7f0f2f..41677d04d 100644
--- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
@@ -53,7 +53,10 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
           else
             users =
               Enum.map(user_ap_ids, &User.get_cached_by_ap_id/1)
-              |> Enum.filter(& &1)
+              |> Enum.filter(fn
+                %{deactivated: false} -> true
+                _ -> false
+              end)
 
             %{
               name: emoji,
-- 
cgit v1.2.3


From 78391a00c579696b1555bbe5373e128b932cfc3f Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Mon, 27 Apr 2020 17:41:38 +0300
Subject: Mastodon API: do not create a following relationship if the
 corresponding follow request doesn't exist when calling `POST
 /api/v1/follow_requests/:id/authorize`

---
 lib/pleroma/web/common_api/common_api.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 091011c6b..511cae63f 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -39,8 +39,8 @@ defmodule Pleroma.Web.CommonAPI do
   end
 
   def accept_follow_request(follower, followed) do
-    with {:ok, follower} <- User.follow(follower, followed),
-         %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
+    with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
+         {:ok, follower} <- User.follow(follower, followed),
          {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
          {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "accept"),
          {:ok, _activity} <-
-- 
cgit v1.2.3


From 6e0b046771304bb8b733cebecc20a31a4df5e616 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Sun, 26 Apr 2020 00:28:57 -0500
Subject: Let blob: pass CSP

---
 lib/pleroma/plugs/http_security_plug.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex
index 81e6b4f2a..6462797b6 100644
--- a/lib/pleroma/plugs/http_security_plug.ex
+++ b/lib/pleroma/plugs/http_security_plug.ex
@@ -75,7 +75,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
       "default-src 'none'",
       "base-uri 'self'",
       "frame-ancestors 'none'",
-      "img-src 'self' data: https:",
+      "img-src 'self' data: blob: https:",
       "media-src 'self' https:",
       "style-src 'self' 'unsafe-inline'",
       "font-src 'self'",
-- 
cgit v1.2.3


From 2e58fe08cd5035e66d34c87f70588f8a307d51ed Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 23 Apr 2020 13:33:30 +0200
Subject: CommonAPI: Don't make repeating announces possible

---
 lib/pleroma/web/common_api/common_api.ex | 27 +++++++++++++++------------
 lib/pleroma/web/common_api/utils.ex      | 18 ------------------
 2 files changed, 15 insertions(+), 30 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 511cae63f..b9fa9fe3a 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -83,8 +83,9 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  def repeat(id_or_ap_id, user, params \\ %{}) do
-    with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)},
+  def repeat(id, user, params \\ %{}) do
+    with {_, %Activity{data: %{"type" => "Create"}} = activity} <-
+           {:find_activity, Activity.get_by_id(id)},
          object <- Object.normalize(activity),
          announce_activity <- Utils.get_existing_announce(user.ap_id, object),
          public <- public_announce?(object, params) do
@@ -99,8 +100,9 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  def unrepeat(id_or_ap_id, user) do
-    with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)} do
+  def unrepeat(id, user) do
+    with {_, %Activity{data: %{"type" => "Create"}} = activity} <-
+           {:find_activity, Activity.get_by_id(id)} do
       object = Object.normalize(activity)
       ActivityPub.unannounce(user, object)
     else
@@ -109,8 +111,8 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  def favorite(id_or_ap_id, user) do
-    with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)},
+  def favorite(id, user) do
+    with {_, %Activity{} = activity} <- {:find_activity, Activity.get_by_id(id)},
          object <- Object.normalize(activity),
          like_activity <- Utils.get_existing_like(user.ap_id, object) do
       if like_activity do
@@ -124,8 +126,9 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  def unfavorite(id_or_ap_id, user) do
-    with {_, %Activity{} = activity} <- {:find_activity, get_by_id_or_ap_id(id_or_ap_id)} do
+  def unfavorite(id, user) do
+    with {_, %Activity{data: %{"type" => "Create"}} = activity} <-
+           {:find_activity, Activity.get_by_id(id)} do
       object = Object.normalize(activity)
       ActivityPub.unlike(user, object)
     else
@@ -316,12 +319,12 @@ defmodule Pleroma.Web.CommonAPI do
     })
   end
 
-  def pin(id_or_ap_id, %{ap_id: user_ap_id} = user) do
+  def pin(id, %{ap_id: user_ap_id} = user) do
     with %Activity{
            actor: ^user_ap_id,
            data: %{"type" => "Create"},
            object: %Object{data: %{"type" => object_type}}
-         } = activity <- get_by_id_or_ap_id(id_or_ap_id),
+         } = activity <- Activity.get_by_id_with_object(id),
          true <- object_type in ["Note", "Article", "Question"],
          true <- Visibility.is_public?(activity),
          {:ok, _user} <- User.add_pinnned_activity(user, activity) do
@@ -332,8 +335,8 @@ defmodule Pleroma.Web.CommonAPI do
     end
   end
 
-  def unpin(id_or_ap_id, user) do
-    with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
+  def unpin(id, user) do
+    with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id),
          {:ok, _user} <- User.remove_pinnned_activity(user, activity) do
       {:ok, activity}
     else
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 635e7cd38..26dcd463c 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -24,24 +24,6 @@ defmodule Pleroma.Web.CommonAPI.Utils do
   require Logger
   require Pleroma.Constants
 
-  # This is a hack for twidere.
-  def get_by_id_or_ap_id(id) do
-    activity =
-      with true <- FlakeId.flake_id?(id),
-           %Activity{} = activity <- Activity.get_by_id_with_object(id) do
-        activity
-      else
-        _ -> Activity.get_create_by_object_ap_id_with_object(id)
-      end
-
-    activity &&
-      if activity.data["type"] == "Create" do
-        activity
-      else
-        Activity.get_create_by_object_ap_id_with_object(activity.data["object"])
-      end
-  end
-
   def attachments_from_ids(%{"media_ids" => ids, "descriptions" => desc} = _) do
     attachments_from_ids_descs(ids, desc)
   end
-- 
cgit v1.2.3


From bf4b5f385631840a0ae25c9214a4e2d39413b1fc Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Wed, 22 Apr 2020 15:46:04 +0000
Subject: Merge branch 'fix/follow-and-blocks-import' into 'develop'

Fix follower/blocks import when nicknames starts with @

Closes #1698

See merge request pleroma/pleroma!2416
---
 .../web/twitter_api/controllers/util_controller.ex | 26 +++++++++++-----------
 lib/pleroma/workers/background_worker.ex           |  4 ++--
 2 files changed, 15 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index bca0e26eb..1873d78df 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -197,15 +197,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   end
 
   def follow_import(%{assigns: %{user: follower}} = conn, %{"list" => list}) do
-    with lines <- String.split(list, "\n"),
-         followed_identifiers <-
-           Enum.map(lines, fn line ->
-             String.split(line, ",") |> List.first()
-           end)
-           |> List.delete("Account address") do
-      User.follow_import(follower, followed_identifiers)
-      json(conn, "job started")
-    end
+    followed_identifiers =
+      list
+      |> String.split("\n")
+      |> Enum.map(&(&1 |> String.split(",") |> List.first()))
+      |> List.delete("Account address")
+      |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@")))
+      |> Enum.reject(&(&1 == ""))
+
+    User.follow_import(follower, followed_identifiers)
+    json(conn, "job started")
   end
 
   def blocks_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
@@ -213,10 +214,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
   end
 
   def blocks_import(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do
-    with blocked_identifiers <- String.split(list) do
-      User.blocks_import(blocker, blocked_identifiers)
-      json(conn, "job started")
-    end
+    blocked_identifiers = list |> String.split() |> Enum.map(&String.trim_leading(&1, "@"))
+    User.blocks_import(blocker, blocked_identifiers)
+    json(conn, "job started")
   end
 
   def change_password(%{assigns: %{user: user}} = conn, params) do
diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex
index 0f8ece2c4..57c3a9c3a 100644
--- a/lib/pleroma/workers/background_worker.ex
+++ b/lib/pleroma/workers/background_worker.ex
@@ -35,7 +35,7 @@ defmodule Pleroma.Workers.BackgroundWorker do
         _job
       ) do
     blocker = User.get_cached_by_id(blocker_id)
-    User.perform(:blocks_import, blocker, blocked_identifiers)
+    {:ok, User.perform(:blocks_import, blocker, blocked_identifiers)}
   end
 
   def perform(
@@ -47,7 +47,7 @@ defmodule Pleroma.Workers.BackgroundWorker do
         _job
       ) do
     follower = User.get_cached_by_id(follower_id)
-    User.perform(:follow_import, follower, followed_identifiers)
+    {:ok, User.perform(:follow_import, follower, followed_identifiers)}
   end
 
   def perform(%{"op" => "media_proxy_preload", "message" => message}, _job) do
-- 
cgit v1.2.3


From 2ff3b85326cdf42e5fc77c8b06836e0a6b97267f Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Wed, 22 Apr 2020 17:37:09 +0000
Subject: Merge branch 'bugfix/1670-user-count' into 'develop'

Stats: Ignore internal users for user count.

Closes #1670

See merge request pleroma/pleroma!2414
---
 lib/pleroma/stats.ex | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex
index 4446562ac..8d2809bbb 100644
--- a/lib/pleroma/stats.ex
+++ b/lib/pleroma/stats.ex
@@ -45,11 +45,11 @@ defmodule Pleroma.Stats do
   end
 
   def init(_args) do
-    {:ok, get_stat_data()}
+    {:ok, calculate_stat_data()}
   end
 
   def handle_call(:force_update, _from, _state) do
-    new_stats = get_stat_data()
+    new_stats = calculate_stat_data()
     {:reply, new_stats, new_stats}
   end
 
@@ -58,12 +58,12 @@ defmodule Pleroma.Stats do
   end
 
   def handle_cast(:run_update, _state) do
-    new_stats = get_stat_data()
+    new_stats = calculate_stat_data()
 
     {:noreply, new_stats}
   end
 
-  defp get_stat_data do
+  def calculate_stat_data do
     peers =
       from(
         u in User,
@@ -77,7 +77,15 @@ defmodule Pleroma.Stats do
 
     status_count = Repo.aggregate(User.Query.build(%{local: true}), :sum, :note_count)
 
-    user_count = Repo.aggregate(User.Query.build(%{local: true, active: true}), :count, :id)
+    users_query =
+      from(u in User,
+        where: u.deactivated != true,
+        where: u.local == true,
+        where: not is_nil(u.nickname),
+        where: not u.invisible
+      )
+
+    user_count = Repo.aggregate(users_query, :count, :id)
 
     %{
       peers: peers,
-- 
cgit v1.2.3


From 1ebf8db2a595c04da0e0ecbcd9c78b8833deecda Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 20 Apr 2020 12:59:16 +0000
Subject: Merge branch 'fix-object_age_policy' into 'develop'

Fix ObjectAgePolicy

See merge request pleroma/pleroma!2404
---
 lib/pleroma/web/activity_pub/mrf/object_age_policy.ex | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
index 4a8bc91ae..b0ccb63c8 100644
--- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
   @moduledoc "Filter activities depending on their age"
   @behaviour Pleroma.Web.ActivityPub.MRF
 
-  defp check_date(%{"published" => published} = message) do
+  defp check_date(%{"object" => %{"published" => published}} = message) do
     with %DateTime{} = now <- DateTime.utc_now(),
          {:ok, %DateTime{} = then, _} <- DateTime.from_iso8601(published),
          max_ttl <- Config.get([:mrf_object_age, :threshold]),
@@ -96,5 +96,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
   def filter(message), do: {:ok, message}
 
   @impl true
-  def describe, do: {:ok, %{}}
+  def describe do
+    mrf_object_age =
+      Pleroma.Config.get(:mrf_object_age)
+      |> Enum.into(%{})
+
+    {:ok, %{mrf_object_age: mrf_object_age}}
+  end
 end
-- 
cgit v1.2.3


From da4923f2e59aac7f97812a756593602083f17626 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 16 Apr 2020 21:58:57 +0000
Subject: Merge branch 'authenticated-api-oauth-check-enforcement' into
 'develop'

Enforcement of OAuth scopes check for authenticated API endpoints

See merge request pleroma/pleroma!2349
---
 lib/pleroma/plugs/auth_expected_plug.ex            | 17 ++++++++
 lib/pleroma/plugs/oauth_scopes_plug.ex             |  5 ++-
 lib/pleroma/plugs/plug_helper.ex                   | 38 +++++++++++++++++
 lib/pleroma/tests/oauth_test_controller.ex         | 31 ++++++++++++++
 lib/pleroma/web/masto_fe_controller.ex             |  2 +-
 .../mastodon_api/controllers/account_controller.ex |  9 +++-
 .../controllers/mastodon_api_controller.ex         | 18 ++++++--
 .../controllers/suggestion_controller.ex           |  9 ++--
 lib/pleroma/web/oauth/oauth_controller.ex          |  2 +
 .../controllers/pleroma_api_controller.ex          |  2 +-
 lib/pleroma/web/router.ex                          | 14 ++++++-
 .../web/twitter_api/twitter_api_controller.ex      |  2 +
 lib/pleroma/web/web.ex                             | 49 ++++++++++++++++++++++
 13 files changed, 185 insertions(+), 13 deletions(-)
 create mode 100644 lib/pleroma/plugs/auth_expected_plug.ex
 create mode 100644 lib/pleroma/plugs/plug_helper.ex
 create mode 100644 lib/pleroma/tests/oauth_test_controller.ex

(limited to 'lib')

diff --git a/lib/pleroma/plugs/auth_expected_plug.ex b/lib/pleroma/plugs/auth_expected_plug.ex
new file mode 100644
index 000000000..f79597dc3
--- /dev/null
+++ b/lib/pleroma/plugs/auth_expected_plug.ex
@@ -0,0 +1,17 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.AuthExpectedPlug do
+  import Plug.Conn
+
+  def init(options), do: options
+
+  def call(conn, _) do
+    put_private(conn, :auth_expected, true)
+  end
+
+  def auth_expected?(conn) do
+    conn.private[:auth_expected]
+  end
+end
diff --git a/lib/pleroma/plugs/oauth_scopes_plug.ex b/lib/pleroma/plugs/oauth_scopes_plug.ex
index 38df074ad..66f48c28c 100644
--- a/lib/pleroma/plugs/oauth_scopes_plug.ex
+++ b/lib/pleroma/plugs/oauth_scopes_plug.ex
@@ -8,12 +8,15 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
 
   alias Pleroma.Config
   alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug
+  alias Pleroma.Plugs.PlugHelper
+
+  use Pleroma.Web, :plug
 
   @behaviour Plug
 
   def init(%{scopes: _} = options), do: options
 
-  def call(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do
+  def perform(%Plug.Conn{assigns: assigns} = conn, %{scopes: scopes} = options) do
     op = options[:op] || :|
     token = assigns[:token]
 
diff --git a/lib/pleroma/plugs/plug_helper.ex b/lib/pleroma/plugs/plug_helper.ex
new file mode 100644
index 000000000..4f83e9414
--- /dev/null
+++ b/lib/pleroma/plugs/plug_helper.ex
@@ -0,0 +1,38 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Plugs.PlugHelper do
+  @moduledoc "Pleroma Plug helper"
+
+  def append_to_called_plugs(conn, plug_module) do
+    append_to_private_list(conn, :called_plugs, plug_module)
+  end
+
+  def append_to_skipped_plugs(conn, plug_module) do
+    append_to_private_list(conn, :skipped_plugs, plug_module)
+  end
+
+  def plug_called?(conn, plug_module) do
+    contained_in_private_list?(conn, :called_plugs, plug_module)
+  end
+
+  def plug_skipped?(conn, plug_module) do
+    contained_in_private_list?(conn, :skipped_plugs, plug_module)
+  end
+
+  def plug_called_or_skipped?(conn, plug_module) do
+    plug_called?(conn, plug_module) || plug_skipped?(conn, plug_module)
+  end
+
+  defp append_to_private_list(conn, private_variable, value) do
+    list = conn.private[private_variable] || []
+    modified_list = Enum.uniq(list ++ [value])
+    Plug.Conn.put_private(conn, private_variable, modified_list)
+  end
+
+  defp contained_in_private_list?(conn, private_variable, value) do
+    list = conn.private[private_variable] || []
+    value in list
+  end
+end
diff --git a/lib/pleroma/tests/oauth_test_controller.ex b/lib/pleroma/tests/oauth_test_controller.ex
new file mode 100644
index 000000000..58d517f78
--- /dev/null
+++ b/lib/pleroma/tests/oauth_test_controller.ex
@@ -0,0 +1,31 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+# A test controller reachable only in :test env.
+# Serves to test OAuth scopes check skipping / enforcement.
+defmodule Pleroma.Tests.OAuthTestController do
+  @moduledoc false
+
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+
+  plug(:skip_plug, OAuthScopesPlug when action == :skipped_oauth)
+
+  plug(OAuthScopesPlug, %{scopes: ["read"]} when action != :missed_oauth)
+
+  def skipped_oauth(conn, _params) do
+    noop(conn)
+  end
+
+  def performed_oauth(conn, _params) do
+    noop(conn)
+  end
+
+  def missed_oauth(conn, _params) do
+    noop(conn)
+  end
+
+  defp noop(conn), do: json(conn, %{})
+end
diff --git a/lib/pleroma/web/masto_fe_controller.ex b/lib/pleroma/web/masto_fe_controller.ex
index 43649ad26..557cde328 100644
--- a/lib/pleroma/web/masto_fe_controller.ex
+++ b/lib/pleroma/web/masto_fe_controller.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.MastoFEController do
     when action == :index
   )
 
-  plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug when action != :index)
+  plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug when action not in [:index, :manifest])
 
   @doc "GET /web/*path"
   def index(%{assigns: %{user: user, token: token}} = conn, _params)
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index 73853c1e4..229d4be28 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -15,10 +15,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI.ListView
   alias Pleroma.Web.MastodonAPI.MastodonAPI
+  alias Pleroma.Web.MastodonAPI.MastodonAPIController
   alias Pleroma.Web.MastodonAPI.StatusView
   alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.TwitterAPI.TwitterAPI
 
+  plug(:skip_plug, OAuthScopesPlug when action == :identity_proofs)
+
   plug(
     OAuthScopesPlug,
     %{fallback: :proceed_unauthenticated, scopes: ["read:accounts"]}
@@ -366,6 +369,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
   end
 
   @doc "GET /api/v1/endorsements"
-  def endorsements(conn, params),
-    do: Pleroma.Web.MastodonAPI.MastodonAPIController.empty_array(conn, params)
+  def endorsements(conn, params), do: MastodonAPIController.empty_array(conn, params)
+
+  @doc "GET /api/v1/identity_proofs"
+  def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params)
 end
diff --git a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex
index 14075307d..ac8c18f24 100644
--- a/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/mastodon_api_controller.ex
@@ -3,21 +3,31 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
+  @moduledoc """
+  Contains stubs for unimplemented Mastodon API endpoints.
+
+  Note: instead of routing directly to this controller's action,
+    it's preferable to define an action in relevant (non-generic) controller,
+    set up OAuth rules for it and call this controller's function from it.
+  """
+
   use Pleroma.Web, :controller
 
   require Logger
 
+  plug(:skip_plug, Pleroma.Plugs.OAuthScopesPlug when action in [:empty_array, :empty_object])
+
+  plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug)
+
   action_fallback(Pleroma.Web.MastodonAPI.FallbackController)
 
-  # Stubs for unimplemented mastodon api
-  #
   def empty_array(conn, _) do
-    Logger.debug("Unimplemented, returning an empty array")
+    Logger.debug("Unimplemented, returning an empty array (list)")
     json(conn, [])
   end
 
   def empty_object(conn, _) do
-    Logger.debug("Unimplemented, returning an empty object")
+    Logger.debug("Unimplemented, returning an empty object (map)")
     json(conn, %{})
   end
 end
diff --git a/lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex b/lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex
index 0cdc7bd8d..c93a43969 100644
--- a/lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/suggestion_controller.ex
@@ -5,10 +5,13 @@
 defmodule Pleroma.Web.MastodonAPI.SuggestionController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Plugs.OAuthScopesPlug
+
   require Logger
 
+  plug(OAuthScopesPlug, %{scopes: ["read"]} when action == :index)
+
   @doc "GET /api/v1/suggestions"
-  def index(conn, _) do
-    json(conn, [])
-  end
+  def index(conn, params),
+    do: Pleroma.Web.MastodonAPI.MastodonAPIController.empty_array(conn, params)
 end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index 46688db7e..0121cd661 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -27,6 +27,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   plug(:fetch_flash)
   plug(RateLimiter, [name: :authentication] when action == :create_authorization)
 
+  plug(:skip_plug, Pleroma.Plugs.OAuthScopesPlug)
+
   action_fallback(Pleroma.Web.OAuth.FallbackController)
 
   @oob_token_redirect_uri "urn:ietf:wg:oauth:2.0:oob"
diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
index 41677d04d..f0867c2c1 100644
--- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
@@ -34,7 +34,7 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
 
   plug(
     OAuthScopesPlug,
-    %{scopes: ["write:conversations"]} when action == :update_conversation
+    %{scopes: ["write:conversations"]} when action in [:update_conversation, :read_conversations]
   )
 
   plug(OAuthScopesPlug, %{scopes: ["write:notifications"]} when action == :read_notification)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index cb590acfb..bc2cf8b44 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -34,6 +34,7 @@ defmodule Pleroma.Web.Router do
   pipeline :authenticated_api do
     plug(:accepts, ["json"])
     plug(:fetch_session)
+    plug(Pleroma.Plugs.AuthExpectedPlug)
     plug(Pleroma.Plugs.OAuthPlug)
     plug(Pleroma.Plugs.BasicAuthDecoderPlug)
     plug(Pleroma.Plugs.UserFetcherPlug)
@@ -334,7 +335,7 @@ defmodule Pleroma.Web.Router do
     get("/accounts/relationships", AccountController, :relationships)
 
     get("/accounts/:id/lists", AccountController, :lists)
-    get("/accounts/:id/identity_proofs", MastodonAPIController, :empty_array)
+    get("/accounts/:id/identity_proofs", AccountController, :identity_proofs)
 
     get("/follow_requests", FollowRequestController, :index)
     get("/blocks", AccountController, :blocks)
@@ -657,6 +658,17 @@ defmodule Pleroma.Web.Router do
     end
   end
 
+  # Test-only routes needed to test action dispatching and plug chain execution
+  if Pleroma.Config.get(:env) == :test do
+    scope "/test/authenticated_api", Pleroma.Tests do
+      pipe_through(:authenticated_api)
+
+      for action <- [:skipped_oauth, :performed_oauth, :missed_oauth] do
+        get("/#{action}", OAuthTestController, action)
+      end
+    end
+  end
+
   scope "/", Pleroma.Web.MongooseIM do
     get("/user_exists", MongooseIMController, :user_exists)
     get("/check_password", MongooseIMController, :check_password)
diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
index 0229aea97..31adc2817 100644
--- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex
@@ -15,6 +15,8 @@ defmodule Pleroma.Web.TwitterAPI.Controller do
 
   plug(OAuthScopesPlug, %{scopes: ["write:notifications"]} when action == :notifications_read)
 
+  plug(:skip_plug, OAuthScopesPlug when action in [:oauth_tokens, :revoke_token])
+
   plug(Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug)
 
   action_fallback(:errors)
diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex
index cf3ac1287..ae7c94640 100644
--- a/lib/pleroma/web/web.ex
+++ b/lib/pleroma/web/web.ex
@@ -29,11 +29,40 @@ defmodule Pleroma.Web do
       import Pleroma.Web.Router.Helpers
       import Pleroma.Web.TranslationHelpers
 
+      alias Pleroma.Plugs.PlugHelper
+
       plug(:set_put_layout)
 
       defp set_put_layout(conn, _) do
         put_layout(conn, Pleroma.Config.get(:app_layout, "app.html"))
       end
+
+      # Marks a plug intentionally skipped and blocks its execution if it's present in plugs chain
+      defp skip_plug(conn, plug_module) do
+        try do
+          plug_module.ensure_skippable()
+        rescue
+          UndefinedFunctionError ->
+            raise "#{plug_module} is not skippable. Append `use Pleroma.Web, :plug` to its code."
+        end
+
+        PlugHelper.append_to_skipped_plugs(conn, plug_module)
+      end
+
+      # Here we can apply before-action hooks (e.g. verify whether auth checks were preformed)
+      defp action(conn, params) do
+        if Pleroma.Plugs.AuthExpectedPlug.auth_expected?(conn) &&
+             not PlugHelper.plug_called_or_skipped?(conn, Pleroma.Plugs.OAuthScopesPlug) do
+          conn
+          |> render_error(
+            :forbidden,
+            "Security violation: OAuth scopes check was neither handled nor explicitly skipped."
+          )
+          |> halt()
+        else
+          super(conn, params)
+        end
+      end
     end
   end
 
@@ -96,6 +125,26 @@ defmodule Pleroma.Web do
     end
   end
 
+  def plug do
+    quote do
+      alias Pleroma.Plugs.PlugHelper
+
+      def ensure_skippable, do: :noop
+
+      @impl Plug
+      @doc "If marked as skipped, returns `conn`, and calls `perform/2` otherwise."
+      def call(%Plug.Conn{} = conn, options) do
+        if PlugHelper.plug_skipped?(conn, __MODULE__) do
+          conn
+        else
+          conn
+          |> PlugHelper.append_to_called_plugs(__MODULE__)
+          |> perform(options)
+        end
+      end
+    end
+  end
+
   @doc """
   When used, dispatch to the appropriate controller/view/etc.
   """
-- 
cgit v1.2.3


From 862d4886c9c600ff0ff85edc744e3c05a3fcd68d Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 17 Apr 2020 21:21:10 +0300
Subject: [#1682] Fixed Basic Auth permissions issue by disabling OAuth scopes
 checks when password is provided. Refactored plugs skipping functionality.

---
 lib/pleroma/plugs/authentication_plug.ex        |  6 +++++-
 lib/pleroma/plugs/legacy_authentication_plug.ex |  3 +++
 lib/pleroma/plugs/plug_helper.ex                | 24 +++++++++++----------
 lib/pleroma/web/web.ex                          | 28 ++++++++++++++++++-------
 4 files changed, 42 insertions(+), 19 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/authentication_plug.ex b/lib/pleroma/plugs/authentication_plug.ex
index 089028d77..0061c69dc 100644
--- a/lib/pleroma/plugs/authentication_plug.ex
+++ b/lib/pleroma/plugs/authentication_plug.ex
@@ -4,8 +4,11 @@
 
 defmodule Pleroma.Plugs.AuthenticationPlug do
   alias Comeonin.Pbkdf2
-  import Plug.Conn
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
+
+  import Plug.Conn
+
   require Logger
 
   def init(options), do: options
@@ -37,6 +40,7 @@ defmodule Pleroma.Plugs.AuthenticationPlug do
     if Pbkdf2.checkpw(password, password_hash) do
       conn
       |> assign(:user, auth_user)
+      |> OAuthScopesPlug.skip_plug()
     else
       conn
     end
diff --git a/lib/pleroma/plugs/legacy_authentication_plug.ex b/lib/pleroma/plugs/legacy_authentication_plug.ex
index 5c5c36c56..d346e01a6 100644
--- a/lib/pleroma/plugs/legacy_authentication_plug.ex
+++ b/lib/pleroma/plugs/legacy_authentication_plug.ex
@@ -4,6 +4,8 @@
 
 defmodule Pleroma.Plugs.LegacyAuthenticationPlug do
   import Plug.Conn
+
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
 
   def init(options) do
@@ -27,6 +29,7 @@ defmodule Pleroma.Plugs.LegacyAuthenticationPlug do
       conn
       |> assign(:auth_user, user)
       |> assign(:user, user)
+      |> OAuthScopesPlug.skip_plug()
     else
       _ ->
         conn
diff --git a/lib/pleroma/plugs/plug_helper.ex b/lib/pleroma/plugs/plug_helper.ex
index 4f83e9414..9c67be8ef 100644
--- a/lib/pleroma/plugs/plug_helper.ex
+++ b/lib/pleroma/plugs/plug_helper.ex
@@ -5,30 +5,32 @@
 defmodule Pleroma.Plugs.PlugHelper do
   @moduledoc "Pleroma Plug helper"
 
-  def append_to_called_plugs(conn, plug_module) do
-    append_to_private_list(conn, :called_plugs, plug_module)
-  end
+  @called_plugs_list_id :called_plugs
+  def called_plugs_list_id, do: @called_plugs_list_id
 
-  def append_to_skipped_plugs(conn, plug_module) do
-    append_to_private_list(conn, :skipped_plugs, plug_module)
-  end
+  @skipped_plugs_list_id :skipped_plugs
+  def skipped_plugs_list_id, do: @skipped_plugs_list_id
 
+  @doc "Returns `true` if specified plug was called."
   def plug_called?(conn, plug_module) do
-    contained_in_private_list?(conn, :called_plugs, plug_module)
+    contained_in_private_list?(conn, @called_plugs_list_id, plug_module)
   end
 
+  @doc "Returns `true` if specified plug was explicitly marked as skipped."
   def plug_skipped?(conn, plug_module) do
-    contained_in_private_list?(conn, :skipped_plugs, plug_module)
+    contained_in_private_list?(conn, @skipped_plugs_list_id, plug_module)
   end
 
+  @doc "Returns `true` if specified plug was either called or explicitly marked as skipped."
   def plug_called_or_skipped?(conn, plug_module) do
     plug_called?(conn, plug_module) || plug_skipped?(conn, plug_module)
   end
 
-  defp append_to_private_list(conn, private_variable, value) do
-    list = conn.private[private_variable] || []
+  # Appends plug to known list (skipped, called). Intended to be used from within plug code only.
+  def append_to_private_list(conn, list_id, value) do
+    list = conn.private[list_id] || []
     modified_list = Enum.uniq(list ++ [value])
-    Plug.Conn.put_private(conn, private_variable, modified_list)
+    Plug.Conn.put_private(conn, list_id, modified_list)
   end
 
   defp contained_in_private_list?(conn, private_variable, value) do
diff --git a/lib/pleroma/web/web.ex b/lib/pleroma/web/web.ex
index ae7c94640..bf48ce26c 100644
--- a/lib/pleroma/web/web.ex
+++ b/lib/pleroma/web/web.ex
@@ -40,17 +40,22 @@ defmodule Pleroma.Web do
       # Marks a plug intentionally skipped and blocks its execution if it's present in plugs chain
       defp skip_plug(conn, plug_module) do
         try do
-          plug_module.ensure_skippable()
+          plug_module.skip_plug(conn)
         rescue
           UndefinedFunctionError ->
             raise "#{plug_module} is not skippable. Append `use Pleroma.Web, :plug` to its code."
         end
-
-        PlugHelper.append_to_skipped_plugs(conn, plug_module)
       end
 
-      # Here we can apply before-action hooks (e.g. verify whether auth checks were preformed)
+      # Executed just before actual controller action, invokes before-action hooks (callbacks)
       defp action(conn, params) do
+        with %Plug.Conn{halted: false} <- maybe_halt_on_missing_oauth_scopes_check(conn) do
+          super(conn, params)
+        end
+      end
+
+      # Halts if authenticated API action neither performs nor explicitly skips OAuth scopes check
+      defp maybe_halt_on_missing_oauth_scopes_check(conn) do
         if Pleroma.Plugs.AuthExpectedPlug.auth_expected?(conn) &&
              not PlugHelper.plug_called_or_skipped?(conn, Pleroma.Plugs.OAuthScopesPlug) do
           conn
@@ -60,7 +65,7 @@ defmodule Pleroma.Web do
           )
           |> halt()
         else
-          super(conn, params)
+          conn
         end
       end
     end
@@ -129,7 +134,16 @@ defmodule Pleroma.Web do
     quote do
       alias Pleroma.Plugs.PlugHelper
 
-      def ensure_skippable, do: :noop
+      @doc """
+      Marks a plug intentionally skipped and blocks its execution if it's present in plugs chain.
+      """
+      def skip_plug(conn) do
+        PlugHelper.append_to_private_list(
+          conn,
+          PlugHelper.skipped_plugs_list_id(),
+          __MODULE__
+        )
+      end
 
       @impl Plug
       @doc "If marked as skipped, returns `conn`, and calls `perform/2` otherwise."
@@ -138,7 +152,7 @@ defmodule Pleroma.Web do
           conn
         else
           conn
-          |> PlugHelper.append_to_called_plugs(__MODULE__)
+          |> PlugHelper.append_to_private_list(PlugHelper.called_plugs_list_id(), __MODULE__)
           |> perform(options)
         end
       end
-- 
cgit v1.2.3


From 8cf4e1619e439b1c9374a52cfc2b0cdf8d549d02 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 14 Apr 2020 11:43:53 -0500
Subject: Fix Oban not receiving :ok from RichMediaHelper job

---
 lib/pleroma/web/rich_media/helpers.ex | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 0314535d2..9d3d7f978 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -64,5 +64,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do
 
   def fetch_data_for_activity(_), do: %{}
 
-  def perform(:fetch, %Activity{} = activity), do: fetch_data_for_activity(activity)
+  def perform(:fetch, %Activity{} = activity) do
+    fetch_data_for_activity(activity)
+    :ok
+  end
 end
-- 
cgit v1.2.3


From a4afeed4266e75279422a6721f0a9a2aece0b9ea Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 15 Apr 2020 12:05:22 +0200
Subject: Uploads: Sandbox them in the CSP.

---
 lib/pleroma/plugs/uploaded_media.ex | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex
index 36ff024a7..94147e0c4 100644
--- a/lib/pleroma/plugs/uploaded_media.ex
+++ b/lib/pleroma/plugs/uploaded_media.ex
@@ -41,6 +41,7 @@ defmodule Pleroma.Plugs.UploadedMedia do
         conn ->
           conn
       end
+      |> merge_resp_headers([{"content-security-policy", "sandbox"}])
 
     config = Pleroma.Config.get(Pleroma.Upload)
 
-- 
cgit v1.2.3


From fb9ec885cc96e51bd7703d98d8e41305ead56b9e Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Fri, 17 Apr 2020 08:55:46 +0000
Subject: Merge branch 'feature/1677-need_reboot-flag-endpoint' into 'develop'

Added need_reboot endpoint to admin api

Closes #1677

See merge request pleroma/pleroma!2373
---
 lib/pleroma/web/admin_api/admin_api_controller.ex | 35 ++++++++---------------
 lib/pleroma/web/router.ex                         |  1 +
 2 files changed, 13 insertions(+), 23 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 6c88549f5..e1869678e 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -911,16 +911,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
         end)
         |> List.flatten()
 
-      response = %{configs: merged}
-
-      response =
-        if Restarter.Pleroma.need_reboot?() do
-          Map.put(response, :need_reboot, true)
-        else
-          response
-        end
-
-      json(conn, response)
+      json(conn, %{configs: merged, need_reboot: Restarter.Pleroma.need_reboot?()})
     end
   end
 
@@ -947,28 +938,22 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
       Config.TransferTask.load_and_update_env(deleted, false)
 
-      need_reboot? =
-        Restarter.Pleroma.need_reboot?() ||
-          Enum.any?(updated, fn config ->
+      if !Restarter.Pleroma.need_reboot?() do
+        changed_reboot_settings? =
+          (updated ++ deleted)
+          |> Enum.any?(fn config ->
             group = ConfigDB.from_string(config.group)
             key = ConfigDB.from_string(config.key)
             value = ConfigDB.from_binary(config.value)
             Config.TransferTask.pleroma_need_restart?(group, key, value)
           end)
 
-      response = %{configs: updated}
-
-      response =
-        if need_reboot? do
-          Restarter.Pleroma.need_reboot()
-          Map.put(response, :need_reboot, need_reboot?)
-        else
-          response
-        end
+        if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot()
+      end
 
       conn
       |> put_view(ConfigView)
-      |> render("index.json", response)
+      |> render("index.json", %{configs: updated, need_reboot: Restarter.Pleroma.need_reboot?()})
     end
   end
 
@@ -980,6 +965,10 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def need_reboot(conn, _params) do
+    json(conn, %{need_reboot: Restarter.Pleroma.need_reboot?()})
+  end
+
   defp configurable_from_database(conn) do
     if Config.get(:configurable_from_database) do
       :ok
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index bc2cf8b44..1da9478db 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -200,6 +200,7 @@ defmodule Pleroma.Web.Router do
     get("/config", AdminAPIController, :config_show)
     post("/config", AdminAPIController, :config_update)
     get("/config/descriptions", AdminAPIController, :config_descriptions)
+    get("/need_reboot", AdminAPIController, :need_reboot)
     get("/restart", AdminAPIController, :restart)
 
     get("/moderation_log", AdminAPIController, :list_log)
-- 
cgit v1.2.3


From 138e6ece1b4185fbe8d8a8f7873ae0c3ee0837e6 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 11 Apr 2020 19:46:04 +0000
Subject: Merge branch 'following-relationships-optimizations' into 'develop'

FollowingRelationship storage & performance optimizations

See merge request pleroma/pleroma!2332
---
 lib/pleroma/ecto_enums.ex                      |  8 ++-
 lib/pleroma/following_relationship.ex          | 79 ++++++++++++++++++--
 lib/pleroma/user.ex                            | 36 ++++++++--
 lib/pleroma/user/query.ex                      |  6 +-
 lib/pleroma/user_relationship.ex               | 99 +++++++++++++++++++++++---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 13 ++--
 lib/pleroma/web/common_api/common_api.ex       |  4 +-
 7 files changed, 211 insertions(+), 34 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex
index d9b601223..6fc47620c 100644
--- a/lib/pleroma/ecto_enums.ex
+++ b/lib/pleroma/ecto_enums.ex
@@ -4,10 +4,16 @@
 
 import EctoEnum
 
-defenum(UserRelationshipTypeEnum,
+defenum(Pleroma.UserRelationship.Type,
   block: 1,
   mute: 2,
   reblog_mute: 3,
   notification_mute: 4,
   inverse_subscription: 5
 )
+
+defenum(Pleroma.FollowingRelationship.State,
+  follow_pending: 1,
+  follow_accept: 2,
+  follow_reject: 3
+)
diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex
index a6d281151..9ccf40495 100644
--- a/lib/pleroma/following_relationship.ex
+++ b/lib/pleroma/following_relationship.ex
@@ -8,12 +8,13 @@ defmodule Pleroma.FollowingRelationship do
   import Ecto.Changeset
   import Ecto.Query
 
+  alias Ecto.Changeset
   alias FlakeId.Ecto.CompatType
   alias Pleroma.Repo
   alias Pleroma.User
 
   schema "following_relationships" do
-    field(:state, :string, default: "accept")
+    field(:state, Pleroma.FollowingRelationship.State, default: :follow_pending)
 
     belongs_to(:follower, User, type: CompatType)
     belongs_to(:following, User, type: CompatType)
@@ -27,6 +28,18 @@ defmodule Pleroma.FollowingRelationship do
     |> put_assoc(:follower, attrs.follower)
     |> put_assoc(:following, attrs.following)
     |> validate_required([:state, :follower, :following])
+    |> unique_constraint(:follower_id,
+      name: :following_relationships_follower_id_following_id_index
+    )
+    |> validate_not_self_relationship()
+  end
+
+  def state_to_enum(state) when state in ["pending", "accept", "reject"] do
+    String.to_existing_atom("follow_#{state}")
+  end
+
+  def state_to_enum(state) do
+    raise "State is not convertible to Pleroma.FollowingRelationship.State: #{state}"
   end
 
   def get(%User{} = follower, %User{} = following) do
@@ -35,7 +48,7 @@ defmodule Pleroma.FollowingRelationship do
     |> Repo.one()
   end
 
-  def update(follower, following, "reject"), do: unfollow(follower, following)
+  def update(follower, following, :follow_reject), do: unfollow(follower, following)
 
   def update(%User{} = follower, %User{} = following, state) do
     case get(follower, following) do
@@ -50,7 +63,7 @@ defmodule Pleroma.FollowingRelationship do
     end
   end
 
-  def follow(%User{} = follower, %User{} = following, state \\ "accept") do
+  def follow(%User{} = follower, %User{} = following, state \\ :follow_accept) do
     %__MODULE__{}
     |> changeset(%{follower: follower, following: following, state: state})
     |> Repo.insert(on_conflict: :nothing)
@@ -80,7 +93,7 @@ defmodule Pleroma.FollowingRelationship do
   def get_follow_requests(%User{id: id}) do
     __MODULE__
     |> join(:inner, [r], f in assoc(r, :follower))
-    |> where([r], r.state == "pending")
+    |> where([r], r.state == ^:follow_pending)
     |> where([r], r.following_id == ^id)
     |> select([r, f], f)
     |> Repo.all()
@@ -88,7 +101,7 @@ defmodule Pleroma.FollowingRelationship do
 
   def following?(%User{id: follower_id}, %User{id: followed_id}) do
     __MODULE__
-    |> where(follower_id: ^follower_id, following_id: ^followed_id, state: "accept")
+    |> where(follower_id: ^follower_id, following_id: ^followed_id, state: ^:follow_accept)
     |> Repo.exists?()
   end
 
@@ -97,7 +110,7 @@ defmodule Pleroma.FollowingRelationship do
       __MODULE__
       |> join(:inner, [r], u in User, on: r.following_id == u.id)
       |> where([r], r.follower_id == ^user.id)
-      |> where([r], r.state == "accept")
+      |> where([r], r.state == ^:follow_accept)
       |> select([r, u], u.follower_address)
       |> Repo.all()
 
@@ -129,4 +142,58 @@ defmodule Pleroma.FollowingRelationship do
         move_following(origin, target)
     end
   end
+
+  def all_between_user_sets(
+        source_users,
+        target_users
+      )
+      when is_list(source_users) and is_list(target_users) do
+    source_user_ids = User.binary_id(source_users)
+    target_user_ids = User.binary_id(target_users)
+
+    __MODULE__
+    |> where(
+      fragment(
+        "(follower_id = ANY(?) AND following_id = ANY(?)) OR \
+        (follower_id = ANY(?) AND following_id = ANY(?))",
+        ^source_user_ids,
+        ^target_user_ids,
+        ^target_user_ids,
+        ^source_user_ids
+      )
+    )
+    |> Repo.all()
+  end
+
+  def find(following_relationships, follower, following) do
+    Enum.find(following_relationships, fn
+      fr -> fr.follower_id == follower.id and fr.following_id == following.id
+    end)
+  end
+
+  defp validate_not_self_relationship(%Changeset{} = changeset) do
+    changeset
+    |> validate_follower_id_following_id_inequality()
+    |> validate_following_id_follower_id_inequality()
+  end
+
+  defp validate_follower_id_following_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :follower_id, fn _, follower_id ->
+      if follower_id == get_field(changeset, :following_id) do
+        [source_id: "can't be equal to following_id"]
+      else
+        []
+      end
+    end)
+  end
+
+  defp validate_following_id_follower_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :following_id, fn _, following_id ->
+      if following_id == get_field(changeset, :follower_id) do
+        [target_id: "can't be equal to follower_id"]
+      else
+        []
+      end
+    end)
+  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 0e5121694..df96f38df 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -227,6 +227,24 @@ defmodule Pleroma.User do
     end
   end
 
+  @doc """
+  Dumps Flake Id to SQL-compatible format (16-byte UUID).
+  E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>>
+  """
+  def binary_id(source_id) when is_binary(source_id) do
+    with {:ok, dumped_id} <- FlakeId.Ecto.CompatType.dump(source_id) do
+      dumped_id
+    else
+      _ -> source_id
+    end
+  end
+
+  def binary_id(source_ids) when is_list(source_ids) do
+    Enum.map(source_ids, &binary_id/1)
+  end
+
+  def binary_id(%User{} = user), do: binary_id(user.id)
+
   @doc "Returns status account"
   @spec account_status(User.t()) :: account_status()
   def account_status(%User{deactivated: true}), do: :deactivated
@@ -689,7 +707,7 @@ defmodule Pleroma.User do
 
   @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()}
   def maybe_direct_follow(%User{} = follower, %User{local: true, locked: true} = followed) do
-    follow(follower, followed, "pending")
+    follow(follower, followed, :follow_pending)
   end
 
   def maybe_direct_follow(%User{} = follower, %User{local: true} = followed) do
@@ -709,14 +727,14 @@ defmodule Pleroma.User do
   def follow_all(follower, followeds) do
     followeds
     |> Enum.reject(fn followed -> blocks?(follower, followed) || blocks?(followed, follower) end)
-    |> Enum.each(&follow(follower, &1, "accept"))
+    |> Enum.each(&follow(follower, &1, :follow_accept))
 
     set_cache(follower)
   end
 
   defdelegate following(user), to: FollowingRelationship
 
-  def follow(%User{} = follower, %User{} = followed, state \\ "accept") do
+  def follow(%User{} = follower, %User{} = followed, state \\ :follow_accept) do
     deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked])
 
     cond do
@@ -743,7 +761,7 @@ defmodule Pleroma.User do
 
   def unfollow(%User{} = follower, %User{} = followed) do
     case get_follow_state(follower, followed) do
-      state when state in ["accept", "pending"] ->
+      state when state in [:follow_pending, :follow_accept] ->
         FollowingRelationship.unfollow(follower, followed)
         {:ok, followed} = update_follower_count(followed)
 
@@ -761,14 +779,18 @@ defmodule Pleroma.User do
 
   defdelegate following?(follower, followed), to: FollowingRelationship
 
+  @doc "Returns follow state as Pleroma.FollowingRelationship.State value"
   def get_follow_state(%User{} = follower, %User{} = following) do
     following_relationship = FollowingRelationship.get(follower, following)
 
     case {following_relationship, following.local} do
       {nil, false} ->
         case Utils.fetch_latest_follow(follower, following) do
-          %{data: %{"state" => state}} when state in ["pending", "accept"] -> state
-          _ -> nil
+          %Activity{data: %{"state" => state}} when state in ["pending", "accept"] ->
+            FollowingRelationship.state_to_enum(state)
+
+          _ ->
+            nil
         end
 
       {%{state: state}, _} ->
@@ -1267,7 +1289,7 @@ defmodule Pleroma.User do
 
   def blocks?(%User{} = user, %User{} = target) do
     blocks_user?(user, target) ||
-      (!User.following?(user, target) && blocks_domain?(user, target))
+      (blocks_domain?(user, target) and not User.following?(user, target))
   end
 
   def blocks_user?(%User{} = user, %User{} = target) do
diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index 884e33039..ec88088cf 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -148,7 +148,7 @@ defmodule Pleroma.User.Query do
       as: :relationships,
       on: r.following_id == ^id and r.follower_id == u.id
     )
-    |> where([relationships: r], r.state == "accept")
+    |> where([relationships: r], r.state == ^:follow_accept)
   end
 
   defp compose_query({:friends, %User{id: id}}, query) do
@@ -158,7 +158,7 @@ defmodule Pleroma.User.Query do
       as: :relationships,
       on: r.following_id == u.id and r.follower_id == ^id
     )
-    |> where([relationships: r], r.state == "accept")
+    |> where([relationships: r], r.state == ^:follow_accept)
   end
 
   defp compose_query({:recipients_from_activity, to}, query) do
@@ -173,7 +173,7 @@ defmodule Pleroma.User.Query do
     )
     |> where(
       [u, following: f, relationships: r],
-      u.ap_id in ^to or (f.follower_address in ^to and r.state == "accept")
+      u.ap_id in ^to or (f.follower_address in ^to and r.state == ^:follow_accept)
     )
     |> distinct(true)
   end
diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex
index 01b6ace9d..ad0d303b1 100644
--- a/lib/pleroma/user_relationship.ex
+++ b/lib/pleroma/user_relationship.ex
@@ -8,6 +8,8 @@ defmodule Pleroma.UserRelationship do
   import Ecto.Changeset
   import Ecto.Query
 
+  alias Ecto.Changeset
+  alias Pleroma.FollowingRelationship
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.UserRelationship
@@ -15,12 +17,12 @@ defmodule Pleroma.UserRelationship do
   schema "user_relationships" do
     belongs_to(:source, User, type: FlakeId.Ecto.CompatType)
     belongs_to(:target, User, type: FlakeId.Ecto.CompatType)
-    field(:relationship_type, UserRelationshipTypeEnum)
+    field(:relationship_type, Pleroma.UserRelationship.Type)
 
     timestamps(updated_at: false)
   end
 
-  for relationship_type <- Keyword.keys(UserRelationshipTypeEnum.__enum_map__()) do
+  for relationship_type <- Keyword.keys(Pleroma.UserRelationship.Type.__enum_map__()) do
     # `def create_block/2`, `def create_mute/2`, `def create_reblog_mute/2`,
     #   `def create_notification_mute/2`, `def create_inverse_subscription/2`
     def unquote(:"create_#{relationship_type}")(source, target),
@@ -37,6 +39,10 @@ defmodule Pleroma.UserRelationship do
       do: exists?(unquote(relationship_type), source, target)
   end
 
+  def user_relationship_types, do: Keyword.keys(user_relationship_mappings())
+
+  def user_relationship_mappings, do: Pleroma.UserRelationship.Type.__enum_map__()
+
   def changeset(%UserRelationship{} = user_relationship, params \\ %{}) do
     user_relationship
     |> cast(params, [:relationship_type, :source_id, :target_id])
@@ -75,18 +81,93 @@ defmodule Pleroma.UserRelationship do
     end
   end
 
-  defp validate_not_self_relationship(%Ecto.Changeset{} = changeset) do
+  def dictionary(
+        source_users,
+        target_users,
+        source_to_target_rel_types \\ nil,
+        target_to_source_rel_types \\ nil
+      )
+      when is_list(source_users) and is_list(target_users) do
+    source_user_ids = User.binary_id(source_users)
+    target_user_ids = User.binary_id(target_users)
+
+    get_rel_type_codes = fn rel_type -> user_relationship_mappings()[rel_type] end
+
+    source_to_target_rel_types =
+      Enum.map(source_to_target_rel_types || user_relationship_types(), &get_rel_type_codes.(&1))
+
+    target_to_source_rel_types =
+      Enum.map(target_to_source_rel_types || user_relationship_types(), &get_rel_type_codes.(&1))
+
+    __MODULE__
+    |> where(
+      fragment(
+        "(source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?)) OR \
+        (source_id = ANY(?) AND target_id = ANY(?) AND relationship_type = ANY(?))",
+        ^source_user_ids,
+        ^target_user_ids,
+        ^source_to_target_rel_types,
+        ^target_user_ids,
+        ^source_user_ids,
+        ^target_to_source_rel_types
+      )
+    )
+    |> select([ur], [ur.relationship_type, ur.source_id, ur.target_id])
+    |> Repo.all()
+  end
+
+  def exists?(dictionary, rel_type, source, target, func) do
+    cond do
+      is_nil(source) or is_nil(target) ->
+        false
+
+      dictionary ->
+        [rel_type, source.id, target.id] in dictionary
+
+      true ->
+        func.(source, target)
+    end
+  end
+
+  @doc ":relationships option for StatusView / AccountView / NotificationView"
+  def view_relationships_option(nil = _reading_user, _actors) do
+    %{user_relationships: [], following_relationships: []}
+  end
+
+  def view_relationships_option(%User{} = reading_user, actors) do
+    user_relationships =
+      UserRelationship.dictionary(
+        [reading_user],
+        actors,
+        [:block, :mute, :notification_mute, :reblog_mute],
+        [:block, :inverse_subscription]
+      )
+
+    following_relationships = FollowingRelationship.all_between_user_sets([reading_user], actors)
+
+    %{user_relationships: user_relationships, following_relationships: following_relationships}
+  end
+
+  defp validate_not_self_relationship(%Changeset{} = changeset) do
     changeset
-    |> validate_change(:target_id, fn _, target_id ->
-      if target_id == get_field(changeset, :source_id) do
-        [target_id: "can't be equal to source_id"]
+    |> validate_source_id_target_id_inequality()
+    |> validate_target_id_source_id_inequality()
+  end
+
+  defp validate_source_id_target_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :source_id, fn _, source_id ->
+      if source_id == get_field(changeset, :target_id) do
+        [source_id: "can't be equal to target_id"]
       else
         []
       end
     end)
-    |> validate_change(:source_id, fn _, source_id ->
-      if source_id == get_field(changeset, :target_id) do
-        [source_id: "can't be equal to target_id"]
+  end
+
+  defp validate_target_id_source_id_inequality(%Changeset{} = changeset) do
+    validate_change(changeset, :target_id, fn _, target_id ->
+      if target_id == get_field(changeset, :source_id) do
+        [target_id: "can't be equal to source_id"]
       else
         []
       end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 09bd9a442..831739c5f 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -491,7 +491,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
            {_, {:ok, follower}} <- {:follow, User.follow(follower, followed)},
            {_, {:ok, _}} <-
              {:follow_state_update, Utils.update_follow_state_for_all(activity, "accept")},
-           {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "accept") do
+           {:ok, _relationship} <-
+             FollowingRelationship.update(follower, followed, :follow_accept) do
         ActivityPub.accept(%{
           to: [follower.ap_id],
           actor: followed,
@@ -501,7 +502,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       else
         {:user_blocked, true} ->
           {:ok, _} = Utils.update_follow_state_for_all(activity, "reject")
-          {:ok, _relationship} = FollowingRelationship.update(follower, followed, "reject")
+          {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_reject)
 
           ActivityPub.reject(%{
             to: [follower.ap_id],
@@ -512,7 +513,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
         {:follow, {:error, _}} ->
           {:ok, _} = Utils.update_follow_state_for_all(activity, "reject")
-          {:ok, _relationship} = FollowingRelationship.update(follower, followed, "reject")
+          {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_reject)
 
           ActivityPub.reject(%{
             to: [follower.ap_id],
@@ -522,7 +523,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           })
 
         {:user_locked, true} ->
-          {:ok, _relationship} = FollowingRelationship.update(follower, followed, "pending")
+          {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_pending)
           :noop
       end
 
@@ -542,7 +543,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "accept") do
+         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept) do
       ActivityPub.accept(%{
         to: follow_activity.data["to"],
         type: "Accept",
@@ -565,7 +566,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "reject"),
+         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject),
          {:ok, activity} <-
            ActivityPub.reject(%{
              to: follow_activity.data["to"],
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index b9fa9fe3a..c84782008 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -42,7 +42,7 @@ defmodule Pleroma.Web.CommonAPI do
     with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
          {:ok, follower} <- User.follow(follower, followed),
          {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "accept"),
+         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept),
          {:ok, _activity} <-
            ActivityPub.accept(%{
              to: [follower.ap_id],
@@ -57,7 +57,7 @@ defmodule Pleroma.Web.CommonAPI do
   def reject_follow_request(follower, followed) do
     with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
          {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"),
-         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, "reject"),
+         {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject),
          {:ok, _activity} <-
            ActivityPub.reject(%{
              to: [follower.ap_id],
-- 
cgit v1.2.3


From b6ca8cc53915b5cd86513b8c5214477626e97f3d Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 1 May 2020 16:48:31 +0000
Subject: Merge branch 'bugfix/1727-fix-signature-decoding' into 'develop'

Bugfix/1727 fix signature decoding

Closes #1727

See merge request pleroma/pleroma!2454
---
 lib/pleroma/plugs/mapped_signature_to_identity_plug.ex |  5 +++--
 lib/pleroma/signature.ex                               | 18 +++++++++++++++---
 2 files changed, 18 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
index 4f124ed4d..4cc93adb0 100644
--- a/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
+++ b/lib/pleroma/plugs/mapped_signature_to_identity_plug.ex
@@ -13,8 +13,9 @@ defmodule Pleroma.Web.Plugs.MappedSignatureToIdentityPlug do
   def init(options), do: options
 
   defp key_id_from_conn(conn) do
-    with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn) do
-      Signature.key_id_to_actor_id(key_id)
+    with %{"keyId" => key_id} <- HTTPSignatures.signature_for_conn(conn),
+         {:ok, ap_id} <- Signature.key_id_to_actor_id(key_id) do
+      ap_id
     else
       _ ->
         nil
diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index 6b0b2c969..d01728361 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Signature do
   alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
+  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   def key_id_to_actor_id(key_id) do
     uri =
@@ -21,12 +22,23 @@ defmodule Pleroma.Signature do
         uri
       end
 
-    URI.to_string(uri)
+    maybe_ap_id = URI.to_string(uri)
+
+    case Types.ObjectID.cast(maybe_ap_id) do
+      {:ok, ap_id} ->
+        {:ok, ap_id}
+
+      _ ->
+        case Pleroma.Web.WebFinger.finger(maybe_ap_id) do
+          %{"ap_id" => ap_id} -> {:ok, ap_id}
+          _ -> {:error, maybe_ap_id}
+        end
+    end
   end
 
   def fetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
-         actor_id <- key_id_to_actor_id(kid),
+         {:ok, actor_id} <- key_id_to_actor_id(kid),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
     else
@@ -37,7 +49,7 @@ defmodule Pleroma.Signature do
 
   def refetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
-         actor_id <- key_id_to_actor_id(kid),
+         {:ok, actor_id} <- key_id_to_actor_id(kid),
          {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
          {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
       {:ok, public_key}
-- 
cgit v1.2.3


From 2646fdea224cb2af7d2adc66265c334ae1c69669 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 1 May 2020 22:29:22 +0300
Subject: signature.ex: Remove the use of ObjectID type

It is not in stable yet
---
 lib/pleroma/signature.ex | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index d01728361..7006eb2c0 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -8,7 +8,6 @@ defmodule Pleroma.Signature do
   alias Pleroma.Keys
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.ObjectValidators.Types
 
   def key_id_to_actor_id(key_id) do
     uri =
@@ -22,16 +21,14 @@ defmodule Pleroma.Signature do
         uri
       end
 
-    maybe_ap_id = URI.to_string(uri)
-
-    case Types.ObjectID.cast(maybe_ap_id) do
-      {:ok, ap_id} ->
-        {:ok, ap_id}
+    case uri do
+      %URI{scheme: scheme} when scheme in ["https", "http"] ->
+        {:ok, URI.to_string(uri)}
 
       _ ->
-        case Pleroma.Web.WebFinger.finger(maybe_ap_id) do
+        case Pleroma.Web.WebFinger.finger(URI.to_string(uri)) do
           %{"ap_id" => ap_id} -> {:ok, ap_id}
-          _ -> {:error, maybe_ap_id}
+          _ -> {:error, URI.to_string(uri)}
         end
     end
   end
-- 
cgit v1.2.3


From 7b0593367756adc459a7b7bbaa0ace567b97742f Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sun, 19 Apr 2020 21:45:20 +0000
Subject: Merge branch '1559-follow-request-notifications' into 'develop'

[#1559] Support for "follow_request" notifications

Closes #1559

See merge request pleroma/pleroma!2354
---
 lib/pleroma/activity.ex                            | 38 +++++++++--
 lib/pleroma/notification.ex                        | 11 +++-
 lib/pleroma/user.ex                                |  2 +
 .../web/mastodon_api/views/notification_view.ex    |  6 +-
 lib/pleroma/web/push/impl.ex                       | 77 ++++++++++++++--------
 lib/pleroma/web/push/subscription.ex               |  8 +++
 6 files changed, 103 insertions(+), 39 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 5a8329e69..6213d0eb7 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -27,17 +27,13 @@ defmodule Pleroma.Activity do
   # https://github.com/tootsuite/mastodon/blob/master/app/models/notification.rb#L19
   @mastodon_notification_types %{
     "Create" => "mention",
-    "Follow" => "follow",
+    "Follow" => ["follow", "follow_request"],
     "Announce" => "reblog",
     "Like" => "favourite",
     "Move" => "move",
     "EmojiReact" => "pleroma:emoji_reaction"
   }
 
-  @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types,
-                                         into: %{},
-                                         do: {v, k}
-
   schema "activities" do
     field(:data, :map)
     field(:local, :boolean, default: true)
@@ -291,15 +287,43 @@ defmodule Pleroma.Activity do
 
   defp purge_web_resp_cache(nil), do: nil
 
-  for {ap_type, type} <- @mastodon_notification_types do
+  def follow_accepted?(
+        %Activity{data: %{"type" => "Follow", "object" => followed_ap_id}} = activity
+      ) do
+    with %User{} = follower <- Activity.user_actor(activity),
+         %User{} = followed <- User.get_cached_by_ap_id(followed_ap_id) do
+      Pleroma.FollowingRelationship.following?(follower, followed)
+    else
+      _ -> false
+    end
+  end
+
+  def follow_accepted?(_), do: false
+
+  @spec mastodon_notification_type(Activity.t()) :: String.t() | nil
+
+  for {ap_type, type} <- @mastodon_notification_types, not is_list(type) do
     def mastodon_notification_type(%Activity{data: %{"type" => unquote(ap_type)}}),
       do: unquote(type)
   end
 
+  def mastodon_notification_type(%Activity{data: %{"type" => "Follow"}} = activity) do
+    if follow_accepted?(activity) do
+      "follow"
+    else
+      "follow_request"
+    end
+  end
+
   def mastodon_notification_type(%Activity{}), do: nil
 
+  @spec from_mastodon_notification_type(String.t()) :: String.t() | nil
+  @doc "Converts Mastodon notification type to AR activity type"
   def from_mastodon_notification_type(type) do
-    Map.get(@mastodon_to_ap_notification_types, type)
+    with {k, _v} <-
+           Enum.find(@mastodon_notification_types, fn {_k, v} -> type in List.wrap(v) end) do
+      k
+    end
   end
 
   def all_by_actor_and_id(actor, status_ids \\ [])
diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 824ba5ecb..94dc0c2b0 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -293,8 +293,17 @@ defmodule Pleroma.Notification do
     end
   end
 
+  def create_notifications(%Activity{data: %{"type" => "Follow"}} = activity) do
+    if Pleroma.Config.get([:notifications, :enable_follow_request_notifications]) ||
+         Activity.follow_accepted?(activity) do
+      do_create_notifications(activity)
+    else
+      {:ok, []}
+    end
+  end
+
   def create_notifications(%Activity{data: %{"type" => type}} = activity)
-      when type in ["Like", "Announce", "Follow", "Move", "EmojiReact"] do
+      when type in ["Like", "Announce", "Move", "EmojiReact"] do
     do_create_notifications(activity)
   end
 
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index df96f38df..2f0333da0 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -706,6 +706,8 @@ defmodule Pleroma.User do
   def needs_update?(_), do: true
 
   @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()}
+
+  # "Locked" (self-locked) users demand explicit authorization of follow requests
   def maybe_direct_follow(%User{} = follower, %User{local: true, locked: true} = followed) do
     follow(follower, followed, :follow_pending)
   end
diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex
index 33145c484..1720fbead 100644
--- a/lib/pleroma/web/mastodon_api/views/notification_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex
@@ -49,12 +49,12 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do
         "move" ->
           put_target(response, activity, user)
 
-        "follow" ->
-          response
-
         "pleroma:emoji_reaction" ->
           put_status(response, parent_activity, user) |> put_emoji(activity)
 
+        type when type in ["follow", "follow_request"] ->
+          response
+
         _ ->
           nil
       end
diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index afa510f08..f1740a6e0 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -16,6 +16,8 @@ defmodule Pleroma.Web.Push.Impl do
   require Logger
   import Ecto.Query
 
+  defdelegate mastodon_notification_type(activity), to: Activity
+
   @types ["Create", "Follow", "Announce", "Like", "Move"]
 
   @doc "Performs sending notifications for user subscriptions"
@@ -24,32 +26,32 @@ defmodule Pleroma.Web.Push.Impl do
         %{
           activity: %{data: %{"type" => activity_type}} = activity,
           user: %User{id: user_id}
-        } = notif
+        } = notification
       )
       when activity_type in @types do
-    actor = User.get_cached_by_ap_id(notif.activity.data["actor"])
+    actor = User.get_cached_by_ap_id(notification.activity.data["actor"])
 
-    type = Activity.mastodon_notification_type(notif.activity)
+    mastodon_type = mastodon_notification_type(notification.activity)
     gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key)
     avatar_url = User.avatar_url(actor)
     object = Object.normalize(activity)
     user = User.get_cached_by_id(user_id)
     direct_conversation_id = Activity.direct_conversation_id(activity, user)
 
-    for subscription <- fetch_subsriptions(user_id),
-        get_in(subscription.data, ["alerts", type]) do
+    for subscription <- fetch_subscriptions(user_id),
+        Subscription.enabled?(subscription, mastodon_type) do
       %{
         access_token: subscription.token.token,
-        notification_id: notif.id,
-        notification_type: type,
+        notification_id: notification.id,
+        notification_type: mastodon_type,
         icon: avatar_url,
         preferred_locale: "en",
         pleroma: %{
-          activity_id: notif.activity.id,
+          activity_id: notification.activity.id,
           direct_conversation_id: direct_conversation_id
         }
       }
-      |> Map.merge(build_content(notif, actor, object))
+      |> Map.merge(build_content(notification, actor, object, mastodon_type))
       |> Jason.encode!()
       |> push_message(build_sub(subscription), gcm_api_key, subscription)
     end
@@ -82,7 +84,7 @@ defmodule Pleroma.Web.Push.Impl do
   end
 
   @doc "Gets user subscriptions"
-  def fetch_subsriptions(user_id) do
+  def fetch_subscriptions(user_id) do
     Subscription
     |> where(user_id: ^user_id)
     |> preload(:token)
@@ -99,28 +101,36 @@ defmodule Pleroma.Web.Push.Impl do
     }
   end
 
+  def build_content(notification, actor, object, mastodon_type \\ nil)
+
   def build_content(
         %{
           activity: %{data: %{"directMessage" => true}},
           user: %{notification_settings: %{privacy_option: true}}
         },
         actor,
-        _
+        _object,
+        _mastodon_type
       ) do
     %{title: "New Direct Message", body: "@#{actor.nickname}"}
   end
 
-  def build_content(notif, actor, object) do
+  def build_content(notification, actor, object, mastodon_type) do
+    mastodon_type = mastodon_type || mastodon_notification_type(notification.activity)
+
     %{
-      title: format_title(notif),
-      body: format_body(notif, actor, object)
+      title: format_title(notification, mastodon_type),
+      body: format_body(notification, actor, object, mastodon_type)
     }
   end
 
+  def format_body(activity, actor, object, mastodon_type \\ nil)
+
   def format_body(
         %{activity: %{data: %{"type" => "Create"}}},
         actor,
-        %{data: %{"content" => content}}
+        %{data: %{"content" => content}},
+        _mastodon_type
       ) do
     "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
@@ -128,33 +138,44 @@ defmodule Pleroma.Web.Push.Impl do
   def format_body(
         %{activity: %{data: %{"type" => "Announce"}}},
         actor,
-        %{data: %{"content" => content}}
+        %{data: %{"content" => content}},
+        _mastodon_type
       ) do
     "@#{actor.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}"
   end
 
   def format_body(
-        %{activity: %{data: %{"type" => type}}},
+        %{activity: %{data: %{"type" => type}}} = notification,
         actor,
-        _object
+        _object,
+        mastodon_type
       )
       when type in ["Follow", "Like"] do
-    case type do
-      "Follow" -> "@#{actor.nickname} has followed you"
-      "Like" -> "@#{actor.nickname} has favorited your post"
+    mastodon_type = mastodon_type || mastodon_notification_type(notification.activity)
+
+    case mastodon_type do
+      "follow" -> "@#{actor.nickname} has followed you"
+      "follow_request" -> "@#{actor.nickname} has requested to follow you"
+      "favourite" -> "@#{actor.nickname} has favorited your post"
     end
   end
 
-  def format_title(%{activity: %{data: %{"directMessage" => true}}}) do
+  def format_title(activity, mastodon_type \\ nil)
+
+  def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_type) do
     "New Direct Message"
   end
 
-  def format_title(%{activity: %{data: %{"type" => type}}}) do
-    case type do
-      "Create" -> "New Mention"
-      "Follow" -> "New Follower"
-      "Announce" -> "New Repeat"
-      "Like" -> "New Favorite"
+  def format_title(%{activity: activity}, mastodon_type) do
+    mastodon_type = mastodon_type || mastodon_notification_type(activity)
+
+    case mastodon_type do
+      "mention" -> "New Mention"
+      "follow" -> "New Follower"
+      "follow_request" -> "New Follow Request"
+      "reblog" -> "New Repeat"
+      "favourite" -> "New Favorite"
+      type -> "New #{String.capitalize(type || "event")}"
     end
   end
 end
diff --git a/lib/pleroma/web/push/subscription.ex b/lib/pleroma/web/push/subscription.ex
index 5c448d6c9..b99b0c5fb 100644
--- a/lib/pleroma/web/push/subscription.ex
+++ b/lib/pleroma/web/push/subscription.ex
@@ -32,6 +32,14 @@ defmodule Pleroma.Web.Push.Subscription do
     %{"alerts" => alerts}
   end
 
+  def enabled?(subscription, "follow_request") do
+    enabled?(subscription, "follow")
+  end
+
+  def enabled?(subscription, alert_type) do
+    get_in(subscription.data, ["alerts", alert_type])
+  end
+
   def create(
         %User{} = user,
         %Token{} = token,
-- 
cgit v1.2.3


From c75840f7b8a117e973713c8cb47523f09deaab79 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 2 May 2020 12:24:49 +0000
Subject: Merge branch 'follow-request-notifications' into 'develop'

Follow request notifications enforcement

See merge request pleroma/pleroma!2451
---
 lib/pleroma/notification.ex | 11 +----------
 1 file changed, 1 insertion(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 94dc0c2b0..815356a5e 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -293,17 +293,8 @@ defmodule Pleroma.Notification do
     end
   end
 
-  def create_notifications(%Activity{data: %{"type" => "Follow"}} = activity) do
-    if Pleroma.Config.get([:notifications, :enable_follow_request_notifications]) ||
-         Activity.follow_accepted?(activity) do
-      do_create_notifications(activity)
-    else
-      {:ok, []}
-    end
-  end
-
   def create_notifications(%Activity{data: %{"type" => type}} = activity)
-      when type in ["Like", "Announce", "Move", "EmojiReact"] do
+      when type in ["Follow", "Like", "Announce", "Move", "EmojiReact"] do
     do_create_notifications(activity)
   end
 
-- 
cgit v1.2.3


From 441e28f36562492ad076443c4d061b33fc0f9813 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Thu, 30 Apr 2020 15:02:35 +0300
Subject: Dismiss the follow request notification on rejection

---
 lib/pleroma/notification.ex              | 10 ++++++++++
 lib/pleroma/web/common_api/common_api.ex |  2 ++
 2 files changed, 12 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 815356a5e..556075fba 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -271,6 +271,16 @@ defmodule Pleroma.Notification do
     |> Repo.delete_all()
   end
 
+  def dismiss(%Pleroma.Activity{} = activity) do
+    Notification
+    |> where([n], n.activity_id == ^activity.id)
+    |> Repo.delete_all()
+    |> case do
+      {_, notifications} -> {:ok, notifications}
+      _ -> {:error, "Cannot dismiss notification"}
+    end
+  end
+
   def dismiss(%{id: user_id} = _user, id) do
     notification = Repo.get(Notification, id)
 
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index c84782008..74e9e8cfa 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.ActivityExpiration
   alias Pleroma.Conversation.Participation
   alias Pleroma.FollowingRelationship
+  alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.ThreadMute
   alias Pleroma.User
@@ -58,6 +59,7 @@ defmodule Pleroma.Web.CommonAPI do
     with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed),
          {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "reject"),
          {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_reject),
+         {:ok, _notifications} <- Notification.dismiss(follow_activity),
          {:ok, _activity} <-
            ActivityPub.reject(%{
              to: [follower.ap_id],
-- 
cgit v1.2.3


From bbdad8556861c60ae1f526f63de9c5857c4ad547 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 8 May 2020 23:06:47 +0300
Subject: Initial implementation of image preview proxy. Media proxy tests
 refactoring.

---
 lib/pleroma/helpers/mogrify_helper.ex              | 25 +++++++
 lib/pleroma/web/mastodon_api/views/status_view.ex  |  3 +-
 lib/pleroma/web/media_proxy/media_proxy.ex         | 53 +++++++++++++--
 .../web/media_proxy/media_proxy_controller.ex      | 76 ++++++++++++++++++----
 lib/pleroma/web/router.ex                          |  2 +
 5 files changed, 141 insertions(+), 18 deletions(-)
 create mode 100644 lib/pleroma/helpers/mogrify_helper.ex

(limited to 'lib')

diff --git a/lib/pleroma/helpers/mogrify_helper.ex b/lib/pleroma/helpers/mogrify_helper.ex
new file mode 100644
index 000000000..67edb35c3
--- /dev/null
+++ b/lib/pleroma/helpers/mogrify_helper.ex
@@ -0,0 +1,25 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.MogrifyHelper do
+  @moduledoc """
+  Handles common Mogrify operations.
+  """
+
+  @spec store_as_temporary_file(String.t(), binary()) :: {:ok, String.t()} | {:error, atom()}
+  @doc "Stores binary content fetched from specified URL as a temporary file."
+  def store_as_temporary_file(url, body) do
+    path = Mogrify.temporary_path_for(%{path: url})
+    with :ok <- File.write(path, body), do: {:ok, path}
+  end
+
+  @spec store_as_temporary_file(String.t(), String.t()) :: Mogrify.Image.t() | any()
+  @doc "Modifies file at specified path by resizing to specified limit dimensions."
+  def in_place_resize_to_limit(path, resize_dimensions) do
+    path
+    |> Mogrify.open()
+    |> Mogrify.resize_to_limit(resize_dimensions)
+    |> Mogrify.save(in_place: true)
+  end
+end
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 24167f66f..2a206f743 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -419,6 +419,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     [attachment_url | _] = attachment["url"]
     media_type = attachment_url["mediaType"] || attachment_url["mimeType"] || "image"
     href = attachment_url["href"] |> MediaProxy.url()
+    href_preview = attachment_url["href"] |> MediaProxy.preview_url()
 
     type =
       cond do
@@ -434,7 +435,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       id: to_string(attachment["id"] || hash_id),
       url: href,
       remote_url: href,
-      preview_url: href,
+      preview_url: href_preview,
       text_url: href,
       type: type,
       description: attachment["name"],
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index b2b524524..f4791c758 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -20,6 +20,14 @@ defmodule Pleroma.Web.MediaProxy do
     end
   end
 
+  def preview_url(url) do
+    if disabled?() or whitelisted?(url) do
+      url
+    else
+      encode_preview_url(url)
+    end
+  end
+
   defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
 
   defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
@@ -43,17 +51,29 @@ defmodule Pleroma.Web.MediaProxy do
     end)
   end
 
-  def encode_url(url) do
+  defp base64_sig64(url) do
     base64 = Base.url_encode64(url, @base64_opts)
 
     sig64 =
       base64
-      |> signed_url
+      |> signed_url()
       |> Base.url_encode64(@base64_opts)
 
+    {base64, sig64}
+  end
+
+  def encode_url(url) do
+    {base64, sig64} = base64_sig64(url)
+
     build_url(sig64, base64, filename(url))
   end
 
+  def encode_preview_url(url) do
+    {base64, sig64} = base64_sig64(url)
+
+    build_preview_url(sig64, base64, filename(url))
+  end
+
   def decode_url(sig, url) do
     with {:ok, sig} <- Base.url_decode64(sig, @base64_opts),
          signature when signature == sig <- signed_url(url) do
@@ -71,10 +91,10 @@ defmodule Pleroma.Web.MediaProxy do
     if path = URI.parse(url_or_path).path, do: Path.basename(path)
   end
 
-  def build_url(sig_base64, url_base64, filename \\ nil) do
+  defp proxy_url(path, sig_base64, url_base64, filename) do
     [
       Pleroma.Config.get([:media_proxy, :base_url], Web.base_url()),
-      "proxy",
+      path,
       sig_base64,
       url_base64,
       filename
@@ -82,4 +102,29 @@ defmodule Pleroma.Web.MediaProxy do
     |> Enum.filter(& &1)
     |> Path.join()
   end
+
+  def build_url(sig_base64, url_base64, filename \\ nil) do
+    proxy_url("proxy", sig_base64, url_base64, filename)
+  end
+
+  def build_preview_url(sig_base64, url_base64, filename \\ nil) do
+    proxy_url("proxy/preview", sig_base64, url_base64, filename)
+  end
+
+  def filename_matches(%{"filename" => _} = _, path, url) do
+    filename = filename(url)
+
+    if filename && not basename_matches?(path, filename) do
+      {:wrong_filename, filename}
+    else
+      :ok
+    end
+  end
+
+  def filename_matches(_, _, _), do: :ok
+
+  defp basename_matches?(path, filename) do
+    basename = Path.basename(path)
+    basename == filename or URI.decode(basename) == filename or URI.encode(basename) == filename
+  end
 end
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 4657a4383..fe3f61c18 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -5,19 +5,21 @@
 defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Config
+  alias Pleroma.Helpers.MogrifyHelper
   alias Pleroma.ReverseProxy
   alias Pleroma.Web.MediaProxy
 
   @default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
 
   def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
-    with config <- Pleroma.Config.get([:media_proxy], []),
-         true <- Keyword.get(config, :enabled, false),
+    with config <- Config.get([:media_proxy], []),
+         {_, true} <- {:enabled, Keyword.get(config, :enabled, false)},
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
-         :ok <- filename_matches(params, conn.request_path, url) do
+         :ok <- MediaProxy.filename_matches(params, conn.request_path, url) do
       ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
     else
-      false ->
+      {:enabled, false} ->
         send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
 
       {:error, :invalid_signature} ->
@@ -28,20 +30,68 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  def filename_matches(%{"filename" => _} = _, path, url) do
-    filename = MediaProxy.filename(url)
+  def preview(conn, %{"sig" => sig64, "url" => url64} = params) do
+    with {_, true} <- {:enabled, Config.get([:media_preview_proxy, :enabled], false)},
+         {:ok, url} <- MediaProxy.decode_url(sig64, url64),
+         :ok <- MediaProxy.filename_matches(params, conn.request_path, url) do
+      handle_preview(conn, url)
+    else
+      {:enabled, false} ->
+        send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
+
+      {:error, :invalid_signature} ->
+        send_resp(conn, 403, Plug.Conn.Status.reason_phrase(403))
+
+      {:wrong_filename, filename} ->
+        redirect(conn, external: MediaProxy.build_preview_url(sig64, url64, filename))
+    end
+  end
 
-    if filename && does_not_match(path, filename) do
-      {:wrong_filename, filename}
+  defp handle_preview(conn, url) do
+    with {:ok, %{status: status} = head_response} when status in 200..299 <- Tesla.head(url),
+         {_, true} <- {:acceptable_content_length, acceptable_body_length?(head_response)} do
+      content_type = Tesla.get_header(head_response, "content-type")
+      handle_preview(content_type, conn, url)
     else
-      :ok
+      {_, %{status: status}} ->
+        send_resp(conn, :failed_dependency, "Can't fetch HTTP headers (HTTP #{status}).")
+
+      {:acceptable_content_length, false} ->
+        send_resp(conn, :unprocessable_entity, "Source file size exceeds limit.")
     end
   end
 
-  def filename_matches(_, _, _), do: :ok
+  defp handle_preview("image/" <> _, %{params: params} = conn, url) do
+    with {:ok, %{status: status, body: body}} when status in 200..299 <- Tesla.get(url),
+         {:ok, path} <- MogrifyHelper.store_as_temporary_file(url, body),
+         resize_dimensions <-
+           Map.get(
+             params,
+             "limit_dimensions",
+             Config.get([:media_preview_proxy, :limit_dimensions])
+           ),
+         %Mogrify.Image{} <- MogrifyHelper.in_place_resize_to_limit(path, resize_dimensions) do
+      send_file(conn, 200, path)
+    else
+      {_, %{status: _}} ->
+        send_resp(conn, :failed_dependency, "Can't fetch the image.")
+
+      _ ->
+        send_resp(conn, :failed_dependency, "Can't handle image preview.")
+    end
+  end
+
+  defp handle_preview(content_type, conn, _url) do
+    send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
+  end
+
+  defp acceptable_body_length?(head_response) do
+    max_body_length = Config.get([:media_preview_proxy, :max_body_length], nil)
+    content_length = Tesla.get_header(head_response, "content-length")
+    content_length = with {int, _} <- Integer.parse(content_length), do: int
 
-  defp does_not_match(path, filename) do
-    basename = Path.basename(path)
-    basename != filename and URI.decode(basename) != filename and URI.encode(basename) != filename
+    content_length == :error or
+      max_body_length in [nil, :infinity] or
+      content_length <= max_body_length
   end
 end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 7a171f9fb..6fb47029a 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -663,6 +663,8 @@ defmodule Pleroma.Web.Router do
   end
 
   scope "/proxy/", Pleroma.Web.MediaProxy do
+    get("/preview/:sig/:url", MediaProxyController, :preview)
+    get("/preview/:sig/:url/:filename", MediaProxyController, :preview)
     get("/:sig/:url", MediaProxyController, :remote)
     get("/:sig/:url/:filename", MediaProxyController, :remote)
   end
-- 
cgit v1.2.3


From 45df70e691495d383a9ceedd620c03a5d3a875ec Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 5 May 2020 10:12:37 +0200
Subject: AP C2S: Restrict creation to `Note`s for now.

---
 lib/pleroma/web/activity_pub/activity_pub_controller.ex | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 779de0e4d..2bb5bd15b 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -370,7 +370,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     |> json(err)
   end
 
-  def handle_user_activity(user, %{"type" => "Create"} = params) do
+  defp handle_user_activity(
+         %User{} = user,
+         %{"type" => "Create", "object" => %{"type" => "Note"}} = params
+       ) do
     object =
       params["object"]
       |> Map.merge(Map.take(params, ["to", "cc"]))
@@ -386,7 +389,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     })
   end
 
-  def handle_user_activity(user, %{"type" => "Delete"} = params) do
+  defp handle_user_activity(user, %{"type" => "Delete"} = params) do
     with %Object{} = object <- Object.normalize(params["object"]),
          true <- user.is_moderator || user.ap_id == object.data["actor"],
          {:ok, delete} <- ActivityPub.delete(object) do
@@ -396,7 +399,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
-  def handle_user_activity(user, %{"type" => "Like"} = params) do
+  defp handle_user_activity(user, %{"type" => "Like"} = params) do
     with %Object{} = object <- Object.normalize(params["object"]),
          {:ok, activity, _object} <- ActivityPub.like(user, object) do
       {:ok, activity}
@@ -405,7 +408,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
-  def handle_user_activity(_, _) do
+  defp handle_user_activity(_, _) do
     {:error, dgettext("errors", "Unhandled activity type")}
   end
 
-- 
cgit v1.2.3


From f7c28ae54484e1c5df48c89560e7822d04c7e5eb Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Sun, 3 May 2020 13:48:01 +0200
Subject: Webfinger: Request account info with the acct scheme

---
 lib/pleroma/web/web_finger/web_finger.ex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index 43a81c75d..8f71820d7 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -193,13 +193,15 @@ defmodule Pleroma.Web.WebFinger do
           URI.parse(account).host
       end
 
+    encoded_account = URI.encode("acct:#{account}")
+
     address =
       case find_lrdd_template(domain) do
         {:ok, template} ->
-          String.replace(template, "{uri}", URI.encode(account))
+          String.replace(template, "{uri}", encoded_account)
 
         _ ->
-          "https://#{domain}/.well-known/webfinger?resource=acct:#{account}"
+          "https://#{domain}/.well-known/webfinger?resource=#{encoded_account}"
       end
 
     with response <-
-- 
cgit v1.2.3


From 3d9a7cf0cc235f4c305c065e264a77d3c9e7d0e3 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 8 May 2020 23:51:59 +0300
Subject: healthcheck: report real amount of memory allocated by beam

as opposed to memory currently in use
---
 lib/pleroma/healthcheck.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/healthcheck.ex b/lib/pleroma/healthcheck.ex
index 8f7f43ec2..92ce83cb7 100644
--- a/lib/pleroma/healthcheck.ex
+++ b/lib/pleroma/healthcheck.ex
@@ -29,7 +29,7 @@ defmodule Pleroma.Healthcheck do
   @spec system_info() :: t()
   def system_info do
     %Healthcheck{
-      memory_used: Float.round(:erlang.memory(:total) / 1024 / 1024, 2)
+      memory_used: Float.round(:recon_alloc.memory(:allocated) / 1024 / 1024, 2)
     }
     |> assign_db_info()
     |> assign_job_queue_stats()
-- 
cgit v1.2.3


From 1b23acf164ebc4fde3fe1e4fdca6e11b7caa90ef Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Mon, 11 May 2020 23:21:53 +0300
Subject: [#2497] Media preview proxy for images: fixes, tweaks, refactoring,
 tests adjustments.

---
 lib/pleroma/reverse_proxy/reverse_proxy.ex         |  4 ++
 lib/pleroma/web/media_proxy/media_proxy.ex         | 33 ++++++++----
 .../web/media_proxy/media_proxy_controller.ex      | 62 +++++++++++++---------
 3 files changed, 64 insertions(+), 35 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex
index 4bbeb493c..aeaf9bd39 100644
--- a/lib/pleroma/reverse_proxy/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex
@@ -16,6 +16,8 @@ defmodule Pleroma.ReverseProxy do
   @failed_request_ttl :timer.seconds(60)
   @methods ~w(GET HEAD)
 
+  def max_read_duration_default, do: @max_read_duration
+
   @moduledoc """
   A reverse proxy.
 
@@ -370,6 +372,8 @@ defmodule Pleroma.ReverseProxy do
 
   defp body_size_constraint(_, _), do: :ok
 
+  defp check_read_duration(nil = _duration, max), do: check_read_duration(@max_read_duration, max)
+
   defp check_read_duration(duration, max)
        when is_integer(duration) and is_integer(max) and max > 0 do
     if duration > max do
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index f4791c758..4e01c14e4 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -13,26 +13,32 @@ defmodule Pleroma.Web.MediaProxy do
   def url("/" <> _ = url), do: url
 
   def url(url) do
-    if disabled?() or local?(url) or whitelisted?(url) do
+    if not enabled?() or local?(url) or whitelisted?(url) do
       url
     else
       encode_url(url)
     end
   end
 
+  # Note: routing all URLs to preview handler (even local and whitelisted).
+  #   Preview handler will call url/1 on decoded URLs, and applicable ones will detour media proxy.
   def preview_url(url) do
-    if disabled?() or whitelisted?(url) do
-      url
-    else
+    if preview_enabled?() do
       encode_preview_url(url)
+    else
+      url
     end
   end
 
-  defp disabled?, do: !Config.get([:media_proxy, :enabled], false)
+  def enabled?, do: Config.get([:media_proxy, :enabled], false)
 
-  defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
+  # Note: media proxy must be enabled for media preview proxy in order to load all
+  #   non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
+  def preview_enabled?, do: enabled?() and Config.get([:media_preview_proxy, :enabled], false)
 
-  defp whitelisted?(url) do
+  def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
+
+  def whitelisted?(url) do
     %{host: domain} = URI.parse(url)
 
     mediaproxy_whitelist = Config.get([:media_proxy, :whitelist])
@@ -111,17 +117,24 @@ defmodule Pleroma.Web.MediaProxy do
     proxy_url("proxy/preview", sig_base64, url_base64, filename)
   end
 
-  def filename_matches(%{"filename" => _} = _, path, url) do
+  def verify_request_path_and_url(
+        %Plug.Conn{params: %{"filename" => _}, request_path: request_path},
+        url
+      ) do
+    verify_request_path_and_url(request_path, url)
+  end
+
+  def verify_request_path_and_url(request_path, url) when is_binary(request_path) do
     filename = filename(url)
 
-    if filename && not basename_matches?(path, filename) do
+    if filename && not basename_matches?(request_path, filename) do
       {:wrong_filename, filename}
     else
       :ok
     end
   end
 
-  def filename_matches(_, _, _), do: :ok
+  def verify_request_path_and_url(_, _), do: :ok
 
   defp basename_matches?(path, filename) do
     basename = Path.basename(path)
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index fe3f61c18..157365e08 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -10,14 +10,12 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   alias Pleroma.ReverseProxy
   alias Pleroma.Web.MediaProxy
 
-  @default_proxy_opts [max_body_length: 25 * 1_048_576, http: [follow_redirect: true]]
-
-  def remote(conn, %{"sig" => sig64, "url" => url64} = params) do
-    with config <- Config.get([:media_proxy], []),
-         {_, true} <- {:enabled, Keyword.get(config, :enabled, false)},
+  def remote(conn, %{"sig" => sig64, "url" => url64}) do
+    with {_, true} <- {:enabled, MediaProxy.enabled?()},
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
-         :ok <- MediaProxy.filename_matches(params, conn.request_path, url) do
-      ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts))
+         :ok <- MediaProxy.verify_request_path_and_url(conn, url) do
+      proxy_opts = Config.get([:media_proxy, :proxy_opts], [])
+      ReverseProxy.call(conn, url, proxy_opts)
     else
       {:enabled, false} ->
         send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
@@ -30,10 +28,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  def preview(conn, %{"sig" => sig64, "url" => url64} = params) do
-    with {_, true} <- {:enabled, Config.get([:media_preview_proxy, :enabled], false)},
+  def preview(conn, %{"sig" => sig64, "url" => url64}) do
+    with {_, true} <- {:enabled, MediaProxy.preview_enabled?()},
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
-         :ok <- MediaProxy.filename_matches(params, conn.request_path, url) do
+         :ok <- MediaProxy.verify_request_path_and_url(conn, url) do
       handle_preview(conn, url)
     else
       {:enabled, false} ->
@@ -48,21 +46,27 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   end
 
   defp handle_preview(conn, url) do
-    with {:ok, %{status: status} = head_response} when status in 200..299 <- Tesla.head(url),
-         {_, true} <- {:acceptable_content_length, acceptable_body_length?(head_response)} do
+    with {:ok, %{status: status} = head_response} when status in 200..299 <-
+           Tesla.head(url, opts: [adapter: [timeout: preview_head_request_timeout()]]) do
       content_type = Tesla.get_header(head_response, "content-type")
       handle_preview(content_type, conn, url)
     else
       {_, %{status: status}} ->
         send_resp(conn, :failed_dependency, "Can't fetch HTTP headers (HTTP #{status}).")
 
-      {:acceptable_content_length, false} ->
-        send_resp(conn, :unprocessable_entity, "Source file size exceeds limit.")
+      {:error, :recv_response_timeout} ->
+        send_resp(conn, :failed_dependency, "HEAD request timeout.")
+
+      _ ->
+        send_resp(conn, :failed_dependency, "Can't fetch HTTP headers.")
     end
   end
 
-  defp handle_preview("image/" <> _, %{params: params} = conn, url) do
-    with {:ok, %{status: status, body: body}} when status in 200..299 <- Tesla.get(url),
+  defp handle_preview("image/" <> _ = content_type, %{params: params} = conn, url) do
+    with {:ok, %{status: status, body: body}} when status in 200..299 <-
+           url
+           |> MediaProxy.url()
+           |> Tesla.get(opts: [adapter: [timeout: preview_timeout()]]),
          {:ok, path} <- MogrifyHelper.store_as_temporary_file(url, body),
          resize_dimensions <-
            Map.get(
@@ -70,12 +74,19 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
              "limit_dimensions",
              Config.get([:media_preview_proxy, :limit_dimensions])
            ),
-         %Mogrify.Image{} <- MogrifyHelper.in_place_resize_to_limit(path, resize_dimensions) do
-      send_file(conn, 200, path)
+         %Mogrify.Image{} <- MogrifyHelper.in_place_resize_to_limit(path, resize_dimensions),
+         {:ok, image_binary} <- File.read(path),
+         _ <- File.rm(path) do
+      conn
+      |> put_resp_header("content-type", content_type)
+      |> send_resp(200, image_binary)
     else
       {_, %{status: _}} ->
         send_resp(conn, :failed_dependency, "Can't fetch the image.")
 
+      {:error, :recv_response_timeout} ->
+        send_resp(conn, :failed_dependency, "Downstream timeout.")
+
       _ ->
         send_resp(conn, :failed_dependency, "Can't handle image preview.")
     end
@@ -85,13 +96,14 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
   end
 
-  defp acceptable_body_length?(head_response) do
-    max_body_length = Config.get([:media_preview_proxy, :max_body_length], nil)
-    content_length = Tesla.get_header(head_response, "content-length")
-    content_length = with {int, _} <- Integer.parse(content_length), do: int
+  defp preview_head_request_timeout do
+    Config.get([:media_preview_proxy, :proxy_opts, :head_request_max_read_duration]) ||
+      preview_timeout()
+  end
 
-    content_length == :error or
-      max_body_length in [nil, :infinity] or
-      content_length <= max_body_length
+  defp preview_timeout do
+    Config.get([:media_preview_proxy, :proxy_opts, :max_read_duration]) ||
+      Config.get([:media_proxy, :proxy_opts, :max_read_duration]) ||
+      ReverseProxy.max_read_duration_default()
   end
 end
-- 
cgit v1.2.3


From 37c2deb3ddcd473be9a55bbbd48b370b67a0384a Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Tue, 12 May 2020 21:06:23 +0200
Subject: Expand and authorize streams in Streamer directly (backport from
 !2519)

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 69 +++++------------------
 lib/pleroma/web/streamer/state.ex                 | 19 ++-----
 lib/pleroma/web/streamer/streamer.ex              | 65 +++++++++++++++++++++
 3 files changed, 83 insertions(+), 70 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 5652a37c1..b1aebe014 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -12,29 +12,15 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
 
   @behaviour :cowboy_websocket
 
-  @streams [
-    "public",
-    "public:local",
-    "public:media",
-    "public:local:media",
-    "user",
-    "user:notification",
-    "direct",
-    "list",
-    "hashtag"
-  ]
-  @anonymous_streams ["public", "public:local", "hashtag"]
-
   # Handled by periodic keepalive in Pleroma.Web.Streamer.Ping.
   @timeout :infinity
 
   def init(%{qs: qs} = req, state) do
-    with params <- :cow_qs.parse_qs(qs),
+    with params <- Enum.into(:cow_qs.parse_qs(qs), %{}),
          sec_websocket <- :cowboy_req.header("sec-websocket-protocol", req, nil),
-         access_token <- List.keyfind(params, "access_token", 0),
-         {_, stream} <- List.keyfind(params, "stream", 0),
-         {:ok, user} <- allow_request(stream, [access_token, sec_websocket]),
-         topic when is_binary(topic) <- expand_topic(stream, params) do
+         access_token <- Map.get(params, "access_token"),
+         {:ok, user} <- authenticate_request(access_token, sec_websocket),
+         {:ok, topic} <- Streamer.get_topic(Map.get(params, "stream"), user, params) do
       req =
         if sec_websocket do
           :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
@@ -44,14 +30,14 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
 
       {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}}
     else
-      {:error, code} ->
-        Logger.debug("#{__MODULE__} denied connection: #{inspect(code)} - #{inspect(req)}")
-        {:ok, req} = :cowboy_req.reply(code, req)
+      {:error, :bad_topic} ->
+        Logger.debug("#{__MODULE__} bad topic #{inspect(req)}")
+        {:ok, req} = :cowboy_req.reply(404, req)
         {:ok, req, state}
 
-      error ->
-        Logger.debug("#{__MODULE__} denied connection: #{inspect(error)} - #{inspect(req)}")
-        {:ok, req} = :cowboy_req.reply(400, req)
+      {:error, :unauthorized} ->
+        Logger.debug("#{__MODULE__} authentication error: #{inspect(req)}")
+        {:ok, req} = :cowboy_req.reply(401, req)
         {:ok, req, state}
     end
   end
@@ -93,50 +79,23 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
   end
 
   # Public streams without authentication.
-  defp allow_request(stream, [nil, nil]) when stream in @anonymous_streams do
+  defp authenticate_request(nil, nil) do
     {:ok, nil}
   end
 
   # Authenticated streams.
-  defp allow_request(stream, [access_token, sec_websocket]) when stream in @streams do
-    token =
-      with {"access_token", token} <- access_token do
-        token
-      else
-        _ -> sec_websocket
-      end
+  defp authenticate_request(access_token, sec_websocket) do
+    token = access_token || sec_websocket
 
     with true <- is_bitstring(token),
          %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
          user = %User{} <- User.get_cached_by_id(user_id) do
       {:ok, user}
     else
-      _ -> {:error, 403}
-    end
-  end
-
-  # Not authenticated.
-  defp allow_request(stream, _) when stream in @streams, do: {:error, 403}
-
-  # No matching stream.
-  defp allow_request(_, _), do: {:error, 404}
-
-  defp expand_topic("hashtag", params) do
-    case List.keyfind(params, "tag", 0) do
-      {_, tag} -> "hashtag:#{tag}"
-      _ -> nil
+      _ -> {:error, :unauthorized}
     end
   end
 
-  defp expand_topic("list", params) do
-    case List.keyfind(params, "list", 0) do
-      {_, list} -> "list:#{list}"
-      _ -> nil
-    end
-  end
-
-  defp expand_topic(topic, _), do: topic
-
   defp streamer_socket(state) do
     %{transport_pid: self(), assigns: state}
   end
diff --git a/lib/pleroma/web/streamer/state.ex b/lib/pleroma/web/streamer/state.ex
index 999550b88..4eb462a1a 100644
--- a/lib/pleroma/web/streamer/state.ex
+++ b/lib/pleroma/web/streamer/state.ex
@@ -36,30 +36,28 @@ defmodule Pleroma.Web.Streamer.State do
   end
 
   def handle_call({:add, topic, socket}, _from, %{sockets: sockets} = state) do
-    internal_topic = internal_topic(topic, socket)
     stream_socket = StreamerSocket.from_socket(socket)
 
     sockets_for_topic =
       sockets
-      |> Map.get(internal_topic, [])
+      |> Map.get(topic, [])
       |> List.insert_at(0, stream_socket)
       |> Enum.uniq()
 
-    state = put_in(state, [:sockets, internal_topic], sockets_for_topic)
+    state = put_in(state, [:sockets, topic], sockets_for_topic)
     Logger.debug("Got new conn for #{topic}")
     {:reply, state, state}
   end
 
   def handle_call({:remove, topic, socket}, _from, %{sockets: sockets} = state) do
-    internal_topic = internal_topic(topic, socket)
     stream_socket = StreamerSocket.from_socket(socket)
 
     sockets_for_topic =
       sockets
-      |> Map.get(internal_topic, [])
+      |> Map.get(topic, [])
       |> List.delete(stream_socket)
 
-    state = Kernel.put_in(state, [:sockets, internal_topic], sockets_for_topic)
+    state = Kernel.put_in(state, [:sockets, topic], sockets_for_topic)
     {:reply, state, state}
   end
 
@@ -70,13 +68,4 @@ defmodule Pleroma.Web.Streamer.State do
   defp do_remove_socket(_env, topic, socket) do
     GenServer.call(__MODULE__, {:remove, topic, socket})
   end
-
-  defp internal_topic(topic, socket)
-       when topic in ~w[user user:notification direct] do
-    "#{topic}:#{socket.assigns[:user].id}"
-  end
-
-  defp internal_topic(topic, _) do
-    topic
-  end
 end
diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 814d5a729..1e5700b6a 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -5,10 +5,75 @@
 defmodule Pleroma.Web.Streamer do
   alias Pleroma.Web.Streamer.State
   alias Pleroma.Web.Streamer.Worker
+  alias Pleroma.User
 
   @timeout 60_000
   @mix_env Mix.env()
 
+  @public_streams ["public", "public:local", "public:media", "public:local:media"]
+  @user_streams ["user", "user:notification", "direct"]
+
+  @doc "Expands and authorizes a stream, and registers the process for streaming."
+  @spec get_topic_and_add_socket(stream :: String.t(), State.t(), Map.t() | nil) ::
+          {:ok, topic :: String.t()} | {:error, :bad_topic} | {:error, :unauthorized}
+  def get_topic_and_add_socket(stream, socket, params \\ %{}) do
+    user =
+      case socket do
+        %{assigns: %{user: user}} -> user
+        _ -> nil
+      end
+
+    case get_topic(stream, user, params) do
+      {:ok, topic} ->
+        add_socket(topic, socket)
+        {:ok, topic}
+
+      error ->
+        error
+    end
+  end
+
+  @doc "Expand and authorizes a stream"
+  @spec get_topic(stream :: String.t(), User.t() | nil, Map.t()) ::
+          {:ok, topic :: String.t()} | {:error, :bad_topic}
+  def get_topic(stream, user, params \\ %{})
+
+  # Allow all public steams.
+  def get_topic(stream, _, _) when stream in @public_streams do
+    {:ok, stream}
+  end
+
+  # Allow all hashtags streams.
+  def get_topic("hashtag", _, %{"tag" => tag}) do
+    {:ok, "hashtag:" <> tag}
+  end
+
+  # Expand user streams.
+  def get_topic(stream, %User{} = user, _) when stream in @user_streams do
+    {:ok, stream <> ":" <> to_string(user.id)}
+  end
+
+  def get_topic(stream, _, _) when stream in @user_streams do
+    {:error, :unauthorized}
+  end
+
+  # List streams.
+  def get_topic("list", %User{} = user, %{"list" => id}) do
+    if Pleroma.List.get(id, user) do
+      {:ok, "list:" <> to_string(id)}
+    else
+      {:error, :bad_topic}
+    end
+  end
+
+  def get_topic("list", _, _) do
+    {:error, :unauthorized}
+  end
+
+  def get_topic(_, _, _) do
+    {:error, :bad_topic}
+  end
+
   def add_socket(topic, socket) do
     State.add_socket(topic, socket)
   end
-- 
cgit v1.2.3


From 40d0775c7f0ed8c52a513146bf04df3f783a8eb1 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Tue, 12 May 2020 21:27:54 +0200
Subject: Reorder alias

---
 lib/pleroma/web/streamer/streamer.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 1e5700b6a..b7294d084 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -3,9 +3,9 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.Streamer do
+  alias Pleroma.User
   alias Pleroma.Web.Streamer.State
   alias Pleroma.Web.Streamer.Worker
-  alias Pleroma.User
 
   @timeout 60_000
   @mix_env Mix.env()
-- 
cgit v1.2.3


From ec5e05780292710a3454b21d32a1af053cb603e0 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 12 May 2020 12:29:37 +0200
Subject: Transmogrifier: On incoming follow accept, update follow counts.

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 831739c5f..3fc4762d6 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -544,6 +544,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, follow_activity} <- Utils.update_follow_state_for_all(follow_activity, "accept"),
          %User{local: true} = follower <- User.get_cached_by_ap_id(follow_activity.data["actor"]),
          {:ok, _relationship} <- FollowingRelationship.update(follower, followed, :follow_accept) do
+      User.update_follower_count(followed)
+      User.update_following_count(follower)
+
       ActivityPub.accept(%{
         to: follow_activity.data["to"],
         type: "Accept",
@@ -553,7 +556,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         activity_id: id
       })
     else
-      _e -> :error
+      _e ->
+        :error
     end
   end
 
-- 
cgit v1.2.3


From a8abf1ada6d00448533917c8d51cf5907ccd94d8 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 12 May 2020 10:52:46 +0200
Subject: ActivityPub: Fix non-federating blocks.

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 4 +---
 lib/pleroma/web/activity_pub/utils.ex        | 8 ++++++--
 2 files changed, 7 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 5f895406d..c4f83f9e1 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -604,7 +604,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp do_block(blocker, blocked, activity_id, local) do
-    outgoing_blocks = Config.get([:activitypub, :outgoing_blocks])
     unfollow_blocked = Config.get([:activitypub, :unfollow_blocked])
 
     if unfollow_blocked do
@@ -612,8 +611,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       if follow_activity, do: unfollow(blocker, blocked, nil, local)
     end
 
-    with true <- outgoing_blocks,
-         block_data <- make_block_data(blocker, blocked, activity_id),
+    with block_data <- make_block_data(blocker, blocked, activity_id),
          {:ok, activity} <- insert(block_data, local),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index 15dd2ed45..a49cfa35e 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   alias Ecto.Changeset
   alias Ecto.UUID
   alias Pleroma.Activity
+  alias Pleroma.Config
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -169,8 +170,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   Enqueues an activity for federation if it's local
   """
   @spec maybe_federate(any()) :: :ok
-  def maybe_federate(%Activity{local: true} = activity) do
-    if Pleroma.Config.get!([:instance, :federating]) do
+  def maybe_federate(%Activity{local: true, data: %{"type" => type}} = activity) do
+    outgoing_blocks = Config.get([:activitypub, :outgoing_blocks])
+
+    with true <- Config.get!([:instance, :federating]),
+         true <- type != "Block" || outgoing_blocks do
       Pleroma.Web.Federator.publish(activity)
     end
 
-- 
cgit v1.2.3


From 3f13437a9cfc0f13cb4921ee04b7821c1b169021 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 11 May 2020 12:50:25 +0200
Subject: User: Truncate bios when updating a remote user.

---
 lib/pleroma/user.ex | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 2f0333da0..3e8f19e30 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -501,7 +501,15 @@ defmodule Pleroma.User do
 
     params = Map.put(params, :last_refreshed_at, NaiveDateTime.utc_now())
 
-    params = if remote?, do: truncate_fields_param(params), else: params
+    params =
+      if remote? do
+        params
+        |> truncate_fields_param()
+        |> truncate_if_exists(:name, name_limit)
+        |> truncate_if_exists(:bio, bio_limit)
+      else
+        params
+      end
 
     struct
     |> cast(
-- 
cgit v1.2.3


From f1f588fd5271c0b3bf09df002a83dbb57c42bae0 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 14 May 2020 20:18:31 +0300
Subject: [#2497] Added support for :eimp for image resizing.

---
 .../web/media_proxy/media_proxy_controller.ex      | 64 ++++++++++++++++++----
 1 file changed, 52 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 157365e08..8d8d073e9 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -62,24 +62,64 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
+  defp thumbnail_max_dimensions(params) do
+    config = Config.get([:media_preview_proxy], [])
+
+    thumbnail_max_width =
+      if w = params["thumbnail_max_width"] do
+        String.to_integer(w)
+      else
+        Keyword.fetch!(config, :thumbnail_max_width)
+      end
+
+    thumbnail_max_height =
+      if h = params["thumbnail_max_height"] do
+        String.to_integer(h)
+      else
+        Keyword.fetch!(config, :thumbnail_max_height)
+      end
+
+    {thumbnail_max_width, thumbnail_max_height}
+  end
+
+  defp thumbnail_binary(url, body, params) do
+    {thumbnail_max_width, thumbnail_max_height} = thumbnail_max_dimensions(params)
+
+    with true <- Config.get([:media_preview_proxy, :enable_eimp]),
+         {:ok, [type: image_type, width: source_width, height: source_height]} <-
+           :eimp.identify(body),
+         scale_factor <-
+           Enum.max([source_width / thumbnail_max_width, source_height / thumbnail_max_height]),
+         {:ok, thumbnail_binary} =
+           :eimp.convert(body, image_type, [
+             {:scale, {round(source_width / scale_factor), round(source_height / scale_factor)}}
+           ]) do
+      {:ok, thumbnail_binary}
+    else
+      _ ->
+        mogrify_dimensions = "#{thumbnail_max_width}x#{thumbnail_max_height}"
+
+        with {:ok, path} <- MogrifyHelper.store_as_temporary_file(url, body),
+             %Mogrify.Image{} <-
+               MogrifyHelper.in_place_resize_to_limit(path, mogrify_dimensions),
+             {:ok, thumbnail_binary} <- File.read(path),
+             _ <- File.rm(path) do
+          {:ok, thumbnail_binary}
+        else
+          _ -> :error
+        end
+    end
+  end
+
   defp handle_preview("image/" <> _ = content_type, %{params: params} = conn, url) do
-    with {:ok, %{status: status, body: body}} when status in 200..299 <-
+    with {:ok, %{status: status, body: image_contents}} when status in 200..299 <-
            url
            |> MediaProxy.url()
            |> Tesla.get(opts: [adapter: [timeout: preview_timeout()]]),
-         {:ok, path} <- MogrifyHelper.store_as_temporary_file(url, body),
-         resize_dimensions <-
-           Map.get(
-             params,
-             "limit_dimensions",
-             Config.get([:media_preview_proxy, :limit_dimensions])
-           ),
-         %Mogrify.Image{} <- MogrifyHelper.in_place_resize_to_limit(path, resize_dimensions),
-         {:ok, image_binary} <- File.read(path),
-         _ <- File.rm(path) do
+         {:ok, thumbnail_binary} <- thumbnail_binary(url, image_contents, params) do
       conn
       |> put_resp_header("content-type", content_type)
-      |> send_resp(200, image_binary)
+      |> send_resp(200, thumbnail_binary)
     else
       {_, %{status: _}} ->
         send_resp(conn, :failed_dependency, "Can't fetch the image.")
-- 
cgit v1.2.3


From 1871a5ddb4a803ebe4fae6943a9b9c94f1f9c1a8 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 20 May 2020 20:26:43 +0300
Subject: [#2497] Image preview proxy: implemented ffmpeg-based resizing,
 removed eimp & mogrify-based resizing.

---
 lib/pleroma/helpers/media_helper.ex                | 62 ++++++++++++++++++++++
 lib/pleroma/helpers/mogrify_helper.ex              | 25 ---------
 .../web/media_proxy/media_proxy_controller.ex      | 50 ++++-------------
 3 files changed, 71 insertions(+), 66 deletions(-)
 create mode 100644 lib/pleroma/helpers/media_helper.ex
 delete mode 100644 lib/pleroma/helpers/mogrify_helper.ex

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
new file mode 100644
index 000000000..6d1f8ab22
--- /dev/null
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -0,0 +1,62 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.MediaHelper do
+  @moduledoc """
+  Handles common media-related operations.
+  """
+
+  @ffmpeg_opts [{:sync, true}, {:stdout, true}]
+
+  def ffmpeg_resize_remote(uri, max_width, max_height) do
+    cmd = ~s"""
+    curl -L "#{uri}" |
+    ffmpeg -i pipe:0 -vf \
+      "scale='min(#{max_width},iw)':min'(#{max_height},ih)':force_original_aspect_ratio=decrease" \
+      -f image2 pipe:1 | \
+    cat
+    """
+
+    with {:ok, [stdout: stdout_list]} <- Exexec.run(cmd, @ffmpeg_opts) do
+      {:ok, Enum.join(stdout_list)}
+    end
+  end
+
+  @doc "Returns a temporary path for an URI"
+  def temporary_path_for(uri) do
+    name = Path.basename(uri)
+    random = rand_uniform(999_999)
+    Path.join(System.tmp_dir(), "#{random}-#{name}")
+  end
+
+  @doc "Stores binary content fetched from specified URL as a temporary file."
+  @spec store_as_temporary_file(String.t(), binary()) :: {:ok, String.t()} | {:error, atom()}
+  def store_as_temporary_file(url, body) do
+    path = temporary_path_for(url)
+    with :ok <- File.write(path, body), do: {:ok, path}
+  end
+
+  @doc "Modifies image file at specified path by resizing to specified limit dimensions."
+  @spec mogrify_resize_to_limit(String.t(), String.t()) :: :ok | any()
+  def mogrify_resize_to_limit(path, resize_dimensions) do
+    with %Mogrify.Image{} <-
+           path
+           |> Mogrify.open()
+           |> Mogrify.resize_to_limit(resize_dimensions)
+           |> Mogrify.save(in_place: true) do
+      :ok
+    end
+  end
+
+  defp rand_uniform(high) do
+    Code.ensure_loaded(:rand)
+
+    if function_exported?(:rand, :uniform, 1) do
+      :rand.uniform(high)
+    else
+      # Erlang/OTP < 19
+      apply(:crypto, :rand_uniform, [1, high])
+    end
+  end
+end
diff --git a/lib/pleroma/helpers/mogrify_helper.ex b/lib/pleroma/helpers/mogrify_helper.ex
deleted file mode 100644
index 67edb35c3..000000000
--- a/lib/pleroma/helpers/mogrify_helper.ex
+++ /dev/null
@@ -1,25 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Helpers.MogrifyHelper do
-  @moduledoc """
-  Handles common Mogrify operations.
-  """
-
-  @spec store_as_temporary_file(String.t(), binary()) :: {:ok, String.t()} | {:error, atom()}
-  @doc "Stores binary content fetched from specified URL as a temporary file."
-  def store_as_temporary_file(url, body) do
-    path = Mogrify.temporary_path_for(%{path: url})
-    with :ok <- File.write(path, body), do: {:ok, path}
-  end
-
-  @spec store_as_temporary_file(String.t(), String.t()) :: Mogrify.Image.t() | any()
-  @doc "Modifies file at specified path by resizing to specified limit dimensions."
-  def in_place_resize_to_limit(path, resize_dimensions) do
-    path
-    |> Mogrify.open()
-    |> Mogrify.resize_to_limit(resize_dimensions)
-    |> Mogrify.save(in_place: true)
-  end
-end
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 8d8d073e9..fb4b80379 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   use Pleroma.Web, :controller
 
   alias Pleroma.Config
-  alias Pleroma.Helpers.MogrifyHelper
+  alias Pleroma.Helpers.MediaHelper
   alias Pleroma.ReverseProxy
   alias Pleroma.Web.MediaProxy
 
@@ -82,51 +82,19 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     {thumbnail_max_width, thumbnail_max_height}
   end
 
-  defp thumbnail_binary(url, body, params) do
-    {thumbnail_max_width, thumbnail_max_height} = thumbnail_max_dimensions(params)
-
-    with true <- Config.get([:media_preview_proxy, :enable_eimp]),
-         {:ok, [type: image_type, width: source_width, height: source_height]} <-
-           :eimp.identify(body),
-         scale_factor <-
-           Enum.max([source_width / thumbnail_max_width, source_height / thumbnail_max_height]),
-         {:ok, thumbnail_binary} =
-           :eimp.convert(body, image_type, [
-             {:scale, {round(source_width / scale_factor), round(source_height / scale_factor)}}
-           ]) do
-      {:ok, thumbnail_binary}
-    else
-      _ ->
-        mogrify_dimensions = "#{thumbnail_max_width}x#{thumbnail_max_height}"
-
-        with {:ok, path} <- MogrifyHelper.store_as_temporary_file(url, body),
-             %Mogrify.Image{} <-
-               MogrifyHelper.in_place_resize_to_limit(path, mogrify_dimensions),
-             {:ok, thumbnail_binary} <- File.read(path),
-             _ <- File.rm(path) do
-          {:ok, thumbnail_binary}
-        else
-          _ -> :error
-        end
-    end
-  end
-
   defp handle_preview("image/" <> _ = content_type, %{params: params} = conn, url) do
-    with {:ok, %{status: status, body: image_contents}} when status in 200..299 <-
-           url
-           |> MediaProxy.url()
-           |> Tesla.get(opts: [adapter: [timeout: preview_timeout()]]),
-         {:ok, thumbnail_binary} <- thumbnail_binary(url, image_contents, params) do
+    with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
+         media_proxy_url <- MediaProxy.url(url),
+         {:ok, thumbnail_binary} <-
+           MediaHelper.ffmpeg_resize_remote(
+             media_proxy_url,
+             thumbnail_max_width,
+             thumbnail_max_height
+           ) do
       conn
       |> put_resp_header("content-type", content_type)
       |> send_resp(200, thumbnail_binary)
     else
-      {_, %{status: _}} ->
-        send_resp(conn, :failed_dependency, "Can't fetch the image.")
-
-      {:error, :recv_response_timeout} ->
-        send_resp(conn, :failed_dependency, "Downstream timeout.")
-
       _ ->
         send_resp(conn, :failed_dependency, "Can't handle image preview.")
     end
-- 
cgit v1.2.3


From 610343edb318654126d9539775ba4b9ff30c8831 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 21 May 2020 17:35:42 +0300
Subject: [#2497] Image preview proxy: image resize & background color fix with
 ffmpeg -filter_complex.

---
 lib/pleroma/helpers/media_helper.ex                | 47 +++-------------------
 .../web/media_proxy/media_proxy_controller.ex      |  7 ++--
 2 files changed, 9 insertions(+), 45 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 6d1f8ab22..ee6b76c41 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -9,12 +9,14 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   @ffmpeg_opts [{:sync, true}, {:stdout, true}]
 
-  def ffmpeg_resize_remote(uri, max_width, max_height) do
+  def ffmpeg_resize_remote(uri, %{max_width: max_width, max_height: max_height}) do
     cmd = ~s"""
     curl -L "#{uri}" |
-    ffmpeg -i pipe:0 -vf \
-      "scale='min(#{max_width},iw)':min'(#{max_height},ih)':force_original_aspect_ratio=decrease" \
-      -f image2 pipe:1 | \
+    ffmpeg -i pipe:0 -f lavfi -i color=c=white \
+      -filter_complex "[0:v] scale='min(#{max_width},iw)':'min(#{max_height},ih)': \
+        force_original_aspect_ratio=decrease [scaled]; \
+        [1][scaled] scale2ref [bg][img]; [bg] setsar=1 [bg]; [bg][img] overlay=shortest=1" \
+      -f image2 -vcodec mjpeg -frames:v 1 pipe:1 | \
     cat
     """
 
@@ -22,41 +24,4 @@ defmodule Pleroma.Helpers.MediaHelper do
       {:ok, Enum.join(stdout_list)}
     end
   end
-
-  @doc "Returns a temporary path for an URI"
-  def temporary_path_for(uri) do
-    name = Path.basename(uri)
-    random = rand_uniform(999_999)
-    Path.join(System.tmp_dir(), "#{random}-#{name}")
-  end
-
-  @doc "Stores binary content fetched from specified URL as a temporary file."
-  @spec store_as_temporary_file(String.t(), binary()) :: {:ok, String.t()} | {:error, atom()}
-  def store_as_temporary_file(url, body) do
-    path = temporary_path_for(url)
-    with :ok <- File.write(path, body), do: {:ok, path}
-  end
-
-  @doc "Modifies image file at specified path by resizing to specified limit dimensions."
-  @spec mogrify_resize_to_limit(String.t(), String.t()) :: :ok | any()
-  def mogrify_resize_to_limit(path, resize_dimensions) do
-    with %Mogrify.Image{} <-
-           path
-           |> Mogrify.open()
-           |> Mogrify.resize_to_limit(resize_dimensions)
-           |> Mogrify.save(in_place: true) do
-      :ok
-    end
-  end
-
-  defp rand_uniform(high) do
-    Code.ensure_loaded(:rand)
-
-    if function_exported?(:rand, :uniform, 1) do
-      :rand.uniform(high)
-    else
-      # Erlang/OTP < 19
-      apply(:crypto, :rand_uniform, [1, high])
-    end
-  end
 end
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index fb4b80379..12d4401fa 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -82,17 +82,16 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     {thumbnail_max_width, thumbnail_max_height}
   end
 
-  defp handle_preview("image/" <> _ = content_type, %{params: params} = conn, url) do
+  defp handle_preview("image/" <> _ = _content_type, %{params: params} = conn, url) do
     with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
          media_proxy_url <- MediaProxy.url(url),
          {:ok, thumbnail_binary} <-
            MediaHelper.ffmpeg_resize_remote(
              media_proxy_url,
-             thumbnail_max_width,
-             thumbnail_max_height
+             %{max_width: thumbnail_max_width, max_height: thumbnail_max_height}
            ) do
       conn
-      |> put_resp_header("content-type", content_type)
+      |> put_resp_header("content-type", "image/jpeg")
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
-- 
cgit v1.2.3


From 3a1e810aaaea3e44c4dfc82a014485cf886d6b88 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 21 May 2020 21:47:32 +0300
Subject: [#2497] Customized `exexec` launch to support root operation
 (currently required by Gitlab CI).

---
 lib/pleroma/exec.ex                 | 38 +++++++++++++++++++++++++++++++++++++
 lib/pleroma/helpers/media_helper.ex |  4 +---
 2 files changed, 39 insertions(+), 3 deletions(-)
 create mode 100644 lib/pleroma/exec.ex

(limited to 'lib')

diff --git a/lib/pleroma/exec.ex b/lib/pleroma/exec.ex
new file mode 100644
index 000000000..1b088d322
--- /dev/null
+++ b/lib/pleroma/exec.ex
@@ -0,0 +1,38 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Exec do
+  @moduledoc "Pleroma wrapper around Exexec commands."
+
+  alias Pleroma.Config
+
+  def ensure_started(options_overrides \\ %{}) do
+    options =
+      if Config.get([:exexec, :root_mode]) || System.get_env("USER") == "root" do
+        # Note: running as `root` is discouraged (yet Gitlab CI does that by default)
+        %{root: true, user: "root", limit_users: ["root"]}
+      else
+        %{}
+      end
+
+    options =
+      options
+      |> Map.merge(Config.get([:exexec, :options], %{}))
+      |> Map.merge(options_overrides)
+
+    with {:error, {:already_started, pid}} <- Exexec.start(options) do
+      {:ok, pid}
+    end
+  end
+
+  def run(cmd, options \\ %{}) do
+    ensure_started()
+    Exexec.run(cmd, options)
+  end
+
+  def cmd(cmd, options \\ %{}) do
+    options = Map.merge(%{sync: true, stdout: true}, options)
+    run(cmd, options)
+  end
+end
diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index ee6b76c41..ecd234558 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -7,8 +7,6 @@ defmodule Pleroma.Helpers.MediaHelper do
   Handles common media-related operations.
   """
 
-  @ffmpeg_opts [{:sync, true}, {:stdout, true}]
-
   def ffmpeg_resize_remote(uri, %{max_width: max_width, max_height: max_height}) do
     cmd = ~s"""
     curl -L "#{uri}" |
@@ -20,7 +18,7 @@ defmodule Pleroma.Helpers.MediaHelper do
     cat
     """
 
-    with {:ok, [stdout: stdout_list]} <- Exexec.run(cmd, @ffmpeg_opts) do
+    with {:ok, [stdout: stdout_list]} <- Pleroma.Exec.cmd(cmd) do
       {:ok, Enum.join(stdout_list)}
     end
   end
-- 
cgit v1.2.3


From a51284b60ab450a6c7ff644f1ea10f797a36aa59 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 29 May 2020 09:46:31 +0000
Subject: Merge branch 'fix/mediaproxy-bypass-emoji' into 'develop'

Fix profile emojis bypassing mediaproxy and harden CSP

Closes #1810

See merge request pleroma/pleroma!2596
---
 lib/pleroma/plugs/http_security_plug.ex | 82 +++++++++++++++++++++++++--------
 1 file changed, 62 insertions(+), 20 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex
index 6462797b6..2208d1d6c 100644
--- a/lib/pleroma/plugs/http_security_plug.ex
+++ b/lib/pleroma/plugs/http_security_plug.ex
@@ -31,7 +31,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
       {"x-content-type-options", "nosniff"},
       {"referrer-policy", referrer_policy},
       {"x-download-options", "noopen"},
-      {"content-security-policy", csp_string() <> ";"}
+      {"content-security-policy", csp_string()}
     ]
 
     if report_uri do
@@ -43,23 +43,46 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         ]
       }
 
-      headers ++ [{"reply-to", Jason.encode!(report_group)}]
+      [{"reply-to", Jason.encode!(report_group)} | headers]
     else
       headers
     end
   end
 
+  static_csp_rules = [
+    "default-src 'none'",
+    "base-uri 'self'",
+    "frame-ancestors 'none'",
+    "style-src 'self' 'unsafe-inline'",
+    "font-src 'self'",
+    "manifest-src 'self'"
+  ]
+
+  @csp_start [Enum.join(static_csp_rules, ";") <> ";"]
+
   defp csp_string do
     scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme]
     static_url = Pleroma.Web.Endpoint.static_url()
     websocket_url = Pleroma.Web.Endpoint.websocket_url()
     report_uri = Config.get([:http_security, :report_uri])
 
-    connect_src = "connect-src 'self' #{static_url} #{websocket_url}"
+    img_src = "img-src 'self' data: blob:"
+    media_src = "media-src 'self'"
+
+    {img_src, media_src} =
+      if Config.get([:media_proxy, :enabled]) &&
+           !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do
+        sources = get_proxy_and_attachment_sources()
+        {[img_src, sources], [media_src, sources]}
+      else
+        {img_src, media_src}
+      end
+
+    connect_src = ["connect-src 'self' ", static_url, ?\s, websocket_url]
 
     connect_src =
       if Pleroma.Config.get(:env) == :dev do
-        connect_src <> " http://localhost:3035/"
+        [connect_src, " http://localhost:3035/"]
       else
         connect_src
       end
@@ -71,27 +94,46 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         "script-src 'self'"
       end
 
-    main_part = [
-      "default-src 'none'",
-      "base-uri 'self'",
-      "frame-ancestors 'none'",
-      "img-src 'self' data: blob: https:",
-      "media-src 'self' https:",
-      "style-src 'self' 'unsafe-inline'",
-      "font-src 'self'",
-      "manifest-src 'self'",
-      connect_src,
-      script_src
-    ]
+    report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"]
+    insecure = if scheme == "https", do: "upgrade-insecure-requests"
+
+    @csp_start
+    |> add_csp_param(img_src)
+    |> add_csp_param(media_src)
+    |> add_csp_param(connect_src)
+    |> add_csp_param(script_src)
+    |> add_csp_param(insecure)
+    |> add_csp_param(report)
+    |> :erlang.iolist_to_binary()
+  end
+
+  defp get_proxy_and_attachment_sources do
+    media_proxy_whitelist =
+      Enum.reduce(Config.get([:media_proxy, :whitelist]), [], fn host, acc ->
+        add_source(acc, host)
+      end)
 
-    report = if report_uri, do: ["report-uri #{report_uri}; report-to csp-endpoint"], else: []
+    upload_base_url =
+      if Config.get([Pleroma.Upload, :base_url]),
+        do: URI.parse(Config.get([Pleroma.Upload, :base_url])).host
 
-    insecure = if scheme == "https", do: ["upgrade-insecure-requests"], else: []
+    s3_endpoint =
+      if Config.get([Pleroma.Upload, :uploader]) == Pleroma.Uploaders.S3,
+        do: URI.parse(Config.get([Pleroma.Uploaders.S3, :public_endpoint])).host
 
-    (main_part ++ report ++ insecure)
-    |> Enum.join("; ")
+    []
+    |> add_source(upload_base_url)
+    |> add_source(s3_endpoint)
+    |> add_source(media_proxy_whitelist)
   end
 
+  defp add_source(iodata, nil), do: iodata
+  defp add_source(iodata, source), do: [[?\s, source] | iodata]
+
+  defp add_csp_param(csp_iodata, nil), do: csp_iodata
+
+  defp add_csp_param(csp_iodata, param), do: [[param, ?;] | csp_iodata]
+
   def warn_if_disabled do
     unless Config.get([:http_security, :enabled]) do
       Logger.warn("
-- 
cgit v1.2.3


From 9396b2f8cf0fa26f6fb5e372112b394b74ae8a4e Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Fri, 5 Jun 2020 14:52:09 +0000
Subject: Merge branch 'features/apc2s-pagination' into 'develop'

Fix AP C2S pagination

Closes #866 and #751

See merge request pleroma/pleroma!2491
---
 .../web/activity_pub/activity_pub_controller.ex    | 49 ++++++++---------
 lib/pleroma/web/activity_pub/views/user_view.ex    | 34 +++++-------
 lib/pleroma/web/controller_helper.ex               | 63 ++++++++++++++--------
 .../controllers/timeline_controller.ex             |  4 +-
 lib/pleroma/web/router.ex                          | 15 ++----
 5 files changed, 83 insertions(+), 82 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 2bb5bd15b..a64199cd6 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -18,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   alias Pleroma.Web.ActivityPub.UserView
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.ControllerHelper
   alias Pleroma.Web.Federator
 
   require Logger
@@ -200,31 +201,29 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
     end
   end
 
-  def outbox(conn, %{"nickname" => nickname, "page" => page?} = params)
+  def outbox(
+        %{assigns: %{user: for_user}} = conn,
+        %{"nickname" => nickname, "page" => page?} = params
+      )
       when page? in [true, "true"] do
     with %User{} = user <- User.get_cached_by_nickname(nickname),
          {:ok, user} <- User.ensure_keys_present(user) do
-      activities =
-        if params["max_id"] do
-          ActivityPub.fetch_user_activities(user, nil, %{
-            "max_id" => params["max_id"],
-            # This is a hack because postgres generates inefficient queries when filtering by
-            # 'Answer', poll votes will be hidden by the visibility filter in this case anyway
-            "include_poll_votes" => true,
-            "limit" => 10
-          })
-        else
-          ActivityPub.fetch_user_activities(user, nil, %{
-            "limit" => 10,
-            "include_poll_votes" => true
-          })
-        end
+      # "include_poll_votes" is a hack because postgres generates inefficient
+      # queries when filtering by 'Answer', poll votes will be hidden by the
+      # visibility filter in this case anyway
+      params =
+        params
+        |> Map.drop(["nickname", "page"])
+        |> Map.put("include_poll_votes", true)
+
+      activities = ActivityPub.fetch_user_activities(user, for_user, params)
 
       conn
       |> put_resp_content_type("application/activity+json")
       |> put_view(UserView)
       |> render("activity_collection_page.json", %{
         activities: activities,
+        pagination: ControllerHelper.get_pagination_fields(conn, activities),
         iri: "#{user.ap_id}/outbox"
       })
     end
@@ -318,21 +317,23 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
         %{"nickname" => nickname, "page" => page?} = params
       )
       when page? in [true, "true"] do
+    params =
+      params
+      |> Map.drop(["nickname", "page"])
+      |> Map.put("blocking_user", user)
+      |> Map.put("user", user)
+
     activities =
-      if params["max_id"] do
-        ActivityPub.fetch_activities([user.ap_id | User.following(user)], %{
-          "max_id" => params["max_id"],
-          "limit" => 10
-        })
-      else
-        ActivityPub.fetch_activities([user.ap_id | User.following(user)], %{"limit" => 10})
-      end
+      [user.ap_id | User.following(user)]
+      |> ActivityPub.fetch_activities(params)
+      |> Enum.reverse()
 
     conn
     |> put_resp_content_type("application/activity+json")
     |> put_view(UserView)
     |> render("activity_collection_page.json", %{
       activities: activities,
+      pagination: ControllerHelper.get_pagination_fields(conn, activities),
       iri: "#{user.ap_id}/inbox"
     })
   end
diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex
index bc21ac6c7..3396777d7 100644
--- a/lib/pleroma/web/activity_pub/views/user_view.ex
+++ b/lib/pleroma/web/activity_pub/views/user_view.ex
@@ -216,34 +216,24 @@ defmodule Pleroma.Web.ActivityPub.UserView do
     |> Map.merge(Utils.make_json_ld_header())
   end
 
-  def render("activity_collection_page.json", %{activities: activities, iri: iri}) do
-    # this is sorted chronologically, so first activity is the newest (max)
-    {max_id, min_id, collection} =
-      if length(activities) > 0 do
-        {
-          Enum.at(activities, 0).id,
-          Enum.at(Enum.reverse(activities), 0).id,
-          Enum.map(activities, fn act ->
-            {:ok, data} = Transmogrifier.prepare_outgoing(act.data)
-            data
-          end)
-        }
-      else
-        {
-          0,
-          0,
-          []
-        }
-      end
+  def render("activity_collection_page.json", %{
+        activities: activities,
+        iri: iri,
+        pagination: pagination
+      }) do
+    collection =
+      Enum.map(activities, fn activity ->
+        {:ok, data} = Transmogrifier.prepare_outgoing(activity.data)
+        data
+      end)
 
     %{
-      "id" => "#{iri}?max_id=#{max_id}&page=true",
       "type" => "OrderedCollectionPage",
       "partOf" => iri,
-      "orderedItems" => collection,
-      "next" => "#{iri}?max_id=#{min_id}&page=true"
+      "orderedItems" => collection
     }
     |> Map.merge(Utils.make_json_ld_header())
+    |> Map.merge(pagination)
   end
 
   defp maybe_put_total_items(map, false, _total), do: map
diff --git a/lib/pleroma/web/controller_helper.ex b/lib/pleroma/web/controller_helper.ex
index c9a3a2585..1e0491a96 100644
--- a/lib/pleroma/web/controller_helper.ex
+++ b/lib/pleroma/web/controller_helper.ex
@@ -5,7 +5,9 @@
 defmodule Pleroma.Web.ControllerHelper do
   use Pleroma.Web, :controller
 
-  # As in MastoAPI, per https://api.rubyonrails.org/classes/ActiveModel/Type/Boolean.html
+  alias Pleroma.Pagination
+
+  # As in Mastodon API, per https://api.rubyonrails.org/classes/ActiveModel/Type/Boolean.html
   @falsy_param_values [false, 0, "0", "f", "F", "false", "False", "FALSE", "off", "OFF"]
   def truthy_param?(blank_value) when blank_value in [nil, ""], do: nil
   def truthy_param?(value), do: value not in @falsy_param_values
@@ -34,38 +36,53 @@ defmodule Pleroma.Web.ControllerHelper do
 
   defp param_to_integer(_, default), do: default
 
-  def add_link_headers(conn, activities, extra_params \\ %{}) do
+  def add_link_headers(conn, activities, extra_params \\ %{})
+
+  def add_link_headers(%{assigns: %{skip_link_headers: true}} = conn, _activities, _extra_params),
+    do: conn
+
+  def add_link_headers(conn, activities, extra_params) do
+    case get_pagination_fields(conn, activities, extra_params) do
+      %{"next" => next_url, "prev" => prev_url} ->
+        put_resp_header(conn, "link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"")
+
+      _ ->
+        conn
+    end
+  end
+
+  def get_pagination_fields(conn, activities, extra_params \\ %{}) do
     case List.last(activities) do
       %{id: max_id} ->
         params =
           conn.params
           |> Map.drop(Map.keys(conn.path_params))
-          |> Map.drop(["since_id", "max_id", "min_id"])
           |> Map.merge(extra_params)
-
-        limit =
-          params
-          |> Map.get("limit", "20")
-          |> String.to_integer()
+          |> Map.drop(Pagination.page_keys() -- ["limit", "order"])
 
         min_id =
-          if length(activities) <= limit do
-            activities
-            |> List.first()
-            |> Map.get(:id)
-          else
-            activities
-            |> Enum.at(limit * -1)
-            |> Map.get(:id)
-          end
-
-        next_url = current_url(conn, Map.merge(params, %{max_id: max_id}))
-        prev_url = current_url(conn, Map.merge(params, %{min_id: min_id}))
-
-        put_resp_header(conn, "link", "<#{next_url}>; rel=\"next\", <#{prev_url}>; rel=\"prev\"")
+          activities
+          |> List.first()
+          |> Map.get(:id)
+
+        fields = %{
+          "next" => current_url(conn, Map.put(params, :max_id, max_id)),
+          "prev" => current_url(conn, Map.put(params, :min_id, min_id))
+        }
+
+        #  Generating an `id` without already present pagination keys would
+        # need a query-restriction with an `q.id >= ^id` or `q.id <= ^id`
+        # instead of the `q.id > ^min_id` and `q.id < ^max_id`.
+        #  This is because we only have ids present inside of the page, while
+        # `min_id`, `since_id` and `max_id` requires to know one outside of it.
+        if Map.take(conn.params, Pagination.page_keys() -- ["limit", "order"]) != [] do
+          Map.put(fields, "id", current_url(conn, conn.params))
+        else
+          fields
+        end
 
       _ ->
-        conn
+        %{}
     end
   end
 
diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
index 09e08271b..c3cebd71e 100644
--- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
@@ -40,10 +40,8 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
       |> Map.put("muting_user", user)
       |> Map.put("user", user)
 
-    recipients = [user.ap_id | User.following(user)]
-
     activities =
-      recipients
+      [user.ap_id | User.following(user)]
       |> ActivityPub.fetch_activities(params)
       |> Enum.reverse()
 
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 1da9478db..cb4cc619a 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -545,19 +545,13 @@ defmodule Pleroma.Web.Router do
     get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe)
   end
 
+  # Server to Server (S2S) AP interactions
   pipeline :activitypub do
-    plug(:accepts, ["activity+json", "json"])
-    plug(Pleroma.Web.Plugs.HTTPSignaturePlug)
-    plug(Pleroma.Web.Plugs.MappedSignatureToIdentityPlug)
-  end
-
-  scope "/", Pleroma.Web.ActivityPub do
-    # XXX: not really ostatus
-    pipe_through(:ostatus)
-
-    get("/users/:nickname/outbox", ActivityPubController, :outbox)
+    plug(:ap_service_actor)
+    plug(:http_signature)
   end
 
+  # Client to Server (C2S) AP interactions
   pipeline :activitypub_client do
     plug(:accepts, ["activity+json", "json"])
     plug(:fetch_session)
@@ -578,6 +572,7 @@ defmodule Pleroma.Web.Router do
     get("/api/ap/whoami", ActivityPubController, :whoami)
     get("/users/:nickname/inbox", ActivityPubController, :read_inbox)
 
+    get("/users/:nickname/outbox", ActivityPubController, :outbox)
     post("/users/:nickname/outbox", ActivityPubController, :update_outbox)
     post("/api/ap/upload_media", ActivityPubController, :upload_media)
 
-- 
cgit v1.2.3


From f10b40828fd5f1e4fc7c4ffa9deb543d6a18eb17 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sun, 7 Jun 2020 23:51:03 +0300
Subject: [stable rewrite]: fix streamer streaming out announces from a muted
 thread

Use parent object instead of activity because context is null of Announce activity from Mastodon.
---
 lib/pleroma/web/streamer/worker.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/streamer/worker.ex b/lib/pleroma/web/streamer/worker.ex
index abfed21c8..c669e917d 100644
--- a/lib/pleroma/web/streamer/worker.ex
+++ b/lib/pleroma/web/streamer/worker.ex
@@ -147,7 +147,7 @@ defmodule Pleroma.Web.Streamer.Worker do
          false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host),
          false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host),
          true <- thread_containment(item, user),
-         false <- CommonAPI.thread_muted?(user, item) do
+         false <- CommonAPI.thread_muted?(user, parent) do
       true
     else
       _ -> false
-- 
cgit v1.2.3


From 3687788cf2ab91f6f40f76f9a82c448c477b1fec Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 27 May 2020 13:45:14 +0000
Subject: Merge branch 'notification-fixes' into 'develop'

Notification performance fixes

See merge request pleroma/pleroma!2595
---
 lib/pleroma/notification.ex | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 556075fba..8c6887a6b 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -70,8 +70,9 @@ defmodule Pleroma.Notification do
     |> join(:left, [n, a], object in Object,
       on:
         fragment(
-          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')",
           object.data,
+          a.data,
           a.data
         )
     )
@@ -195,7 +196,7 @@ defmodule Pleroma.Notification do
     |> Repo.all()
   end
 
-  def set_read_up_to(%{id: user_id} = _user, id) do
+  def set_read_up_to(%{id: user_id} = user, id) do
     query =
       from(
         n in Notification,
@@ -215,18 +216,8 @@ defmodule Pleroma.Notification do
 
     {_, notification_ids} = Repo.update_all(query, [])
 
-    Notification
+    for_user_query(user)
     |> where([n], n.id in ^notification_ids)
-    |> join(:inner, [n], activity in assoc(n, :activity))
-    |> join(:left, [n, a], object in Object,
-      on:
-        fragment(
-          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
-          object.data,
-          a.data
-        )
-    )
-    |> preload([n, a, o], activity: {a, object: o})
     |> Repo.all()
   end
 
-- 
cgit v1.2.3


From e9aa6a3fa4e0c417c39fce0a5408294f6d4d1842 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 27 May 2020 09:46:12 +0200
Subject: ActivityPub: Change ordering to `nulls last` in favorites query

This makes it use our existing index and speeds up the query.
---
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index c4f83f9e1..5ce91a8d3 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1318,7 +1318,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> Activity.with_joined_object()
     |> Object.with_joined_activity()
     |> select([_like, object, activity], %{activity | object: object})
-    |> order_by([like, _, _], desc: like.id)
+    |> order_by([like, _, _], desc_nulls_last: like.id)
     |> Pagination.fetch_paginated(
       Map.merge(params, %{"skip_order" => true}),
       pagination,
-- 
cgit v1.2.3


From c29ad60c2014069929bc2906aa3af106a04ef215 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 26 May 2020 17:46:16 +0200
Subject: Activity.Queries: Use correct actor restriction.

---
 lib/pleroma/activity/queries.ex | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex
index 04593b9fb..633eca0b3 100644
--- a/lib/pleroma/activity/queries.ex
+++ b/lib/pleroma/activity/queries.ex
@@ -24,10 +24,7 @@ defmodule Pleroma.Activity.Queries do
 
   @spec by_actor(query, String.t()) :: query
   def by_actor(query \\ Activity, actor) do
-    from(
-      activity in query,
-      where: fragment("(?)->>'actor' = ?", activity.data, ^actor)
-    )
+    from(a in query, where: a.actor == ^actor)
   end
 
   @spec by_author(query, User.t()) :: query
-- 
cgit v1.2.3


From ada9d15eeecd820947ca6bcc4b3f65c8f874c99c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 19 May 2020 14:32:27 +0000
Subject: Merge branch 'recipients-query-speedup' into 'develop'

Greatly speed up recipients query

See merge request pleroma/pleroma!2558
---
 lib/pleroma/user.ex       |  8 ++++++--
 lib/pleroma/user/query.ex | 24 +++++++++++-------------
 2 files changed, 17 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 3e8f19e30..fbfdc68e8 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1199,8 +1199,12 @@ defmodule Pleroma.User do
   end
 
   @spec get_recipients_from_activity(Activity.t()) :: [User.t()]
-  def get_recipients_from_activity(%Activity{recipients: to}) do
-    User.Query.build(%{recipients_from_activity: to, local: true, deactivated: false})
+  def get_recipients_from_activity(%Activity{recipients: to, actor: actor}) do
+    to = [actor | to]
+
+    query = User.Query.build(%{recipients_from_activity: to, local: true, deactivated: false})
+
+    query
     |> Repo.all()
   end
 
diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex
index ec88088cf..a387f9b77 100644
--- a/lib/pleroma/user/query.ex
+++ b/lib/pleroma/user/query.ex
@@ -162,20 +162,18 @@ defmodule Pleroma.User.Query do
   end
 
   defp compose_query({:recipients_from_activity, to}, query) do
-    query
-    |> join(:left, [u], r in FollowingRelationship,
-      as: :relationships,
-      on: r.follower_id == u.id
-    )
-    |> join(:left, [relationships: r], f in User,
-      as: :following,
-      on: f.id == r.following_id
-    )
-    |> where(
-      [u, following: f, relationships: r],
-      u.ap_id in ^to or (f.follower_address in ^to and r.state == ^:follow_accept)
+    following_query =
+      from(u in User,
+        join: f in FollowingRelationship,
+        on: u.id == f.following_id,
+        where: f.state == ^:follow_accept,
+        where: u.follower_address in ^to,
+        select: f.follower_id
+      )
+
+    from(u in query,
+      where: u.ap_id in ^to or u.id in subquery(following_query)
     )
-    |> distinct(true)
   end
 
   defp compose_query({:order_by, key}, query) do
-- 
cgit v1.2.3


From 8b8b8599e9359ea5c1212144c50ab406025016c5 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 9 Jun 2020 21:49:24 +0400
Subject: Fix atom leak in Rich Media Parser

---
 lib/pleroma/web/mastodon_api/views/status_view.ex      | 14 ++++++--------
 lib/pleroma/web/rich_media/helpers.ex                  |  6 +++---
 lib/pleroma/web/rich_media/parser.ex                   | 12 ++++--------
 lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex |  8 ++++----
 lib/pleroma/web/rich_media/parsers/oembed_parser.ex    | 18 ++++++------------
 5 files changed, 23 insertions(+), 35 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index a042075f5..f0bdb49c3 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -307,8 +307,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     page_url_data = URI.parse(page_url)
 
     page_url_data =
-      if rich_media[:url] != nil do
-        URI.merge(page_url_data, URI.parse(rich_media[:url]))
+      if is_binary(rich_media["url"]) do
+        URI.merge(page_url_data, URI.parse(rich_media["url"]))
       else
         page_url_data
       end
@@ -316,11 +316,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     page_url = page_url_data |> to_string
 
     image_url =
-      if rich_media[:image] != nil do
-        URI.merge(page_url_data, URI.parse(rich_media[:image]))
+      if is_binary(rich_media["image"]) do
+        URI.merge(page_url_data, URI.parse(rich_media["image"]))
         |> to_string
-      else
-        nil
       end
 
     %{
@@ -329,8 +327,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       provider_url: page_url_data.scheme <> "://" <> page_url_data.host,
       url: page_url,
       image: image_url |> MediaProxy.url(),
-      title: rich_media[:title] || "",
-      description: rich_media[:description] || "",
+      title: rich_media["title"] || "",
+      description: rich_media["description"] || "",
       pleroma: %{
         opengraph: rich_media
       }
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 9d3d7f978..1729141e9 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -9,7 +9,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Object
   alias Pleroma.Web.RichMedia.Parser
 
-  @spec validate_page_url(any()) :: :ok | :error
+  @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
     validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
 
@@ -18,8 +18,8 @@ defmodule Pleroma.Web.RichMedia.Helpers do
     |> parse_uri(page_url)
   end
 
-  defp validate_page_url(%URI{host: host, scheme: scheme, authority: authority})
-       when scheme == "https" and not is_nil(authority) do
+  defp validate_page_url(%URI{host: host, scheme: "https", authority: authority})
+       when is_binary(authority) do
     cond do
       host in Config.get([:rich_media, :ignore_hosts], []) ->
         :error
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 0779065ee..7b45ecb9c 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -83,7 +83,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
       html
       |> parse_html()
       |> maybe_parse()
-      |> Map.put(:url, url)
+      |> Map.put("url", url)
       |> clean_parsed_data()
       |> check_parsed_data()
     rescue
@@ -103,8 +103,8 @@ defmodule Pleroma.Web.RichMedia.Parser do
     end)
   end
 
-  defp check_parsed_data(%{title: title} = data)
-       when is_binary(title) and byte_size(title) > 0 do
+  defp check_parsed_data(%{"title" => title} = data)
+       when is_binary(title) and title != "" do
     {:ok, data}
   end
 
@@ -115,11 +115,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
   defp clean_parsed_data(data) do
     data
     |> Enum.reject(fn {key, val} ->
-      with {:ok, _} <- Jason.encode(%{key => val}) do
-        false
-      else
-        _ -> true
-      end
+      not match?({:ok, _}, Jason.encode(%{key => val}))
     end)
     |> Map.new()
   end
diff --git a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
index ae0f36702..2762b5902 100644
--- a/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/meta_tags_parser.ex
@@ -29,19 +29,19 @@ defmodule Pleroma.Web.RichMedia.Parsers.MetaTagsParser do
     {_tag, attributes, _children} = html_node
 
     data =
-      Enum.into(attributes, %{}, fn {name, value} ->
+      Map.new(attributes, fn {name, value} ->
         {name, String.trim_leading(value, "#{prefix}:")}
       end)
 
-    %{String.to_atom(data[key_name]) => data[value_name]}
+    %{data[key_name] => data[value_name]}
   end
 
-  defp maybe_put_title(%{title: _} = meta, _), do: meta
+  defp maybe_put_title(%{"title" => _} = meta, _), do: meta
 
   defp maybe_put_title(meta, html) when meta != %{} do
     case get_page_title(html) do
       "" -> meta
-      title -> Map.put_new(meta, :title, title)
+      title -> Map.put_new(meta, "title", title)
     end
   end
 
diff --git a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
index 8f32bf91b..db8ccf15d 100644
--- a/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
+++ b/lib/pleroma/web/rich_media/parsers/oembed_parser.ex
@@ -5,7 +5,7 @@
 defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
   def parse(html, _data) do
     with elements = [_ | _] <- get_discovery_data(html),
-         {:ok, oembed_url} <- get_oembed_url(elements),
+         oembed_url when is_binary(oembed_url) <- get_oembed_url(elements),
          {:ok, oembed_data} <- get_oembed_data(oembed_url) do
       {:ok, oembed_data}
     else
@@ -17,19 +17,13 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do
     html |> Floki.find("link[type='application/json+oembed']")
   end
 
-  defp get_oembed_url(nodes) do
-    {"link", attributes, _children} = nodes |> hd()
-
-    {:ok, Enum.into(attributes, %{})["href"]}
+  defp get_oembed_url([{"link", attributes, _children} | _]) do
+    Enum.find_value(attributes, fn {k, v} -> if k == "href", do: v end)
   end
 
   defp get_oembed_data(url) do
-    {:ok, %Tesla.Env{body: json}} = Pleroma.HTTP.get(url, [], adapter: [pool: :media])
-
-    {:ok, data} = Jason.decode(json)
-
-    data = data |> Map.new(fn {k, v} -> {String.to_atom(k), v} end)
-
-    {:ok, data}
+    with {:ok, %Tesla.Env{body: json}} <- Pleroma.HTTP.get(url, [], adapter: [pool: :media]) do
+      Jason.decode(json)
+    end
   end
 end
-- 
cgit v1.2.3


From 8efd1614ecfe694a7c6c58a6c59e3acf88e99c91 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Wed, 10 Jun 2020 17:34:23 +0200
Subject: Remove use of atoms in MRF.UserAllowListPolicy

---
 lib/pleroma/config/deprecation_warnings.ex         | 25 +++++++++++++++++++++-
 .../web/activity_pub/mrf/user_allow_list_policy.ex |  2 +-
 2 files changed, 25 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex
index c39a8984b..b68ded01f 100644
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@@ -4,9 +4,10 @@
 
 defmodule Pleroma.Config.DeprecationWarnings do
   require Logger
+  alias Pleroma.Config
 
   def check_hellthread_threshold do
-    if Pleroma.Config.get([:mrf_hellthread, :threshold]) do
+    if Config.get([:mrf_hellthread, :threshold]) do
       Logger.warn("""
       !!!DEPRECATION WARNING!!!
       You are using the old configuration mechanism for the hellthread filter. Please check config.md.
@@ -14,7 +15,29 @@ defmodule Pleroma.Config.DeprecationWarnings do
     end
   end
 
+  def mrf_user_allowlist do
+    config = Config.get(:mrf_user_allowlist)
+
+    if config && Enum.any?(config, fn {k, _} -> is_atom(k) end) do
+      rewritten =
+        Enum.reduce(Config.get(:mrf_user_allowlist), Map.new(), fn {k, v}, acc ->
+          Map.put(acc, to_string(k), v)
+        end)
+
+      Config.put(:mrf_user_allowlist, rewritten)
+
+      Logger.error("""
+      !!!DEPRECATION WARNING!!!
+      As of Pleroma 2.0.7, the `mrf_user_allowlist` setting changed of format.
+      Pleroma 2.1 will remove support for the old format. Please change your configuration to match this:
+
+      config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
+      """)
+    end
+  end
+
   def warn do
     check_hellthread_threshold()
+    mrf_user_allowlist()
   end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
index a927a4ed8..651aed70f 100644
--- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
 
     allow_list =
       Config.get(
-        [:mrf_user_allowlist, String.to_atom(actor_info.host)],
+        [:mrf_user_allowlist, actor_info.host],
         []
       )
 
-- 
cgit v1.2.3


From e313aa0977fc6023067d6580b36a7dc71a4d1b5a Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 1 May 2020 21:15:43 +0200
Subject: static-fe.css: Restore and move to /priv/static/static-fe

---
 lib/pleroma/constants.ex                            | 5 +++++
 lib/pleroma/plugs/instance_static.ex                | 7 +++----
 lib/pleroma/web/endpoint.ex                         | 5 +++--
 lib/pleroma/web/templates/layout/static_fe.html.eex | 2 +-
 4 files changed, 12 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex
index 4ba39b53f..3a9eec5ea 100644
--- a/lib/pleroma/constants.ex
+++ b/lib/pleroma/constants.ex
@@ -20,4 +20,9 @@ defmodule Pleroma.Constants do
       "deleted_activity_id"
     ]
   )
+
+  const(static_only_files,
+    do:
+      ~w(index.html robots.txt static static-fe finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc)
+  )
 end
diff --git a/lib/pleroma/plugs/instance_static.ex b/lib/pleroma/plugs/instance_static.ex
index 927fa2663..7516f75c3 100644
--- a/lib/pleroma/plugs/instance_static.ex
+++ b/lib/pleroma/plugs/instance_static.ex
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Plugs.InstanceStatic do
+  require Pleroma.Constants
+
   @moduledoc """
   This is a shim to call `Plug.Static` but with runtime `from` configuration.
 
@@ -21,9 +23,6 @@ defmodule Pleroma.Plugs.InstanceStatic do
     end
   end
 
-  @only ~w(index.html robots.txt static emoji packs sounds images instance favicon.png sw.js
-  sw-pleroma.js)
-
   def init(opts) do
     opts
     |> Keyword.put(:from, "__unconfigured_instance_static_plug")
@@ -31,7 +30,7 @@ defmodule Pleroma.Plugs.InstanceStatic do
     |> Plug.Static.init()
   end
 
-  for only <- @only do
+  for only <- Pleroma.Constants.static_only_files() do
     at = Plug.Router.Utils.split("/")
 
     def call(%{request_path: "/" <> unquote(only) <> _} = conn, opts) do
diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 72cb3ee27..226d42c2c 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -5,6 +5,8 @@
 defmodule Pleroma.Web.Endpoint do
   use Phoenix.Endpoint, otp_app: :pleroma
 
+  require Pleroma.Constants
+
   socket("/socket", Pleroma.Web.UserSocket)
 
   plug(Pleroma.Plugs.SetLocalePlug)
@@ -34,8 +36,7 @@ defmodule Pleroma.Web.Endpoint do
     Plug.Static,
     at: "/",
     from: :pleroma,
-    only:
-      ~w(index.html robots.txt static finmoji emoji packs sounds images instance sw.js sw-pleroma.js favicon.png schemas doc),
+    only: Pleroma.Constants.static_only_files(),
     # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
     gzip: true,
     cache_control_for_etags: @static_cache_control,
diff --git a/lib/pleroma/web/templates/layout/static_fe.html.eex b/lib/pleroma/web/templates/layout/static_fe.html.eex
index 819632cec..dc0ee2a5c 100644
--- a/lib/pleroma/web/templates/layout/static_fe.html.eex
+++ b/lib/pleroma/web/templates/layout/static_fe.html.eex
@@ -5,7 +5,7 @@
     <meta name="viewport" content="width=device-width,initial-scale=1,minimal-ui" />
     <title><%= Pleroma.Config.get([:instance, :name]) %></title>
     <%= Phoenix.HTML.raw(assigns[:meta] || "") %>
-    <link rel="stylesheet" href="/static/static-fe.css">
+    <link rel="stylesheet" href="/static-fe/static-fe.css">
   </head>
   <body>
     <div class="container">
-- 
cgit v1.2.3


From cd2df734dde6151faa6a73edb296a5cf768e9a34 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 29 May 2020 21:23:49 +0000
Subject: Merge branch 'bugfix/csp-unproxied' into 'develop'

http_security_plug.ex: Fix non-proxied media

See merge request pleroma/pleroma!2610
---
 lib/pleroma/plugs/http_security_plug.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex
index 2208d1d6c..589072535 100644
--- a/lib/pleroma/plugs/http_security_plug.ex
+++ b/lib/pleroma/plugs/http_security_plug.ex
@@ -75,7 +75,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         sources = get_proxy_and_attachment_sources()
         {[img_src, sources], [media_src, sources]}
       else
-        {img_src, media_src}
+        {[img_src, " https:"], [media_src, " https:"]}
       end
 
     connect_src = ["connect-src 'self' ", static_url, ?\s, websocket_url]
-- 
cgit v1.2.3


From 90676bdfe3df526fe9596c25ad63d59fc602eb7a Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Thu, 11 Jun 2020 16:18:06 +0000
Subject: Merge branch 'fix/csp-mediaproxy-base-url' into 'develop'

HTTP security plug: add media proxy base url host to csp

See merge request pleroma/pleroma!2638
---
 lib/pleroma/plugs/http_security_plug.ex | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex
index 589072535..cad0ad4a0 100644
--- a/lib/pleroma/plugs/http_security_plug.ex
+++ b/lib/pleroma/plugs/http_security_plug.ex
@@ -113,6 +113,10 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         add_source(acc, host)
       end)
 
+    media_proxy_base_url =
+      if Config.get([:media_proxy, :base_url]),
+        do: URI.parse(Config.get([:media_proxy, :base_url])).host
+
     upload_base_url =
       if Config.get([Pleroma.Upload, :base_url]),
         do: URI.parse(Config.get([Pleroma.Upload, :base_url])).host
@@ -122,6 +126,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do
         do: URI.parse(Config.get([Pleroma.Uploaders.S3, :public_endpoint])).host
 
     []
+    |> add_source(media_proxy_base_url)
     |> add_source(upload_base_url)
     |> add_source(s3_endpoint)
     |> add_source(media_proxy_whitelist)
-- 
cgit v1.2.3


From 9faa63203717e71d666afb6755ff0b781b491823 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sun, 5 Jul 2020 19:02:43 +0300
Subject: [#2497] Fixed merge issue.

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 0f4575e2f..583c177f2 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -12,8 +12,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   def remote(conn, %{"sig" => sig64, "url" => url64}) do
     with {_, true} <- {:enabled, MediaProxy.enabled?()},
-         {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
+         {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
          :ok <- MediaProxy.verify_request_path_and_url(conn, url) do
       proxy_opts = Config.get([:media_proxy, :proxy_opts], [])
       ReverseProxy.call(conn, url, proxy_opts)
-- 
cgit v1.2.3


From 6a25f72a75f90b29f0a82dd8fcb1bdca25996de7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Jul 2020 13:02:48 +0200
Subject: FrontendStatic: Work correctly for other frontend types.

---
 lib/pleroma/plugs/frontend_static.ex | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/frontend_static.ex b/lib/pleroma/plugs/frontend_static.ex
index f549ca75f..11a0d5382 100644
--- a/lib/pleroma/plugs/frontend_static.ex
+++ b/lib/pleroma/plugs/frontend_static.ex
@@ -30,6 +30,7 @@ defmodule Pleroma.Plugs.FrontendStatic do
     opts
     |> Keyword.put(:from, "__unconfigured_frontend_static_plug")
     |> Plug.Static.init()
+    |> Map.put(:frontend_type, opts[:frontend_type])
   end
 
   def call(conn, opts) do
-- 
cgit v1.2.3


From 66974e17a06bc26d7ea0be26bdd77f82b80afdaa Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 29 Jul 2020 13:03:04 +0200
Subject: Endpoint: Serve a dynamically configured admin interface

---
 lib/pleroma/web/endpoint.ex | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex
index 527fb288d..8b153763d 100644
--- a/lib/pleroma/web/endpoint.ex
+++ b/lib/pleroma/web/endpoint.ex
@@ -39,6 +39,18 @@ defmodule Pleroma.Web.Endpoint do
     }
   )
 
+  plug(Plug.Static.IndexHtml, at: "/pleroma/admin/")
+
+  plug(Pleroma.Plugs.FrontendStatic,
+    at: "/pleroma/admin",
+    frontend_type: :admin,
+    gzip: true,
+    cache_control_for_etags: @static_cache_control,
+    headers: %{
+      "cache-control" => @static_cache_control
+    }
+  )
+
   # Serve at "/" the static files from "priv/static" directory.
   #
   # You should set gzip to true if you are running phoenix.digest
@@ -56,8 +68,6 @@ defmodule Pleroma.Web.Endpoint do
     }
   )
 
-  plug(Plug.Static.IndexHtml, at: "/pleroma/admin/")
-
   plug(Plug.Static,
     at: "/pleroma/admin/",
     from: {:pleroma, "priv/static/adminfe/"}
-- 
cgit v1.2.3


From 2e20ceee523084a11c07c5a3a99fa2de3be15e7a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 30 Jul 2020 14:12:41 +0200
Subject: Mix tasks: Add frontend task to download and install frontends.

Co-authored-by: Roman Chvanikov <chvanikoff@pm.me>
---
 lib/mix/tasks/pleroma/frontend.ex | 121 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 121 insertions(+)
 create mode 100644 lib/mix/tasks/pleroma/frontend.ex

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
new file mode 100644
index 000000000..bd65e9e36
--- /dev/null
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -0,0 +1,121 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Frontend do
+  use Mix.Task
+
+  import Mix.Pleroma
+
+  @shortdoc "Manages bundled Pleroma frontends"
+
+  # @moduledoc File.read!("docs/administration/CLI_tasks/frontend.md")
+
+  def run(["install", "none" | _args]) do
+    shell_info("Skipping frontend installation because none was requested")
+    "none"
+  end
+
+  def run(["install", frontend | args]) do
+    log_level = Logger.level()
+    Logger.configure(level: :warn)
+    start_pleroma()
+
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          ref: :string,
+          static_dir: :string,
+          build_url: :string
+        ]
+      )
+
+    instance_static_dir =
+      with nil <- options[:static_dir] do
+        Pleroma.Config.get!([:instance, :static_dir])
+      end
+
+    cmd_frontend_info = %{
+      "name" => frontend,
+      "ref" => options[:ref],
+      "build_url" => options[:build_url]
+    }
+
+    config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{})
+
+    frontend_info =
+      Map.merge(config_frontend_info, cmd_frontend_info, fn _key, config, cmd ->
+        # This only overrides things that are actually set
+        cmd || config
+      end)
+
+    ref = frontend_info["ref"]
+
+    unless ref do
+      raise "No ref given or configured"
+    end
+
+    dest =
+      Path.join([
+        instance_static_dir,
+        "frontends",
+        frontend,
+        ref
+      ])
+
+    fe_label = "#{frontend} (#{ref})"
+
+    shell_info("Downloading pre-built bundle for #{fe_label}")
+    tmp_dir = Path.join(dest, "tmp")
+
+    with {_, :ok} <- {:download, download_build(frontend_info, tmp_dir)},
+         shell_info("Installing #{fe_label} to #{dest}"),
+         :ok <- install_frontend(frontend_info, tmp_dir, dest) do
+      File.rm_rf!(tmp_dir)
+      shell_info("Frontend #{fe_label} installed to #{dest}")
+
+      Logger.configure(level: log_level)
+    else
+      {:download, _} ->
+        shell_info("Could not download the frontend")
+
+      _e ->
+        shell_info("Could not install the frontend")
+    end
+  end
+
+  defp download_build(frontend_info, dest) do
+    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+
+    with {:ok, %{status: 200, body: zip_body}} <-
+           Pleroma.HTTP.get(url, [], timeout: 120_000, recv_timeout: 120_000),
+         {:ok, unzipped} <- :zip.unzip(zip_body, [:memory]) do
+      File.rm_rf!(dest)
+      File.mkdir_p!(dest)
+
+      Enum.each(unzipped, fn {filename, data} ->
+        path = filename
+
+        new_file_path = Path.join(dest, path)
+
+        new_file_path
+        |> Path.dirname()
+        |> File.mkdir_p!()
+
+        File.write!(new_file_path, data)
+      end)
+
+      :ok
+    else
+      e -> {:error, e}
+    end
+  end
+
+  defp install_frontend(frontend_info, source, dest) do
+    from = frontend_info["build_dir"] || "dist"
+    File.mkdir_p!(dest)
+    File.cp_r!(Path.join([source, from]), dest)
+    :ok
+  end
+end
-- 
cgit v1.2.3


From 7e01339dddf78d99f609fdac934e89724f8254c3 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 31 Jul 2020 17:58:50 +0200
Subject: Frontend mix task: Support installation from local file.

---
 lib/mix/tasks/pleroma/frontend.ex | 41 ++++++++++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
index bd65e9e36..c385c355a 100644
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -27,7 +27,9 @@ defmodule Mix.Tasks.Pleroma.Frontend do
         strict: [
           ref: :string,
           static_dir: :string,
-          build_url: :string
+          build_url: :string,
+          build_dir: :string,
+          file: :string
         ]
       )
 
@@ -39,7 +41,8 @@ defmodule Mix.Tasks.Pleroma.Frontend do
     cmd_frontend_info = %{
       "name" => frontend,
       "ref" => options[:ref],
-      "build_url" => options[:build_url]
+      "build_url" => options[:build_url],
+      "build_dir" => options[:build_dir]
     }
 
     config_frontend_info = Pleroma.Config.get([:frontends, :available, frontend], %{})
@@ -66,10 +69,10 @@ defmodule Mix.Tasks.Pleroma.Frontend do
 
     fe_label = "#{frontend} (#{ref})"
 
-    shell_info("Downloading pre-built bundle for #{fe_label}")
     tmp_dir = Path.join(dest, "tmp")
 
-    with {_, :ok} <- {:download, download_build(frontend_info, tmp_dir)},
+    with {_, :ok} <-
+           {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])},
          shell_info("Installing #{fe_label} to #{dest}"),
          :ok <- install_frontend(frontend_info, tmp_dir, dest) do
       File.rm_rf!(tmp_dir)
@@ -77,20 +80,26 @@ defmodule Mix.Tasks.Pleroma.Frontend do
 
       Logger.configure(level: log_level)
     else
-      {:download, _} ->
-        shell_info("Could not download the frontend")
+      {:download_or_unzip, _} ->
+        shell_info("Could not download or unzip the frontend")
 
       _e ->
         shell_info("Could not install the frontend")
     end
   end
 
-  defp download_build(frontend_info, dest) do
-    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+  defp download_or_unzip(frontend_info, temp_dir, file) do
+    if file do
+      with {:ok, zip} <- File.read(Path.expand(file)) do
+        unzip(zip, temp_dir)
+      end
+    else
+      download_build(frontend_info, temp_dir)
+    end
+  end
 
-    with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [], timeout: 120_000, recv_timeout: 120_000),
-         {:ok, unzipped} <- :zip.unzip(zip_body, [:memory]) do
+  def unzip(zip, dest) do
+    with {:ok, unzipped} <- :zip.unzip(zip, [:memory]) do
       File.rm_rf!(dest)
       File.mkdir_p!(dest)
 
@@ -107,6 +116,16 @@ defmodule Mix.Tasks.Pleroma.Frontend do
       end)
 
       :ok
+    end
+  end
+
+  defp download_build(frontend_info, dest) do
+    shell_info("Downloading pre-built bundle for #{frontend_info["name"]}")
+    url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
+
+    with {:ok, %{status: 200, body: zip_body}} <-
+           Pleroma.HTTP.get(url, [], timeout: 120_000, recv_timeout: 120_000) do
+      unzip(zip_body, dest)
     else
       e -> {:error, e}
     end
-- 
cgit v1.2.3


From e5ab5fbe764dc8a1326fb31fb9754e5986ee53ee Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 7 Aug 2020 15:01:08 +0200
Subject: Mix task frontend: Read the docs.

---
 lib/mix/tasks/pleroma/frontend.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
index c385c355a..2adbf8d72 100644
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
 
   @shortdoc "Manages bundled Pleroma frontends"
 
-  # @moduledoc File.read!("docs/administration/CLI_tasks/frontend.md")
+  @moduledoc File.read!("docs/administration/CLI_tasks/frontend.md")
 
   def run(["install", "none" | _args]) do
     shell_info("Skipping frontend installation because none was requested")
-- 
cgit v1.2.3


From 0865f36965f1583085af3a424dbbc89de724fd33 Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Sat, 15 Aug 2020 15:27:41 +0200
Subject: Mark notifications about statuses from muted users as read
 automatically

---
 lib/pleroma/notification.ex | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 0b171563b..b4719896e 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -441,6 +441,7 @@ defmodule Pleroma.Notification do
         |> Multi.insert(:notification, %Notification{
           user_id: user.id,
           activity: activity,
+          seen: mark_as_read?(activity, user),
           type: type_from_activity(activity)
         })
         |> Marker.multi_set_last_read_id(user, "notifications")
@@ -634,6 +635,11 @@ defmodule Pleroma.Notification do
 
   def skip?(_, _, _), do: false
 
+  def mark_as_read?(activity, target_user) do
+    user = Activity.user_actor(activity)
+    User.mutes_user?(target_user, user)
+  end
+
   def for_user_and_activity(user, activity) do
     from(n in __MODULE__,
       where: n.user_id == ^user.id,
-- 
cgit v1.2.3


From 25c69e271a3ea6687805e0bd0d4b902cda06e364 Mon Sep 17 00:00:00 2001
From: Sergey Suprunenko <suprunenko.s@gmail.com>
Date: Mon, 17 Aug 2020 00:07:23 +0200
Subject: Make notifications about new statuses from muted threads read

---
 lib/pleroma/notification.ex              | 3 ++-
 lib/pleroma/web/common_api/common_api.ex | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index b4719896e..c1825f810 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -15,6 +15,7 @@ defmodule Pleroma.Notification do
   alias Pleroma.Repo
   alias Pleroma.ThreadMute
   alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.Push
   alias Pleroma.Web.Streamer
@@ -637,7 +638,7 @@ defmodule Pleroma.Notification do
 
   def mark_as_read?(activity, target_user) do
     user = Activity.user_actor(activity)
-    User.mutes_user?(target_user, user)
+    User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(target_user, activity)
   end
 
   def for_user_and_activity(user, activity) do
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index a8141b28f..5ad2b91c2 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -465,7 +465,7 @@ defmodule Pleroma.Web.CommonAPI do
   end
 
   def thread_muted?(%User{id: user_id}, %{data: %{"context" => context}})
-      when is_binary("context") do
+      when is_binary(context) do
     ThreadMute.exists?(user_id, context)
   end
 
-- 
cgit v1.2.3


From da116d81fb0028913c2a0f30ac35532fb500e8fc Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 18 Aug 2020 18:23:27 +0300
Subject: [#2497] Added video preview proxy. Switched from exexec to Port.

---
 lib/pleroma/exec.ex                                | 38 ----------------
 lib/pleroma/helpers/media_helper.ex                | 19 +++++---
 .../web/media_proxy/media_proxy_controller.ex      | 50 +++++++++++++---------
 3 files changed, 41 insertions(+), 66 deletions(-)
 delete mode 100644 lib/pleroma/exec.ex

(limited to 'lib')

diff --git a/lib/pleroma/exec.ex b/lib/pleroma/exec.ex
deleted file mode 100644
index 1b088d322..000000000
--- a/lib/pleroma/exec.ex
+++ /dev/null
@@ -1,38 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Exec do
-  @moduledoc "Pleroma wrapper around Exexec commands."
-
-  alias Pleroma.Config
-
-  def ensure_started(options_overrides \\ %{}) do
-    options =
-      if Config.get([:exexec, :root_mode]) || System.get_env("USER") == "root" do
-        # Note: running as `root` is discouraged (yet Gitlab CI does that by default)
-        %{root: true, user: "root", limit_users: ["root"]}
-      else
-        %{}
-      end
-
-    options =
-      options
-      |> Map.merge(Config.get([:exexec, :options], %{}))
-      |> Map.merge(options_overrides)
-
-    with {:error, {:already_started, pid}} <- Exexec.start(options) do
-      {:ok, pid}
-    end
-  end
-
-  def run(cmd, options \\ %{}) do
-    ensure_started()
-    Exexec.run(cmd, options)
-  end
-
-  def cmd(cmd, options \\ %{}) do
-    options = Map.merge(%{sync: true, stdout: true}, options)
-    run(cmd, options)
-  end
-end
diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index ecd234558..ca46698cc 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -7,19 +7,24 @@ defmodule Pleroma.Helpers.MediaHelper do
   Handles common media-related operations.
   """
 
-  def ffmpeg_resize_remote(uri, %{max_width: max_width, max_height: max_height}) do
+  def ffmpeg_resize(uri_or_path, %{max_width: max_width, max_height: max_height}) do
     cmd = ~s"""
-    curl -L "#{uri}" |
-    ffmpeg -i pipe:0 -f lavfi -i color=c=white \
+    ffmpeg -i #{uri_or_path} -f lavfi -i color=c=white \
       -filter_complex "[0:v] scale='min(#{max_width},iw)':'min(#{max_height},ih)': \
         force_original_aspect_ratio=decrease [scaled]; \
         [1][scaled] scale2ref [bg][img]; [bg] setsar=1 [bg]; [bg][img] overlay=shortest=1" \
-      -f image2 -vcodec mjpeg -frames:v 1 pipe:1 | \
-    cat
+      -loglevel quiet -f image2 -vcodec mjpeg -frames:v 1 pipe:1
     """
 
-    with {:ok, [stdout: stdout_list]} <- Pleroma.Exec.cmd(cmd) do
-      {:ok, Enum.join(stdout_list)}
+    pid = Port.open({:spawn, cmd}, [:use_stdio, :in, :stream, :exit_status, :binary])
+
+    receive do
+      {^pid, {:data, data}} ->
+        send(pid, {self(), :close})
+        {:ok, data}
+
+      {^pid, {:exit_status, status}} when status > 0 ->
+        {:error, status}
     end
   end
 end
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 583c177f2..8861398dd 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -66,31 +66,23 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp thumbnail_max_dimensions(params) do
-    config = Config.get([:media_preview_proxy], [])
-
-    thumbnail_max_width =
-      if w = params["thumbnail_max_width"] do
-        String.to_integer(w)
-      else
-        Keyword.fetch!(config, :thumbnail_max_width)
-      end
+  defp handle_preview("image/" <> _ = _content_type, conn, url) do
+    handle_image_or_video_preview(conn, url)
+  end
 
-    thumbnail_max_height =
-      if h = params["thumbnail_max_height"] do
-        String.to_integer(h)
-      else
-        Keyword.fetch!(config, :thumbnail_max_height)
-      end
+  defp handle_preview("video/" <> _ = _content_type, conn, url) do
+    handle_image_or_video_preview(conn, url)
+  end
 
-    {thumbnail_max_width, thumbnail_max_height}
+  defp handle_preview(content_type, conn, _url) do
+    send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
   end
 
-  defp handle_preview("image/" <> _ = _content_type, %{params: params} = conn, url) do
+  defp handle_image_or_video_preview(%{params: params} = conn, url) do
     with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
          media_proxy_url <- MediaProxy.url(url),
          {:ok, thumbnail_binary} <-
-           MediaHelper.ffmpeg_resize_remote(
+           MediaHelper.ffmpeg_resize(
              media_proxy_url,
              %{max_width: thumbnail_max_width, max_height: thumbnail_max_height}
            ) do
@@ -99,12 +91,28 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
-        send_resp(conn, :failed_dependency, "Can't handle image preview.")
+        send_resp(conn, :failed_dependency, "Can't handle preview.")
     end
   end
 
-  defp handle_preview(content_type, conn, _url) do
-    send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
+  defp thumbnail_max_dimensions(params) do
+    config = Config.get([:media_preview_proxy], [])
+
+    thumbnail_max_width =
+      if w = params["thumbnail_max_width"] do
+        String.to_integer(w)
+      else
+        Keyword.fetch!(config, :thumbnail_max_width)
+      end
+
+    thumbnail_max_height =
+      if h = params["thumbnail_max_height"] do
+        String.to_integer(h)
+      else
+        Keyword.fetch!(config, :thumbnail_max_height)
+      end
+
+    {thumbnail_max_width, thumbnail_max_height}
   end
 
   defp preview_head_request_timeout do
-- 
cgit v1.2.3


From 5316e231b0b007ce05bc1bffdf6ce0244749fb9e Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Wed, 19 Aug 2020 00:05:48 +0200
Subject: Pipeline Ingestion: Audio (Part 2)

---
 lib/pleroma/web/activity_pub/activity_pub.ex       |  2 +-
 .../object_validators/attachment_validator.ex      | 42 +++++++++++-----------
 .../object_validators/audio_validator.ex           | 18 ++++++++--
 .../object_validators/create_generic_validator.ex  | 11 ++++++
 .../object_validators/note_validator.ex            |  2 +-
 .../object_validators/question_validator.ex        |  2 +-
 lib/pleroma/web/activity_pub/transmogrifier.ex     |  5 ++-
 7 files changed, 53 insertions(+), 29 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index bde1fe708..db1867494 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -85,7 +85,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ["ChatMessage", "Question", "Answer"]
+  @object_types ~w[ChatMessage Question Answer Audio]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
index f53bb02be..c8b148280 100644
--- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
@@ -41,34 +41,34 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   end
 
   def fix_media_type(data) do
-    data =
-      data
-      |> Map.put_new("mediaType", data["mimeType"])
+    data = Map.put_new(data, "mediaType", data["mimeType"])
 
     if MIME.valid?(data["mediaType"]) do
       data
     else
-      data
-      |> Map.put("mediaType", "application/octet-stream")
+      Map.put(data, "mediaType", "application/octet-stream")
     end
   end
 
-  def fix_url(data) do
-    case data["url"] do
-      url when is_binary(url) ->
-        data
-        |> Map.put(
-          "url",
-          [
-            %{
-              "href" => url,
-              "type" => "Link",
-              "mediaType" => data["mediaType"]
-            }
-          ]
-        )
-
-      _ ->
+  defp handle_href(href, mediaType) do
+    [
+      %{
+        "href" => href,
+        "type" => "Link",
+        "mediaType" => mediaType
+      }
+    ]
+  end
+
+  defp fix_url(data) do
+    cond do
+      is_binary(data["url"]) ->
+        Map.put(data, "url", handle_href(data["url"], data["mediaType"]))
+
+      is_binary(data["href"]) and data["url"] == nil ->
+        Map.put(data, "url", handle_href(data["href"], data["mediaType"]))
+
+      true ->
         data
     end
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
index 5d9bf345f..d1869f188 100644
--- a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
@@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
     field(:inReplyTo, :string)
-    field(:uri, ObjectValidators.Uri)
+    field(:url, ObjectValidators.Uri)
     # short identifier for PleromaFE to group statuses by context
     field(:context_id, :integer)
 
@@ -66,10 +66,24 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     |> changeset(data)
   end
 
+  defp fix_url(%{"url" => url} = data) when is_list(url) do
+    attachment =
+      Enum.find(url, fn x -> is_map(x) and String.starts_with?(x["mimeType"], "audio/") end)
+
+    link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end)
+
+    data
+    |> Map.put("attachment", [attachment])
+    |> Map.put("url", link_element["href"])
+  end
+
+  defp fix_url(data), do: data
+
   defp fix(data) do
     data
     |> CommonFixes.fix_defaults()
     |> CommonFixes.fix_attribution()
+    |> fix_url()
   end
 
   def changeset(struct, data) do
@@ -83,7 +97,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
   def validate_data(data_cng) do
     data_cng
     |> validate_inclusion(:type, ["Audio"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
     |> CommonValidations.validate_any_presence([:cc, :to])
     |> CommonValidations.validate_fields_match([:actor, :attributedTo])
     |> CommonValidations.validate_actor_presence()
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index 60868eae0..b3dbeea57 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -61,9 +61,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     end
   end
 
+  defp fix_addressing(data, meta) do
+    if object = meta[:object_data] do
+      data
+      |> Map.put_new("to", object["to"] || [])
+      |> Map.put_new("cc", object["cc"] || [])
+    else
+      data
+    end
+  end
+
   defp fix(data, meta) do
     data
     |> fix_context(meta)
+    |> fix_addressing(meta)
   end
 
   def validate_data(cng, meta \\ []) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
index 14ae29cb6..3e1f13a88 100644
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
@@ -35,7 +35,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
     field(:inReplyTo, ObjectValidators.ObjectID)
-    field(:uri, ObjectValidators.Uri)
+    field(:url, ObjectValidators.Uri)
 
     field(:likes, {:array, :string}, default: [])
     field(:announcements, {:array, :string}, default: [])
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index a7ca42b2f..712047424 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -43,7 +43,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:like_count, :integer, default: 0)
     field(:announcement_count, :integer, default: 0)
     field(:inReplyTo, ObjectValidators.ObjectID)
-    field(:uri, ObjectValidators.Uri)
+    field(:url, ObjectValidators.Uri)
     # short identifier for PleromaFE to group statuses by context
     field(:context_id, :integer)
 
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 6be17e0ed..7c860af9f 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -276,13 +276,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     Map.put(object, "url", url["href"])
   end
 
-  def fix_url(%{"type" => object_type, "url" => url} = object)
-      when object_type in ["Video", "Audio"] and is_list(url) do
+  def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
     attachment =
       Enum.find(url, fn x ->
         media_type = x["mediaType"] || x["mimeType"] || ""
 
-        is_map(x) and String.starts_with?(media_type, ["audio/", "video/"])
+        is_map(x) and String.starts_with?(media_type, "video/")
       end)
 
     link_element =
-- 
cgit v1.2.3


From 7dc275b69bbd50e7a6944c76c5541c0a9c41a051 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 18 Aug 2020 18:21:34 +0300
Subject: relay fix for admin-fe

---
 lib/mix/tasks/pleroma/relay.ex                     | 10 +++-
 lib/pleroma/following_relationship.ex              |  8 ++++
 lib/pleroma/user.ex                                | 17 +++----
 lib/pleroma/web/activity_pub/activity_pub.ex       |  7 ++-
 lib/pleroma/web/activity_pub/builder.ex            |  2 +-
 lib/pleroma/web/activity_pub/relay.ex              | 56 +++++++++++-----------
 .../web/admin_api/controllers/relay_controller.ex  |  2 +-
 .../api_spec/operations/admin/relay_operation.ex   | 50 +++++++++++--------
 8 files changed, 84 insertions(+), 68 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex
index c3312507e..a6d8d6c1c 100644
--- a/lib/mix/tasks/pleroma/relay.ex
+++ b/lib/mix/tasks/pleroma/relay.ex
@@ -35,10 +35,16 @@ defmodule Mix.Tasks.Pleroma.Relay do
   def run(["list"]) do
     start_pleroma()
 
-    with {:ok, list} <- Relay.list(true) do
-      list |> Enum.each(&shell_info(&1))
+    with {:ok, list} <- Relay.list() do
+      Enum.each(list, &print_relay_url/1)
     else
       {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
     end
   end
+
+  defp print_relay_url(%{followed_back: false} = relay) do
+    shell_info("#{relay.actor} - no Accept received (relay didn't follow back)")
+  end
+
+  defp print_relay_url(relay), do: shell_info(relay.actor)
 end
diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex
index 83b366dd4..2039a259d 100644
--- a/lib/pleroma/following_relationship.ex
+++ b/lib/pleroma/following_relationship.ex
@@ -264,4 +264,12 @@ defmodule Pleroma.FollowingRelationship do
       end
     end)
   end
+
+  @spec following_ap_ids(User.t()) :: [String.t()]
+  def following_ap_ids(%User{} = user) do
+    user
+    |> following_query()
+    |> select([r, u], u.ap_id)
+    |> Repo.all()
+  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index a9820affa..d2ad9516f 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -247,6 +247,13 @@ defmodule Pleroma.User do
     end
   end
 
+  defdelegate following_count(user), to: FollowingRelationship
+  defdelegate following(user), to: FollowingRelationship
+  defdelegate following?(follower, followed), to: FollowingRelationship
+  defdelegate following_ap_ids(user), to: FollowingRelationship
+  defdelegate get_follow_requests(user), to: FollowingRelationship
+  defdelegate search(query, opts \\ []), to: User.Search
+
   @doc """
   Dumps Flake Id to SQL-compatible format (16-byte UUID).
   E.g. "9pQtDGXuq4p3VlcJEm" -> <<0, 0, 1, 110, 179, 218, 42, 92, 213, 41, 44, 227, 95, 213, 0, 0>>
@@ -372,8 +379,6 @@ defmodule Pleroma.User do
     from(u in query, where: u.deactivated != ^true)
   end
 
-  defdelegate following_count(user), to: FollowingRelationship
-
   defp truncate_fields_param(params) do
     if Map.has_key?(params, :fields) do
       Map.put(params, :fields, Enum.map(params[:fields], &truncate_field/1))
@@ -868,8 +873,6 @@ defmodule Pleroma.User do
     set_cache(follower)
   end
 
-  defdelegate following(user), to: FollowingRelationship
-
   def follow(%User{} = follower, %User{} = followed, state \\ :follow_accept) do
     deny_follow_blocked = Config.get([:user, :deny_follow_blocked])
 
@@ -923,8 +926,6 @@ defmodule Pleroma.User do
     end
   end
 
-  defdelegate following?(follower, followed), to: FollowingRelationship
-
   @doc "Returns follow state as Pleroma.FollowingRelationship.State value"
   def get_follow_state(%User{} = follower, %User{} = following) do
     following_relationship = FollowingRelationship.get(follower, following)
@@ -1189,8 +1190,6 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
-  defdelegate get_follow_requests(user), to: FollowingRelationship
-
   def increase_note_count(%User{} = user) do
     User
     |> where(id: ^user.id)
@@ -2163,8 +2162,6 @@ defmodule Pleroma.User do
     |> Repo.all()
   end
 
-  defdelegate search(query, opts \\ []), to: User.Search
-
   defp put_password_hash(
          %Ecto.Changeset{valid?: true, changes: %{password: password}} = changeset
        ) do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index bde1fe708..04478bc33 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1344,9 +1344,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def maybe_handle_clashing_nickname(data) do
-    nickname = data[:nickname]
-
-    with %User{} = old_user <- User.get_by_nickname(nickname),
+    with nickname when is_binary(nickname) <- data[:nickname],
+         %User{} = old_user <- User.get_by_nickname(nickname),
          {_, false} <- {:ap_id_comparison, data[:ap_id] == old_user.ap_id} do
       Logger.info(
         "Found an old user for #{nickname}, the old ap id is #{old_user.ap_id}, new one is #{
@@ -1360,7 +1359,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     else
       {:ap_id_comparison, true} ->
         Logger.info(
-          "Found an old user for #{nickname}, but the ap id #{data[:ap_id]} is the same as the new user. Race condition? Not changing anything."
+          "Found an old user for #{data[:nickname]}, but the ap id #{data[:ap_id]} is the same as the new user. Race condition? Not changing anything."
         )
 
       _ ->
diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex
index f2392ce79..9a7b7d9de 100644
--- a/lib/pleroma/web/activity_pub/builder.ex
+++ b/lib/pleroma/web/activity_pub/builder.ex
@@ -215,7 +215,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do
 
     to =
       cond do
-        actor.ap_id == Relay.relay_ap_id() ->
+        actor.ap_id == Relay.ap_id() ->
           [actor.follower_address]
 
         public? ->
diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex
index b09764d2b..b65710a94 100644
--- a/lib/pleroma/web/activity_pub/relay.ex
+++ b/lib/pleroma/web/activity_pub/relay.ex
@@ -10,19 +10,13 @@ defmodule Pleroma.Web.ActivityPub.Relay do
   alias Pleroma.Web.CommonAPI
   require Logger
 
-  @relay_nickname "relay"
+  @nickname "relay"
 
-  def get_actor do
-    actor =
-      relay_ap_id()
-      |> User.get_or_create_service_actor_by_ap_id(@relay_nickname)
+  @spec ap_id() :: String.t()
+  def ap_id, do: "#{Pleroma.Web.Endpoint.url()}/#{@nickname}"
 
-    actor
-  end
-
-  def relay_ap_id do
-    "#{Pleroma.Web.Endpoint.url()}/relay"
-  end
+  @spec get_actor() :: User.t() | nil
+  def get_actor, do: User.get_or_create_service_actor_by_ap_id(ap_id(), @nickname)
 
   @spec follow(String.t()) :: {:ok, Activity.t()} | {:error, any()}
   def follow(target_instance) do
@@ -61,34 +55,38 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def publish(_), do: {:error, "Not implemented"}
 
-  @spec list(boolean()) :: {:ok, [String.t()]} | {:error, any()}
-  def list(with_not_accepted \\ false) do
+  @spec list() :: {:ok, [%{actor: String.t(), followed_back: boolean()}]} | {:error, any()}
+  def list do
     with %User{} = user <- get_actor() do
       accepted =
         user
-        |> User.following()
-        |> Enum.map(fn entry -> URI.parse(entry).host end)
-        |> Enum.uniq()
-
-      list =
-        if with_not_accepted do
-          without_accept =
-            user
-            |> Pleroma.Activity.following_requests_for_actor()
-            |> Enum.map(fn a -> URI.parse(a.data["object"]).host <> " (no Accept received)" end)
-            |> Enum.uniq()
+        |> following()
+        |> Enum.map(fn actor -> %{actor: actor, followed_back: true} end)
 
-          accepted ++ without_accept
-        else
-          accepted
-        end
+      without_accept =
+        user
+        |> Pleroma.Activity.following_requests_for_actor()
+        |> Enum.map(fn activity -> %{actor: activity.data["object"], followed_back: false} end)
+        |> Enum.uniq()
 
-      {:ok, list}
+      {:ok, accepted ++ without_accept}
     else
       error -> format_error(error)
     end
   end
 
+  @spec following() :: [String.t()]
+  def following do
+    get_actor()
+    |> following()
+  end
+
+  defp following(user) do
+    user
+    |> User.following_ap_ids()
+    |> Enum.uniq()
+  end
+
   defp format_error({:error, error}), do: format_error(error)
 
   defp format_error(error) do
diff --git a/lib/pleroma/web/admin_api/controllers/relay_controller.ex b/lib/pleroma/web/admin_api/controllers/relay_controller.ex
index cf9f3a14b..95d06dde7 100644
--- a/lib/pleroma/web/admin_api/controllers/relay_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/relay_controller.ex
@@ -39,7 +39,7 @@ defmodule Pleroma.Web.AdminAPI.RelayController do
         target: target
       })
 
-      json(conn, target)
+      json(conn, %{actor: target, followed_back: target in Relay.following()})
     else
       _ ->
         conn
diff --git a/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex b/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex
index 67ee5eee0..e06b2d164 100644
--- a/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex
@@ -27,8 +27,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do
             properties: %{
               relays: %Schema{
                 type: :array,
-                items: %Schema{type: :string},
-                example: ["lain.com", "mstdn.io"]
+                items: relay()
               }
             }
           })
@@ -43,19 +42,9 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do
       operationId: "AdminAPI.RelayController.follow",
       security: [%{"oAuth" => ["write:follows"]}],
       parameters: admin_api_params(),
-      requestBody:
-        request_body("Parameters", %Schema{
-          type: :object,
-          properties: %{
-            relay_url: %Schema{type: :string, format: :uri}
-          }
-        }),
+      requestBody: request_body("Parameters", relay_url()),
       responses: %{
-        200 =>
-          Operation.response("Status", "application/json", %Schema{
-            type: :string,
-            example: "http://mastodon.example.org/users/admin"
-          })
+        200 => Operation.response("Status", "application/json", relay())
       }
     }
   end
@@ -67,13 +56,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do
       operationId: "AdminAPI.RelayController.unfollow",
       security: [%{"oAuth" => ["write:follows"]}],
       parameters: admin_api_params(),
-      requestBody:
-        request_body("Parameters", %Schema{
-          type: :object,
-          properties: %{
-            relay_url: %Schema{type: :string, format: :uri}
-          }
-        }),
+      requestBody: request_body("Parameters", relay_url()),
       responses: %{
         200 =>
           Operation.response("Status", "application/json", %Schema{
@@ -83,4 +66,29 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do
       }
     }
   end
+
+  defp relay do
+    %Schema{
+      type: :object,
+      properties: %{
+        actor: %Schema{
+          type: :string,
+          example: "https://example.com/relay"
+        },
+        followed_back: %Schema{
+          type: :boolean,
+          description: "Is relay followed back by this actor?"
+        }
+      }
+    }
+  end
+
+  defp relay_url do
+    %Schema{
+      type: :object,
+      properties: %{
+        relay_url: %Schema{type: :string, format: :uri}
+      }
+    }
+  end
 end
-- 
cgit v1.2.3


From 4727030f59a5d879a579c4bccd0f1612c5221670 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 19 Aug 2020 11:06:03 +0300
Subject: fixes for mix tasks

- fix for `mix pleroma.database update_users_following_followers_counts`
- raise error, if fetch was unsuccessful in emoji tasks
- fix for `pleroma.digest test` task
---
 lib/mix/pleroma.ex               |  2 +-
 lib/mix/tasks/pleroma/emoji.ex   | 10 ++++++----
 lib/pleroma/emails/user_email.ex | 31 ++++++++++++++++++++-----------
 3 files changed, 27 insertions(+), 16 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index 074492a46..fe9b0d16c 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -14,7 +14,7 @@ defmodule Mix.Pleroma do
     :swoosh,
     :timex
   ]
-  @cachex_children ["object", "user"]
+  @cachex_children ["object", "user", "scrubber"]
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index f4eaeac98..8f52ee98d 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -15,7 +15,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     {options, [], []} = parse_global_opts(args)
 
     url_or_path = options[:manifest] || default_manifest()
-    manifest = fetch_and_decode(url_or_path)
+    manifest = fetch_and_decode!(url_or_path)
 
     Enum.each(manifest, fn {name, info} ->
       to_print = [
@@ -42,7 +42,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
     url_or_path = options[:manifest] || default_manifest()
 
-    manifest = fetch_and_decode(url_or_path)
+    manifest = fetch_and_decode!(url_or_path)
 
     for pack_name <- pack_names do
       if Map.has_key?(manifest, pack_name) do
@@ -92,7 +92,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
           ])
         )
 
-        files = fetch_and_decode(files_loc)
+        files = fetch_and_decode!(files_loc)
 
         IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name]))
 
@@ -243,9 +243,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     IO.puts("Emoji packs have been reloaded.")
   end
 
-  defp fetch_and_decode(from) do
+  defp fetch_and_decode!(from) do
     with {:ok, json} <- fetch(from) do
       Jason.decode!(json)
+    else
+      {:error, error} -> raise "#{from} cannot be fetched. Error: #{error} occur."
     end
   end
 
diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex
index 313533859..1d8c72ae9 100644
--- a/lib/pleroma/emails/user_email.ex
+++ b/lib/pleroma/emails/user_email.ex
@@ -107,25 +107,34 @@ defmodule Pleroma.Emails.UserEmail do
       |> Enum.filter(&(&1.activity.data["type"] == "Create"))
       |> Enum.map(fn notification ->
         object = Pleroma.Object.normalize(notification.activity)
-        object = update_in(object.data["content"], &format_links/1)
 
-        %{
-          data: notification,
-          object: object,
-          from: User.get_by_ap_id(notification.activity.actor)
-        }
+        if not is_nil(object) do
+          object = update_in(object.data["content"], &format_links/1)
+
+          %{
+            data: notification,
+            object: object,
+            from: User.get_by_ap_id(notification.activity.actor)
+          }
+        end
       end)
+      |> Enum.filter(& &1)
 
     followers =
       notifications
       |> Enum.filter(&(&1.activity.data["type"] == "Follow"))
       |> Enum.map(fn notification ->
-        %{
-          data: notification,
-          object: Pleroma.Object.normalize(notification.activity),
-          from: User.get_by_ap_id(notification.activity.actor)
-        }
+        from = User.get_by_ap_id(notification.activity.actor)
+
+        if not is_nil(from) do
+          %{
+            data: notification,
+            object: Pleroma.Object.normalize(notification.activity),
+            from: User.get_by_ap_id(notification.activity.actor)
+          }
+        end
       end)
+      |> Enum.filter(& &1)
 
     unless Enum.empty?(mentions) do
       styling = Config.get([__MODULE__, :styling])
-- 
cgit v1.2.3


From 4ee15e991efb5bd5bf69d84d27dbbee81443d1dc Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 19 Aug 2020 21:36:26 +0300
Subject: [#2497] Media preview proxy config refactoring & documentation.

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 8861398dd..31d18c119 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -15,8 +15,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
          {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)},
          :ok <- MediaProxy.verify_request_path_and_url(conn, url) do
-      proxy_opts = Config.get([:media_proxy, :proxy_opts], [])
-      ReverseProxy.call(conn, url, proxy_opts)
+      ReverseProxy.call(conn, url, media_proxy_opts())
     else
       {:enabled, false} ->
         send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
@@ -116,13 +115,16 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   end
 
   defp preview_head_request_timeout do
-    Config.get([:media_preview_proxy, :proxy_opts, :head_request_max_read_duration]) ||
-      preview_timeout()
+    Keyword.get(media_preview_proxy_opts(), :head_request_max_read_duration) ||
+      Keyword.get(media_proxy_opts(), :max_read_duration) ||
+      ReverseProxy.max_read_duration_default()
   end
 
-  defp preview_timeout do
-    Config.get([:media_preview_proxy, :proxy_opts, :max_read_duration]) ||
-      Config.get([:media_proxy, :proxy_opts, :max_read_duration]) ||
-      ReverseProxy.max_read_duration_default()
+  defp media_proxy_opts do
+    Config.get([:media_proxy, :proxy_opts], [])
+  end
+
+  defp media_preview_proxy_opts do
+    Config.get([:media_preview_proxy, :proxy_opts], [])
   end
 end
-- 
cgit v1.2.3


From 02ad1cd8e97c44824b92b53ea1879a965bbd8358 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 20 Aug 2020 09:58:50 +0300
Subject: [#2497] Media preview proxy: added Content-Disposition header with
 filename to response.

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 31d18c119..5513432f0 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -87,6 +87,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
            ) do
       conn
       |> put_resp_header("content-type", "image/jpeg")
+      |> put_resp_header("content-disposition", "inline; filename=\"preview.jpg\"")
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
-- 
cgit v1.2.3


From 36c125a071e1fe5a3c0bb1f33a18ba60965437ab Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 20 Aug 2020 18:41:42 +0200
Subject: Pipeline Ingestion: Event

---
 lib/pleroma/web/activity_pub/activity_pub.ex       |  2 +-
 lib/pleroma/web/activity_pub/object_validator.ex   | 17 +++-
 .../object_validators/event_validator.ex           | 96 ++++++++++++++++++++++
 .../object_validators/note_validator.ex            | 11 ++-
 lib/pleroma/web/activity_pub/side_effects.ex       |  2 +-
 lib/pleroma/web/activity_pub/transmogrifier.ex     |  4 +-
 lib/pleroma/web/mastodon_api/views/status_view.ex  | 19 +----
 7 files changed, 126 insertions(+), 25 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/event_validator.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index db1867494..8c5b7dac2 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -85,7 +85,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio]
+  @object_types ~w[ChatMessage Question Answer Audio Event]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index d770ce1be..b77c06395 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -23,6 +23,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.EventValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator
@@ -43,6 +44,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(%{"type" => "Event"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> EventValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
   def validate(%{"type" => "Follow"} = object, meta) do
     with {:ok, object} <-
            object
@@ -187,7 +198,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
       )
-      when objtype in ~w[Question Answer Audio] do
+      when objtype in ~w[Question Answer Audio Event] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -225,6 +236,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     AudioValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Event"} = object) do
+    EventValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
   # is_struct/1 isn't present in Elixir 1.8.x
diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex
new file mode 100644
index 000000000..07e4821a4
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex
@@ -0,0 +1,96 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  # Extends from NoteValidator
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:bto, ObjectValidators.Recipients, default: [])
+    field(:bcc, ObjectValidators.Recipients, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
+    field(:content, :string)
+
+    field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:published, ObjectValidators.DateTime)
+    # TODO: Write type
+    field(:emoji, :map, default: %{})
+    field(:sensitive, :boolean, default: false)
+    embeds_many(:attachment, AttachmentValidator)
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, ObjectValidators.ObjectID)
+    field(:url, ObjectValidators.Uri)
+
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  defp fix(data) do
+    data
+    |> CommonFixes.fix_defaults()
+    |> CommonFixes.fix_attribution()
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast_embed(:attachment)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Event"])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_host_match()
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
index 3e1f13a88..20e735619 100644
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
@@ -20,11 +20,17 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     # TODO: Write type
     field(:tag, {:array, :map}, default: [])
     field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
     field(:content, :string)
+
     field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
     field(:actor, ObjectValidators.ObjectID)
     field(:attributedTo, ObjectValidators.ObjectID)
-    field(:summary, :string)
     field(:published, ObjectValidators.DateTime)
     # TODO: Write type
     field(:emoji, :map, default: %{})
@@ -39,9 +45,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
 
     field(:likes, {:array, :string}, default: [])
     field(:announcements, {:array, :string}, default: [])
-
-    # see if needed
-    field(:context_id, :string)
   end
 
   def cast_and_validate(data) do
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 3dc66c60b..a5e2323bd 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -341,7 +341,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   end
 
   def handle_object_creation(%{"type" => objtype} = object, meta)
-      when objtype in ~w[Audio Question] do
+      when objtype in ~w[Audio Question Event] do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
     end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 7c860af9f..76298c4a0 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -460,7 +460,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ~w{Article Event Note Video Page} do
+      when objtype in ~w{Article Note Video Page} do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -554,7 +554,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype}} = data,
         _options
       )
-      when objtype in ~w{Question Answer ChatMessage Audio} do
+      when objtype in ~w{Question Answer ChatMessage Audio Event} do
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 91b41ef59..01b8bb6bb 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -473,23 +473,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end
   end
 
-  def render_content(%{data: %{"type" => object_type}} = object)
-      when object_type in ["Video", "Event", "Audio"] do
-    with name when not is_nil(name) and name != "" <- object.data["name"] do
-      "<p><a href=\"#{object.data["id"]}\">#{name}</a></p>#{object.data["content"]}"
-    else
-      _ -> object.data["content"] || ""
-    end
-  end
+  def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do
+    url = object.data["url"] || object.data["id"]
 
-  def render_content(%{data: %{"type" => object_type}} = object)
-      when object_type in ["Article", "Page"] do
-    with summary when not is_nil(summary) and summary != "" <- object.data["name"],
-         url when is_bitstring(url) <- object.data["url"] do
-      "<p><a href=\"#{url}\">#{summary}</a></p>#{object.data["content"]}"
-    else
-      _ -> object.data["content"] || ""
-    end
+    "<p><a href=\"#{url}\">#{name}</a></p>#{object.data["content"]}"
   end
 
   def render_content(object), do: object.data["content"] || ""
-- 
cgit v1.2.3


From aa0a5ffb4849880b5adbcc9188de01ef778381e3 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Fri, 21 Aug 2020 08:59:08 +0300
Subject: [#2497] Media preview proxy: added `quality` config setting, adjusted
 width/height defaults.

---
 lib/pleroma/helpers/media_helper.ex                   | 6 ++++--
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 4 +++-
 2 files changed, 7 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index ca46698cc..e11038052 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -7,13 +7,15 @@ defmodule Pleroma.Helpers.MediaHelper do
   Handles common media-related operations.
   """
 
-  def ffmpeg_resize(uri_or_path, %{max_width: max_width, max_height: max_height}) do
+  def ffmpeg_resize(uri_or_path, %{max_width: max_width, max_height: max_height} = options) do
+    quality = options[:quality] || 1
+
     cmd = ~s"""
     ffmpeg -i #{uri_or_path} -f lavfi -i color=c=white \
       -filter_complex "[0:v] scale='min(#{max_width},iw)':'min(#{max_height},ih)': \
         force_original_aspect_ratio=decrease [scaled]; \
         [1][scaled] scale2ref [bg][img]; [bg] setsar=1 [bg]; [bg][img] overlay=shortest=1" \
-      -loglevel quiet -f image2 -vcodec mjpeg -frames:v 1 pipe:1
+      -loglevel quiet -f image2 -vcodec mjpeg -frames:v 1 -q:v #{quality} pipe:1
     """
 
     pid = Port.open({:spawn, cmd}, [:use_stdio, :in, :stream, :exit_status, :binary])
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 5513432f0..1c51aa5e3 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -78,12 +78,14 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   end
 
   defp handle_image_or_video_preview(%{params: params} = conn, url) do
+    quality = Config.get!([:media_preview_proxy, :quality])
+
     with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
          media_proxy_url <- MediaProxy.url(url),
          {:ok, thumbnail_binary} <-
            MediaHelper.ffmpeg_resize(
              media_proxy_url,
-             %{max_width: thumbnail_max_width, max_height: thumbnail_max_height}
+             %{max_width: thumbnail_max_width, max_height: thumbnail_max_height, quality: quality}
            ) do
       conn
       |> put_resp_header("content-type", "image/jpeg")
-- 
cgit v1.2.3


From 1f8c32b773b56e950285ce96cb9a62f045f2a225 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 21 Aug 2020 10:38:56 +0300
Subject: adding actor type in user show

---
 lib/pleroma/web/admin_api/views/account_view.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index 333e72e42..9c477feab 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -79,7 +79,8 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
       "confirmation_pending" => user.confirmation_pending,
       "approval_pending" => user.approval_pending,
       "url" => user.uri || user.ap_id,
-      "registration_reason" => user.registration_reason
+      "registration_reason" => user.registration_reason,
+      "actor_type" => user.actor_type
     }
   end
 
-- 
cgit v1.2.3


From 967afa064bb0dc85c054495b795a57a13cdf3b3c Mon Sep 17 00:00:00 2001
From: href <href+git-pleroma@random.sh>
Date: Fri, 21 Aug 2020 17:02:57 +0000
Subject: Fix truncated images

---
 lib/pleroma/helpers/media_helper.ex | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index e11038052..f87be8874 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -19,14 +19,24 @@ defmodule Pleroma.Helpers.MediaHelper do
     """
 
     pid = Port.open({:spawn, cmd}, [:use_stdio, :in, :stream, :exit_status, :binary])
+    loop_recv(pid)
+  end
+
+  defp loop_recv(pid) do
+    loop_recv(pid, <<>>)
+  end
 
+  defp loop_recv(pid, acc) do
     receive do
       {^pid, {:data, data}} ->
-        send(pid, {self(), :close})
-        {:ok, data}
+        loop_recv(pid, acc <> data)
 
-      {^pid, {:exit_status, status}} when status > 0 ->
+      {^pid, {:exit_status, 0}} ->
+        {:ok, acc}
+
+      {^pid, {:exit_status, status}} ->
         {:error, status}
     end
   end
+
 end
-- 
cgit v1.2.3


From 4e6eb22b4af70e611cc61f94ba3d81758036a392 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 21 Aug 2020 12:19:35 -0500
Subject: Try to warm the cache with the preview image if preview proxy enabled

---
 lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index dfab105a3..5d8bb72aa 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
       end
 
     url
-    |> MediaProxy.url()
+    |> MediaProxy.preview_url()
     |> HTTP.get([], adapter: opts)
   end
 
-- 
cgit v1.2.3


From edde0d9b54b45a366ecdec01e9826f1ee8d1dc3a Mon Sep 17 00:00:00 2001
From: href <href+git-pleroma@random.sh>
Date: Fri, 21 Aug 2020 17:40:49 +0000
Subject: Remove newline for linter

---
 lib/pleroma/helpers/media_helper.ex | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index f87be8874..89dd4204b 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -38,5 +38,4 @@ defmodule Pleroma.Helpers.MediaHelper do
         {:error, status}
     end
   end
-
 end
-- 
cgit v1.2.3


From 98f8851f29f940051656caa1715820bce70f8c29 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Sat, 22 Aug 2020 15:12:11 -0500
Subject: Use the image thumbnail for rich metadata (OGP/Twittercards)

---
 lib/pleroma/web/metadata/utils.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex
index 2f0dfb474..8a206e019 100644
--- a/lib/pleroma/web/metadata/utils.ex
+++ b/lib/pleroma/web/metadata/utils.ex
@@ -38,7 +38,7 @@ defmodule Pleroma.Web.Metadata.Utils do
   def scrub_html(content), do: content
 
   def attachment_url(url) do
-    MediaProxy.url(url)
+    MediaProxy.preview_url(url)
   end
 
   def user_name_string(user) do
-- 
cgit v1.2.3


From 112bec52252108de57e708ed47cf43abd9f3b2f1 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 25 Aug 2020 17:35:59 +0200
Subject: Webfinger: Handle bogus ids better.

---
 lib/pleroma/web/web_finger/web_finger.ex | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index 71ccf251a..c4051e63e 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -149,6 +149,18 @@ defmodule Pleroma.Web.WebFinger do
     end
   end
 
+  defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do
+    case find_lrdd_template(domain) do
+      {:ok, template} ->
+        String.replace(template, "{uri}", encoded_account)
+
+      _ ->
+        "https://#{domain}/.well-known/webfinger?resource=#{encoded_account}"
+    end
+  end
+
+  defp get_address_from_domain(_, _), do: nil
+
   @spec finger(String.t()) :: {:ok, map()} | {:error, any()}
   def finger(account) do
     account = String.trim_leading(account, "@")
@@ -163,16 +175,8 @@ defmodule Pleroma.Web.WebFinger do
 
     encoded_account = URI.encode("acct:#{account}")
 
-    address =
-      case find_lrdd_template(domain) do
-        {:ok, template} ->
-          String.replace(template, "{uri}", encoded_account)
-
-        _ ->
-          "https://#{domain}/.well-known/webfinger?resource=#{encoded_account}"
-      end
-
-    with response <-
+    with address when is_binary(address) <- get_address_from_domain(domain, encoded_account),
+         response <-
            HTTP.get(
              address,
              [{"accept", "application/xrd+xml,application/jrd+json"}]
-- 
cgit v1.2.3


From 899ea2da3e77ca64598e45eba986d5315b523120 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 25 Aug 2020 17:18:22 -0500
Subject: Switch to imagemagick, only support videos

---
 lib/pleroma/helpers/media_helper.ex                   | 13 ++++++-------
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 15 +++++----------
 2 files changed, 11 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 89dd4204b..07e6dba5e 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -7,18 +7,17 @@ defmodule Pleroma.Helpers.MediaHelper do
   Handles common media-related operations.
   """
 
-  def ffmpeg_resize(uri_or_path, %{max_width: max_width, max_height: max_height} = options) do
-    quality = options[:quality] || 1
+  def image_resize(url, %{max_width: max_width, max_height: max_height} = options) do
+    quality = options[:quality] || 85
 
     cmd = ~s"""
-    ffmpeg -i #{uri_or_path} -f lavfi -i color=c=white \
-      -filter_complex "[0:v] scale='min(#{max_width},iw)':'min(#{max_height},ih)': \
-        force_original_aspect_ratio=decrease [scaled]; \
-        [1][scaled] scale2ref [bg][img]; [bg] setsar=1 [bg]; [bg][img] overlay=shortest=1" \
-      -loglevel quiet -f image2 -vcodec mjpeg -frames:v 1 -q:v #{quality} pipe:1
+    convert - -resize '#{max_width}x#{max_height}>' -quality #{quality} -
     """
 
     pid = Port.open({:spawn, cmd}, [:use_stdio, :in, :stream, :exit_status, :binary])
+    {:ok, env} = url |> Pleroma.Web.MediaProxy.url() |> Pleroma.HTTP.get()
+    image = env.body
+    Port.command(pid, image)
     loop_recv(pid)
   end
 
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 1c51aa5e3..b925973ba 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -66,25 +66,20 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   end
 
   defp handle_preview("image/" <> _ = _content_type, conn, url) do
-    handle_image_or_video_preview(conn, url)
-  end
-
-  defp handle_preview("video/" <> _ = _content_type, conn, url) do
-    handle_image_or_video_preview(conn, url)
+    handle_image_preview(conn, url)
   end
 
   defp handle_preview(content_type, conn, _url) do
     send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
   end
 
-  defp handle_image_or_video_preview(%{params: params} = conn, url) do
-    quality = Config.get!([:media_preview_proxy, :quality])
+  defp handle_image_preview(%{params: params} = conn, url) do
+    quality = Config.get!([:media_preview_proxy, :image_quality])
 
     with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
-         media_proxy_url <- MediaProxy.url(url),
          {:ok, thumbnail_binary} <-
-           MediaHelper.ffmpeg_resize(
-             media_proxy_url,
+           MediaHelper.image_resize(
+             url,
              %{max_width: thumbnail_max_width, max_height: thumbnail_max_height, quality: quality}
            ) do
       conn
-- 
cgit v1.2.3


From ddbddc08fc9fe5458edc983c81a77671da34a71f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 25 Aug 2020 17:31:55 -0500
Subject: Redirects for videos right now

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index b925973ba..6abbf9e23 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -69,6 +69,12 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     handle_image_preview(conn, url)
   end
 
+  defp handle_preview("video/" <> _ = _content_type, conn, url) do
+    mediaproxy_url = url |> MediaProxy.url()
+
+    redirect(conn, external: mediaproxy_url)
+  end
+
   defp handle_preview(content_type, conn, _url) do
     send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
   end
-- 
cgit v1.2.3


From afa03ca8e2cffc85628beb5f9a70401d984ab216 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 25 Aug 2020 17:36:53 -0500
Subject: Allow both stdin and stdout

---
 lib/pleroma/helpers/media_helper.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 07e6dba5e..5fe135584 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Helpers.MediaHelper do
     convert - -resize '#{max_width}x#{max_height}>' -quality #{quality} -
     """
 
-    pid = Port.open({:spawn, cmd}, [:use_stdio, :in, :stream, :exit_status, :binary])
+    pid = Port.open({:spawn, cmd}, [:use_stdio, :stream, :exit_status, :binary])
     {:ok, env} = url |> Pleroma.Web.MediaProxy.url() |> Pleroma.HTTP.get()
     image = env.body
     Port.command(pid, image)
-- 
cgit v1.2.3


From a136e7e9b590e3f23e472bf27c7c6a81d8d7792b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 25 Aug 2020 18:10:27 -0500
Subject: Try specifying fd0, force jpg out

---
 lib/pleroma/helpers/media_helper.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 5fe135584..01f42d9b0 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Helpers.MediaHelper do
     quality = options[:quality] || 85
 
     cmd = ~s"""
-    convert - -resize '#{max_width}x#{max_height}>' -quality #{quality} -
+    convert fd:0 -resize '#{max_width}x#{max_height}>' -quality #{quality} jpg:-
     """
 
     pid = Port.open({:spawn, cmd}, [:use_stdio, :stream, :exit_status, :binary])
-- 
cgit v1.2.3


From cfc99fe05c31d5e2140c35f3a2d223635dc07a2f Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 26 Aug 2020 15:37:30 +0200
Subject: TimelineController: Keys are atoms now.

Closes #2078
Closes #2070
---
 lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
index 9244316ed..5272790d3 100644
--- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex
@@ -182,11 +182,10 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do
     with %Pleroma.List{title: _title, following: following} <- Pleroma.List.get(id, user) do
       params =
         params
-        |> Map.new(fn {key, value} -> {to_string(key), value} end)
-        |> Map.put("type", "Create")
-        |> Map.put("blocking_user", user)
-        |> Map.put("user", user)
-        |> Map.put("muting_user", user)
+        |> Map.put(:type, "Create")
+        |> Map.put(:blocking_user, user)
+        |> Map.put(:user, user)
+        |> Map.put(:muting_user, user)
 
       # we must filter the following list for the user to avoid leaking statuses the user
       # does not actually have permission to see (for more info, peruse security issue #270).
-- 
cgit v1.2.3


From bc94f0c6da2405e2f1cdae89696970728b6e987f Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Wed, 26 Aug 2020 16:12:34 +0200
Subject: Use mkfifo to feed ImageMagick

---
 lib/pleroma/helpers/media_helper.ex | 70 +++++++++++++++++++++++++++++++------
 1 file changed, 60 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 01f42d9b0..a43352ae0 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -7,18 +7,66 @@ defmodule Pleroma.Helpers.MediaHelper do
   Handles common media-related operations.
   """
 
-  def image_resize(url, %{max_width: max_width, max_height: max_height} = options) do
+  @tmp_base "/tmp/pleroma-media_preview-pipe"
+
+  def image_resize(url, options) do
+    with executable when is_binary(executable) <- System.find_executable("convert"),
+         {:ok, args} <- prepare_resize_args(options),
+         url = Pleroma.Web.MediaProxy.url(url),
+         {:ok, env} <- Pleroma.HTTP.get(url),
+         {:ok, fifo_path} <- mkfifo()
+    do
+      run_fifo(fifo_path, env, executable, args)
+    else
+      nil -> {:error, {:convert, :command_not_found}}
+      {:error, _} = error -> error
+    end
+  end
+
+  defp prepare_resize_args(%{max_width: max_width, max_height: max_height} = options) do
     quality = options[:quality] || 85
+    resize = Enum.join([max_width, "x", max_height, ">"])
+    args = [
+    "-auto-orient", # Support for EXIF rotation
+    "-resize", resize,
+    "-quality", to_string(quality)
+    ]
+    {:ok, args}
+  end
 
-    cmd = ~s"""
-    convert fd:0 -resize '#{max_width}x#{max_height}>' -quality #{quality} jpg:-
-    """
+  defp prepare_resize_args(_), do: {:error, :missing_options}
 
-    pid = Port.open({:spawn, cmd}, [:use_stdio, :stream, :exit_status, :binary])
-    {:ok, env} = url |> Pleroma.Web.MediaProxy.url() |> Pleroma.HTTP.get()
-    image = env.body
-    Port.command(pid, image)
+  defp run_fifo(fifo_path, env, executable, args) do
+    args = List.flatten([fifo_path, args, "jpg:fd:1"])
+    pid = Port.open({:spawn_executable, executable}, [:use_stdio, :stream, :exit_status, :binary, args: args])
+    fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
+    true = Port.command(fifo, env.body)
+    :erlang.port_close(fifo)
     loop_recv(pid)
+  after
+    File.rm(fifo_path)
+  end
+
+  defp mkfifo() do
+    path = "#{@tmp_base}#{to_charlist(:erlang.phash2(self()))}"
+    case System.cmd("mkfifo", [path]) do
+      {_, 0} ->
+        spawn(fifo_guard(path))
+        {:ok, path}
+      {_, err} ->
+        {:error, {:fifo_failed, err}}
+    end
+  end
+
+  defp fifo_guard(path) do
+    pid = self()
+    fn() ->
+      ref = Process.monitor(pid)
+      receive do
+        {:DOWN, ^ref, :process, ^pid, _} ->
+          File.rm(path)
+      end
+    end
   end
 
   defp loop_recv(pid) do
@@ -29,12 +77,14 @@ defmodule Pleroma.Helpers.MediaHelper do
     receive do
       {^pid, {:data, data}} ->
         loop_recv(pid, acc <> data)
-
       {^pid, {:exit_status, 0}} ->
         {:ok, acc}
-
       {^pid, {:exit_status, status}} ->
         {:error, status}
+     after
+       5000 ->
+         :erlang.port_close(pid)
+         {:error, :timeout}
     end
   end
 end
-- 
cgit v1.2.3


From d4d1192341868d978e19777c17be85e331367264 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 26 Aug 2020 14:28:25 +0000
Subject: Remove auto-orient; don't use it on previews, only originals

---
 lib/pleroma/helpers/media_helper.ex | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index a43352ae0..db0c4b0cf 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -27,7 +27,6 @@ defmodule Pleroma.Helpers.MediaHelper do
     quality = options[:quality] || 85
     resize = Enum.join([max_width, "x", max_height, ">"])
     args = [
-    "-auto-orient", # Support for EXIF rotation
     "-resize", resize,
     "-quality", to_string(quality)
     ]
-- 
cgit v1.2.3


From 2c95533ead56217ec27e09e0ead0050e110dff22 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 26 Aug 2020 15:37:45 +0000
Subject: Change method of convert using stdout, make progressive jpegs

---
 lib/pleroma/helpers/media_helper.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index db0c4b0cf..3256802a0 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -27,6 +27,7 @@ defmodule Pleroma.Helpers.MediaHelper do
     quality = options[:quality] || 85
     resize = Enum.join([max_width, "x", max_height, ">"])
     args = [
+    "-interlace", "Plane",
     "-resize", resize,
     "-quality", to_string(quality)
     ]
@@ -36,7 +37,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   defp prepare_resize_args(_), do: {:error, :missing_options}
 
   defp run_fifo(fifo_path, env, executable, args) do
-    args = List.flatten([fifo_path, args, "jpg:fd:1"])
+    args = List.flatten([fifo_path, args, "jpg:-"])
     pid = Port.open({:spawn_executable, executable}, [:use_stdio, :stream, :exit_status, :binary, args: args])
     fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
     true = Port.command(fifo, env.body)
-- 
cgit v1.2.3


From eead2276e79f29c4d0e10d23eb7524a9ee5f5045 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 26 Aug 2020 16:18:11 -0500
Subject: Ensure GIFs are redirected to the original or they become static.

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 6abbf9e23..d465ce8d1 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -65,6 +65,12 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
+  defp handle_preview("image/gif" = _content_type, conn, url) do
+    mediaproxy_url = url |> MediaProxy.url()
+
+    redirect(conn, external: mediaproxy_url)
+  end
+
   defp handle_preview("image/" <> _ = _content_type, conn, url) do
     handle_image_preview(conn, url)
   end
-- 
cgit v1.2.3


From 9567b96c7927be433eac4f023051adc5cbd6610c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 26 Aug 2020 16:40:13 -0500
Subject: Rename to make it obvious this is for images not videos

---
 lib/pleroma/helpers/media_helper.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 3256802a0..fe11dd460 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   def image_resize(url, options) do
     with executable when is_binary(executable) <- System.find_executable("convert"),
-         {:ok, args} <- prepare_resize_args(options),
+         {:ok, args} <- prepare_image_resize_args(options),
          url = Pleroma.Web.MediaProxy.url(url),
          {:ok, env} <- Pleroma.HTTP.get(url),
          {:ok, fifo_path} <- mkfifo()
@@ -23,7 +23,7 @@ defmodule Pleroma.Helpers.MediaHelper do
     end
   end
 
-  defp prepare_resize_args(%{max_width: max_width, max_height: max_height} = options) do
+  defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do
     quality = options[:quality] || 85
     resize = Enum.join([max_width, "x", max_height, ">"])
     args = [
@@ -34,7 +34,7 @@ defmodule Pleroma.Helpers.MediaHelper do
     {:ok, args}
   end
 
-  defp prepare_resize_args(_), do: {:error, :missing_options}
+  defp prepare_image_resize_args(_), do: {:error, :missing_options}
 
   defp run_fifo(fifo_path, env, executable, args) do
     args = List.flatten([fifo_path, args, "jpg:-"])
-- 
cgit v1.2.3


From 697bea04731614bcd2e1e10f0564863dc49a49fa Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 26 Aug 2020 17:43:25 -0500
Subject: Move arg for images to the list so we can reuse these fifo functions
 for videos

---
 lib/pleroma/helpers/media_helper.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index fe11dd460..0299b16ae 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -29,7 +29,8 @@ defmodule Pleroma.Helpers.MediaHelper do
     args = [
     "-interlace", "Plane",
     "-resize", resize,
-    "-quality", to_string(quality)
+    "-quality", to_string(quality),
+    "jpg:-"
     ]
     {:ok, args}
   end
@@ -37,7 +38,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   defp prepare_image_resize_args(_), do: {:error, :missing_options}
 
   defp run_fifo(fifo_path, env, executable, args) do
-    args = List.flatten([fifo_path, args, "jpg:-"])
+    args = List.flatten([fifo_path, args])
     pid = Port.open({:spawn_executable, executable}, [:use_stdio, :stream, :exit_status, :binary, args: args])
     fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
     true = Port.command(fifo, env.body)
-- 
cgit v1.2.3


From 78939c1d161f09ac38348fc02e8f4a83d8d82d2d Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 27 Aug 2020 12:13:18 +0200
Subject: ChatController: Don't die if the recipient is gone.

---
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index e8a1746d4..1f2e953f7 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -149,7 +149,9 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
       from(c in Chat,
         where: c.user_id == ^user_id,
         where: c.recipient not in ^blocked_ap_ids,
-        order_by: [desc: c.updated_at]
+        order_by: [desc: c.updated_at],
+        inner_join: u in User,
+        on: u.ap_id == c.recipient
       )
       |> Repo.all()
 
-- 
cgit v1.2.3


From 157ecf402230c0b786f5765dd8b709d45c45974a Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 27 Aug 2020 11:46:56 -0500
Subject: Follow redirects.

I think we should be using some global adapter options here, though.
---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index d465ce8d1..736b7db56 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -50,7 +50,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   defp handle_preview(conn, url) do
     with {:ok, %{status: status} = head_response} when status in 200..299 <-
-           Tesla.head(url, opts: [adapter: [timeout: preview_head_request_timeout()]]) do
+           Tesla.head(url,
+             opts: [adapter: [timeout: preview_head_request_timeout(), follow_redirect: true]]
+           ) do
       content_type = Tesla.get_header(head_response, "content-type")
       handle_preview(content_type, conn, url)
     else
-- 
cgit v1.2.3


From ef9d12fcc500d7429bee0d6ccffe3596434aee52 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 27 Aug 2020 12:31:55 -0500
Subject: Attempt at supporting video thumbnails via ffmpeg

---
 lib/pleroma/helpers/media_helper.ex                   | 19 +++++++++++++++++++
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 17 ++++++++++++++---
 2 files changed, 33 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 0299b16ae..7e1af8bac 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -37,6 +37,25 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   defp prepare_image_resize_args(_), do: {:error, :missing_options}
 
+  def video_framegrab(url) do
+    with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
+         args = [
+           "-i", "-",
+           "-vframes", "1",
+           "-f", "mjpeg",
+           "-loglevel", "error",
+           "-"
+         ],
+         url = Pleroma.Web.MediaProxy.url(url),
+         {:ok, env} <- Pleroma.HTTP.get(url),
+         {:ok, fifo_path} <- mkfifo() do
+      run_fifo(fifo_path, env, executable, args)
+    else
+      nil -> {:error, {:ffmpeg, :command_not_found}}
+      {:error, _} = error -> error
+    end
+  end
+
   defp run_fifo(fifo_path, env, executable, args) do
     args = List.flatten([fifo_path, args])
     pid = Port.open({:spawn_executable, executable}, [:use_stdio, :stream, :exit_status, :binary, args: args])
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 736b7db56..7ac1a97e2 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -78,9 +78,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   end
 
   defp handle_preview("video/" <> _ = _content_type, conn, url) do
-    mediaproxy_url = url |> MediaProxy.url()
-
-    redirect(conn, external: mediaproxy_url)
+    handle_video_preview(conn, url)
   end
 
   defp handle_preview(content_type, conn, _url) do
@@ -106,6 +104,19 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
+  defp handle_video_preview(conn, url) do
+    with {:ok, thumbnail_binary} <-
+           MediaHelper.video_framegrab(url) do
+      conn
+      |> put_resp_header("content-type", "image/jpeg")
+      |> put_resp_header("content-disposition", "inline; filename=\"preview.jpg\"")
+      |> send_resp(200, thumbnail_binary)
+    else
+      _ ->
+        send_resp(conn, :failed_dependency, "Can't handle preview.")
+    end
+  end
+
   defp thumbnail_max_dimensions(params) do
     config = Config.get([:media_preview_proxy], [])
 
-- 
cgit v1.2.3


From f1218a2b4e16178c8c1285157f7cd995dc950e3e Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 27 Aug 2020 12:47:29 -0500
Subject: ffmpeg needs input from fifo path, not stdin

---
 lib/pleroma/helpers/media_helper.ex | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 7e1af8bac..7c2bfbc53 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -39,16 +39,16 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
+         url = Pleroma.Web.MediaProxy.url(url),
+         {:ok, env} <- Pleroma.HTTP.get(url),
+         {:ok, fifo_path} <- mkfifo(),
          args = [
-           "-i", "-",
+           "-i", fifo_path,
            "-vframes", "1",
            "-f", "mjpeg",
            "-loglevel", "error",
            "-"
-         ],
-         url = Pleroma.Web.MediaProxy.url(url),
-         {:ok, env} <- Pleroma.HTTP.get(url),
-         {:ok, fifo_path} <- mkfifo() do
+         ] do
       run_fifo(fifo_path, env, executable, args)
     else
       nil -> {:error, {:ffmpeg, :command_not_found}}
@@ -57,7 +57,12 @@ defmodule Pleroma.Helpers.MediaHelper do
   end
 
   defp run_fifo(fifo_path, env, executable, args) do
-    args = List.flatten([fifo_path, args])
+    args =
+      if _executable = System.find_executable("convert") do
+        List.flatten([fifo_path, args])
+      else
+        args
+      end
     pid = Port.open({:spawn_executable, executable}, [:use_stdio, :stream, :exit_status, :binary, args: args])
     fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
     true = Port.command(fifo, env.body)
-- 
cgit v1.2.3


From dd1de994d57e3d9c99bb4e4c7019c696b5153f50 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 27 Aug 2020 13:10:40 -0500
Subject: Try to trick ffmpeg into working with this named pipe

---
 lib/pleroma/helpers/media_helper.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 7c2bfbc53..385a4df81 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -43,11 +43,12 @@ defmodule Pleroma.Helpers.MediaHelper do
          {:ok, env} <- Pleroma.HTTP.get(url),
          {:ok, fifo_path} <- mkfifo(),
          args = [
+           "-y",
            "-i", fifo_path,
            "-vframes", "1",
            "-f", "mjpeg",
            "-loglevel", "error",
-           "-"
+           "pipe:"
          ] do
       run_fifo(fifo_path, env, executable, args)
     else
-- 
cgit v1.2.3


From 3a5231ec8fd0583d7f4bf05378d8bb81096c4f40 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 27 Aug 2020 16:33:37 -0500
Subject: Keep args construction within video/image scopes instead of mangling
 down in fifo town

---
 lib/pleroma/helpers/media_helper.ex | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 385a4df81..b42612ccb 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.Helpers.MediaHelper do
          {:ok, env} <- Pleroma.HTTP.get(url),
          {:ok, fifo_path} <- mkfifo()
     do
+      args = List.flatten([fifo_path, args])
       run_fifo(fifo_path, env, executable, args)
     else
       nil -> {:error, {:convert, :command_not_found}}
@@ -58,12 +59,6 @@ defmodule Pleroma.Helpers.MediaHelper do
   end
 
   defp run_fifo(fifo_path, env, executable, args) do
-    args =
-      if _executable = System.find_executable("convert") do
-        List.flatten([fifo_path, args])
-      else
-        args
-      end
     pid = Port.open({:spawn_executable, executable}, [:use_stdio, :stream, :exit_status, :binary, args: args])
     fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
     true = Port.command(fifo, env.body)
-- 
cgit v1.2.3


From 67c79394e81cf9f5404afad29a397acf32dece33 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 27 Aug 2020 17:15:23 -0500
Subject: Support static avatars and header images with Mediaproxy Preview

---
 lib/pleroma/web/mastodon_api/views/account_view.ex | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 864c0417f..eef45b35d 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -181,8 +181,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     user = User.sanitize_html(user, User.html_filter_policy(opts[:for]))
     display_name = user.name || user.nickname
 
-    image = User.avatar_url(user) |> MediaProxy.url()
+    avatar = User.avatar_url(user) |> MediaProxy.url()
+    avatar_static = User.avatar_url(user) |> MediaProxy.preview_url()
     header = User.banner_url(user) |> MediaProxy.url()
+    header_static = User.banner_url(user) |> MediaProxy.preview_url()
 
     following_count =
       if !user.hide_follows_count or !user.hide_follows or opts[:for] == user do
@@ -247,10 +249,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       statuses_count: user.note_count,
       note: user.bio || "",
       url: user.uri || user.ap_id,
-      avatar: image,
-      avatar_static: image,
+      avatar: avatar,
+      avatar_static: avatar_static,
       header: header,
-      header_static: header,
+      header_static: header_static,
       emojis: emojis,
       fields: user.fields,
       bot: bot,
-- 
cgit v1.2.3


From 5b4d483f522f470b9d2cdb7f43d98dde427a1241 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 27 Aug 2020 17:28:21 -0500
Subject: Add a note about the avatars and banners situation

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 7ac1a97e2..411dc95d0 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -67,6 +67,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
+  # TODO: find a workaround so avatar_static and banner_static can work.
+  # Those only permit GIFs for animation, so we have to permit a way to
+  # allow those to get real static variants.
   defp handle_preview("image/gif" = _content_type, conn, url) do
     mediaproxy_url = url |> MediaProxy.url()
 
-- 
cgit v1.2.3


From f0fefc4f5c3aa4fa62f2edee72ee864a16e7176d Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Fri, 28 Aug 2020 18:17:44 +0300
Subject: marks notifications as read after mute

---
 lib/pleroma/notification.ex              | 12 ++++++++++++
 lib/pleroma/web/common_api/common_api.ex |  3 ++-
 2 files changed, 14 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index c1825f810..b952e81fa 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -648,4 +648,16 @@ defmodule Pleroma.Notification do
     )
     |> Repo.one()
   end
+
+  @spec mark_as_read(User.t(), Activity.t()) :: {integer(), nil | [term()]}
+  def mark_as_read(%User{id: id}, %Activity{data: %{"context" => context}}) do
+    from(
+      n in Notification,
+      join: a in assoc(n, :activity),
+      where: n.user_id == ^id,
+      where: n.seen == false,
+      where: fragment("?->>'context'", a.data) == ^context
+    )
+    |> Repo.update_all(set: [seen: true])
+  end
 end
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 5ad2b91c2..43e9e39a8 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -452,7 +452,8 @@ defmodule Pleroma.Web.CommonAPI do
   end
 
   def add_mute(user, activity) do
-    with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]) do
+    with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]),
+         _ <- Pleroma.Notification.mark_as_read(user, activity) do
       {:ok, activity}
     else
       {:error, _} -> {:error, dgettext("errors", "conversation is already muted")}
-- 
cgit v1.2.3


From dfceb03cf47374fdeab60784476b2e266208a4bb Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Fri, 28 Aug 2020 21:14:28 +0200
Subject: Rewrite MP4/MOV binaries to be faststart

In some cases, MP4/MOV files can have the data _before_ the meta-data.

Thus, ffmpeg (and all similar tools) cannot really process the input if
it's given over stdin/streaming/pipes.

BUT I REALLY DON'T WANT TO MAKE TEMPORARY FILES

so here we go, an implementation of qtfaststart in elixir.
---
 lib/pleroma/helpers/media_helper.ex  |  59 +++++++++++-----
 lib/pleroma/helpers/qt_fast_start.ex | 131 +++++++++++++++++++++++++++++++++++
 2 files changed, 172 insertions(+), 18 deletions(-)
 create mode 100644 lib/pleroma/helpers/qt_fast_start.ex

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index b42612ccb..5ac75b326 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -14,8 +14,7 @@ defmodule Pleroma.Helpers.MediaHelper do
          {:ok, args} <- prepare_image_resize_args(options),
          url = Pleroma.Web.MediaProxy.url(url),
          {:ok, env} <- Pleroma.HTTP.get(url),
-         {:ok, fifo_path} <- mkfifo()
-    do
+         {:ok, fifo_path} <- mkfifo() do
       args = List.flatten([fifo_path, args])
       run_fifo(fifo_path, env, executable, args)
     else
@@ -27,12 +26,17 @@ defmodule Pleroma.Helpers.MediaHelper do
   defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do
     quality = options[:quality] || 85
     resize = Enum.join([max_width, "x", max_height, ">"])
+
     args = [
-    "-interlace", "Plane",
-    "-resize", resize,
-    "-quality", to_string(quality),
-    "jpg:-"
+      "-interlace",
+      "Plane",
+      "-resize",
+      resize,
+      "-quality",
+      to_string(quality),
+      "jpg:-"
     ]
+
     {:ok, args}
   end
 
@@ -45,11 +49,15 @@ defmodule Pleroma.Helpers.MediaHelper do
          {:ok, fifo_path} <- mkfifo(),
          args = [
            "-y",
-           "-i", fifo_path,
-           "-vframes", "1",
-           "-f", "mjpeg",
-           "-loglevel", "error",
-           "pipe:"
+           "-i",
+           fifo_path,
+           "-vframes",
+           "1",
+           "-f",
+           "mjpeg",
+           "-loglevel",
+           "error",
+           "-"
          ] do
       run_fifo(fifo_path, env, executable, args)
     else
@@ -59,9 +67,18 @@ defmodule Pleroma.Helpers.MediaHelper do
   end
 
   defp run_fifo(fifo_path, env, executable, args) do
-    pid = Port.open({:spawn_executable, executable}, [:use_stdio, :stream, :exit_status, :binary, args: args])
+    pid =
+      Port.open({:spawn_executable, executable}, [
+        :use_stdio,
+        :stream,
+        :exit_status,
+        :binary,
+        args: args
+      ])
+
     fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out])
-    true = Port.command(fifo, env.body)
+    fix = Pleroma.Helpers.QtFastStart.fix(env.body)
+    true = Port.command(fifo, fix)
     :erlang.port_close(fifo)
     loop_recv(pid)
   after
@@ -70,10 +87,12 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   defp mkfifo() do
     path = "#{@tmp_base}#{to_charlist(:erlang.phash2(self()))}"
+
     case System.cmd("mkfifo", [path]) do
       {_, 0} ->
         spawn(fifo_guard(path))
         {:ok, path}
+
       {_, err} ->
         {:error, {:fifo_failed, err}}
     end
@@ -81,8 +100,10 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   defp fifo_guard(path) do
     pid = self()
-    fn() ->
+
+    fn ->
       ref = Process.monitor(pid)
+
       receive do
         {:DOWN, ^ref, :process, ^pid, _} ->
           File.rm(path)
@@ -98,14 +119,16 @@ defmodule Pleroma.Helpers.MediaHelper do
     receive do
       {^pid, {:data, data}} ->
         loop_recv(pid, acc <> data)
+
       {^pid, {:exit_status, 0}} ->
         {:ok, acc}
+
       {^pid, {:exit_status, status}} ->
         {:error, status}
-     after
-       5000 ->
-         :erlang.port_close(pid)
-         {:error, :timeout}
+    after
+      5000 ->
+        :erlang.port_close(pid)
+        {:error, :timeout}
     end
   end
 end
diff --git a/lib/pleroma/helpers/qt_fast_start.ex b/lib/pleroma/helpers/qt_fast_start.ex
new file mode 100644
index 000000000..694b583b9
--- /dev/null
+++ b/lib/pleroma/helpers/qt_fast_start.ex
@@ -0,0 +1,131 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Helpers.QtFastStart do
+  @moduledoc """
+  (WIP) Converts a "slow start" (data before metadatas) mov/mp4 file to a "fast start" one (metadatas before data).
+  """
+
+  # TODO: Cleanup and optimizations
+  # Inspirations: https://www.ffmpeg.org/doxygen/3.4/qt-faststart_8c_source.html
+  #               https://github.com/danielgtaylor/qtfaststart/blob/master/qtfaststart/processor.py
+  #               ISO/IEC 14496-12:2015, ISO/IEC 15444-12:2015
+  #               Paracetamol
+
+  def fix(binary = <<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::binary>>) do
+    index = fix(binary, binary, 0, [])
+
+    case index do
+      [{"ftyp", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
+      [{"ftyp", _, _, _, _}, {"free", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
+      _ -> binary
+    end
+  end
+
+  def fix(binary) do
+    binary
+  end
+
+  defp fix(<<>>, _bin, _pos, acc) do
+    :lists.reverse(acc)
+  end
+
+  defp fix(
+         <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), rest::binary>>,
+         bin,
+         pos,
+         acc
+       ) do
+    if fourcc == "mdat" && size == 0 do
+      # mdat with 0 size means "seek to the end" -- also, in that case the file is probably OK.
+      acc = [
+        {fourcc, pos, byte_size(bin) - pos, byte_size(bin) - pos,
+         <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), rest::binary>>}
+        | acc
+      ]
+
+      fix(<<>>, bin, byte_size(bin), acc)
+    else
+      full_size = size - 8
+      <<data::binary-size(full_size), rest::binary>> = rest
+
+      acc = [
+        {fourcc, pos, pos + size, size,
+         <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), data::binary>>}
+        | acc
+      ]
+
+      fix(rest, bin, pos + size, acc)
+    end
+  end
+
+  defp faststart(index) do
+    {{_ftyp, _, _, _, ftyp}, index} = List.keytake(index, "ftyp", 0)
+
+    # Skip re-writing the free fourcc as it's kind of useless. Why stream useless bytes when you can do without?
+    {free_size, index} =
+      case List.keytake(index, "free", 0) do
+        {{_, _, _, size, _}, index} -> {size, index}
+        _ -> {0, index}
+      end
+
+    {{_moov, _, _, moov_size, moov}, index} = List.keytake(index, "moov", 0)
+    offset = -free_size + moov_size
+    rest = for {_, _, _, _, data} <- index, do: data, into: <<>>
+    <<moov_head::binary-size(8), moov_data::binary>> = moov
+    new_moov = fix_moov(moov_data, offset)
+    <<ftyp::binary, moov_head::binary, new_moov::binary, rest::binary>>
+  end
+
+  defp fix_moov(moov, offset) do
+    fix_moov(moov, offset, <<>>)
+  end
+
+  defp fix_moov(<<>>, _, acc), do: acc
+
+  defp fix_moov(
+         <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), rest::binary>>,
+         offset,
+         acc
+       ) do
+    full_size = size - 8
+    <<data::binary-size(full_size), rest::binary>> = rest
+
+    data =
+      cond do
+        fourcc in ["trak", "mdia", "minf", "stbl"] ->
+          # Theses contains sto or co64 part
+          <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4),
+            fix_moov(data, offset, <<>>)::binary>>
+
+        fourcc in ["stco", "co64"] ->
+          # fix the damn thing
+          <<version::integer-big-size(4)-unit(8), count::integer-big-size(4)-unit(8),
+            rest::binary>> = data
+
+          entry_size =
+            case fourcc do
+              "stco" -> 4
+              "co64" -> 8
+            end
+
+          {_, result} =
+            Enum.reduce(1..count, {rest, <<>>}, fn _,
+                                                   {<<pos::integer-big-size(entry_size)-unit(8),
+                                                      rest::binary>>, acc} ->
+              {rest, <<acc::binary, pos + offset::integer-big-size(entry_size)-unit(8)>>}
+            end)
+
+          <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4),
+            version::integer-big-size(4)-unit(8), count::integer-big-size(4)-unit(8),
+            result::binary>>
+
+        true ->
+          <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), data::binary>>
+      end
+
+    acc = <<acc::binary, data::binary>>
+    fix_moov(rest, offset, acc)
+  end
+end
-- 
cgit v1.2.3


From 24d522c3b366b54b23bebaf07371145d50820d4a Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Sat, 29 Aug 2020 13:05:23 +0200
Subject: QtFastStart: optimize

~4-6x faster
~3~4x memory usage reduction (now mostly adds what we are rewriting in
the metadatas)
---
 lib/pleroma/helpers/qt_fast_start.ex | 115 +++++++++++++++++------------------
 1 file changed, 57 insertions(+), 58 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/qt_fast_start.ex b/lib/pleroma/helpers/qt_fast_start.ex
index 694b583b9..8cba06e54 100644
--- a/lib/pleroma/helpers/qt_fast_start.ex
+++ b/lib/pleroma/helpers/qt_fast_start.ex
@@ -13,10 +13,11 @@ defmodule Pleroma.Helpers.QtFastStart do
   #               ISO/IEC 14496-12:2015, ISO/IEC 15444-12:2015
   #               Paracetamol
 
-  def fix(binary = <<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::binary>>) do
-    index = fix(binary, binary, 0, [])
+  def fix(binary = <<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::bits>>) do
+    index = fix(binary, 0, nil, nil, [])
 
     case index do
+      :abort -> binary
       [{"ftyp", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
       [{"ftyp", _, _, _, _}, {"free", _, _, _, _}, {"mdat", _, _, _, _} | _] -> faststart(index)
       _ -> binary
@@ -27,37 +28,32 @@ defmodule Pleroma.Helpers.QtFastStart do
     binary
   end
 
-  defp fix(<<>>, _bin, _pos, acc) do
-    :lists.reverse(acc)
+  # MOOV have been seen before MDAT- abort
+  defp fix(<<_::bits>>, _, true, false, _) do
+    :abort
   end
 
   defp fix(
-         <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), rest::binary>>,
-         bin,
+         <<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
          pos,
+         got_moov,
+         got_mdat,
          acc
        ) do
-    if fourcc == "mdat" && size == 0 do
-      # mdat with 0 size means "seek to the end" -- also, in that case the file is probably OK.
-      acc = [
-        {fourcc, pos, byte_size(bin) - pos, byte_size(bin) - pos,
-         <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), rest::binary>>}
-        | acc
-      ]
-
-      fix(<<>>, bin, byte_size(bin), acc)
-    else
-      full_size = size - 8
-      <<data::binary-size(full_size), rest::binary>> = rest
-
-      acc = [
-        {fourcc, pos, pos + size, size,
-         <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), data::binary>>}
-        | acc
-      ]
-
-      fix(rest, bin, pos + size, acc)
-    end
+    full_size = (size - 8) * 8
+    <<data::bits-size(full_size), rest::bits>> = rest
+
+    acc = [
+      {fourcc, pos, pos + size, size,
+       <<size::integer-big-size(32), fourcc::bits-size(32), data::bits>>}
+      | acc
+    ]
+
+    fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc)
+  end
+
+  defp fix(<<>>, _pos, _, _, acc) do
+    :lists.reverse(acc)
   end
 
   defp faststart(index) do
@@ -72,60 +68,63 @@ defmodule Pleroma.Helpers.QtFastStart do
 
     {{_moov, _, _, moov_size, moov}, index} = List.keytake(index, "moov", 0)
     offset = -free_size + moov_size
-    rest = for {_, _, _, _, data} <- index, do: data, into: <<>>
-    <<moov_head::binary-size(8), moov_data::binary>> = moov
-    new_moov = fix_moov(moov_data, offset)
-    <<ftyp::binary, moov_head::binary, new_moov::binary, rest::binary>>
-  end
-
-  defp fix_moov(moov, offset) do
-    fix_moov(moov, offset, <<>>)
+    rest = for {_, _, _, _, data} <- index, do: data, into: []
+    <<moov_head::bits-size(64), moov_data::bits>> = moov
+    [ftyp, moov_head, fix_moov(moov_data, offset, []), rest]
   end
 
-  defp fix_moov(<<>>, _, acc), do: acc
-
   defp fix_moov(
-         <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), rest::binary>>,
+         <<size::integer-big-size(32), fourcc::bits-size(32), rest::bits>>,
          offset,
          acc
        ) do
-    full_size = size - 8
-    <<data::binary-size(full_size), rest::binary>> = rest
+    full_size = (size - 8) * 8
+    <<data::bits-size(full_size), rest::bits>> = rest
 
     data =
       cond do
         fourcc in ["trak", "mdia", "minf", "stbl"] ->
           # Theses contains sto or co64 part
-          <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4),
-            fix_moov(data, offset, <<>>)::binary>>
+          [<<size::integer-big-size(32), fourcc::bits-size(32)>>, fix_moov(data, offset, [])]
 
         fourcc in ["stco", "co64"] ->
           # fix the damn thing
-          <<version::integer-big-size(4)-unit(8), count::integer-big-size(4)-unit(8),
-            rest::binary>> = data
+          <<version::integer-big-size(32), count::integer-big-size(32), rest::bits>> = data
 
           entry_size =
             case fourcc do
-              "stco" -> 4
-              "co64" -> 8
+              "stco" -> 32
+              "co64" -> 64
             end
 
-          {_, result} =
-            Enum.reduce(1..count, {rest, <<>>}, fn _,
-                                                   {<<pos::integer-big-size(entry_size)-unit(8),
-                                                      rest::binary>>, acc} ->
-              {rest, <<acc::binary, pos + offset::integer-big-size(entry_size)-unit(8)>>}
-            end)
-
-          <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4),
-            version::integer-big-size(4)-unit(8), count::integer-big-size(4)-unit(8),
-            result::binary>>
+          [
+            <<size::integer-big-size(32), fourcc::bits-size(32), version::integer-big-size(32),
+              count::integer-big-size(32)>>,
+            rewrite_entries(entry_size, offset, rest, [])
+          ]
 
         true ->
-          <<size::integer-big-size(4)-unit(8), fourcc::binary-size(4), data::binary>>
+          [<<size::integer-big-size(32), fourcc::bits-size(32)>>, data]
       end
 
-    acc = <<acc::binary, data::binary>>
+    acc = [acc | data]
     fix_moov(rest, offset, acc)
   end
+
+  defp fix_moov(<<>>, _, acc), do: acc
+
+  for size <- [32, 64] do
+    defp rewrite_entries(
+           unquote(size),
+           offset,
+           <<pos::integer-big-size(unquote(size)), rest::bits>>,
+           acc
+         ) do
+      rewrite_entries(unquote(size), offset, rest, [
+        acc | <<pos + offset::integer-big-size(unquote(size))>>
+      ])
+    end
+  end
+
+  defp rewrite_entries(_, _, <<>>, acc), do: acc
 end
-- 
cgit v1.2.3


From 2d2af75777ae468fb08a2b09dc5af4636106a04b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Sun, 30 Aug 2020 09:17:24 -0500
Subject: Support PNG previews to preserve alpha channels

---
 lib/pleroma/helpers/media_helper.ex                | 17 ++++++++++++
 .../web/media_proxy/media_proxy_controller.ex      | 32 ++++++++++++++++++++--
 2 files changed, 47 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 5ac75b326..d8a6db4e1 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -23,6 +23,23 @@ defmodule Pleroma.Helpers.MediaHelper do
     end
   end
 
+  defp prepare_image_resize_args(
+         %{max_width: max_width, max_height: max_height, format: "png"} = options
+       ) do
+    quality = options[:quality] || 85
+    resize = Enum.join([max_width, "x", max_height, ">"])
+
+    args = [
+      "-resize",
+      resize,
+      "-quality",
+      to_string(quality),
+      "png:-"
+    ]
+
+    {:ok, args}
+  end
+
   defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do
     quality = options[:quality] || 85
     resize = Enum.join([max_width, "x", max_height, ">"])
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 411dc95d0..94fae6cac 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -76,8 +76,12 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     redirect(conn, external: mediaproxy_url)
   end
 
+  defp handle_preview("image/png" <> _ = _content_type, conn, url) do
+    handle_png_preview(conn, url)
+  end
+
   defp handle_preview("image/" <> _ = _content_type, conn, url) do
-    handle_image_preview(conn, url)
+    handle_jpeg_preview(conn, url)
   end
 
   defp handle_preview("video/" <> _ = _content_type, conn, url) do
@@ -88,7 +92,31 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
   end
 
-  defp handle_image_preview(%{params: params} = conn, url) do
+  defp handle_png_preview(%{params: params} = conn, url) do
+    quality = Config.get!([:media_preview_proxy, :image_quality])
+
+    with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
+         {:ok, thumbnail_binary} <-
+           MediaHelper.image_resize(
+             url,
+             %{
+               max_width: thumbnail_max_width,
+               max_height: thumbnail_max_height,
+               quality: quality,
+               format: "png"
+             }
+           ) do
+      conn
+      |> put_resp_header("content-type", "image/png")
+      |> put_resp_header("content-disposition", "inline; filename=\"preview.png\"")
+      |> send_resp(200, thumbnail_binary)
+    else
+      _ ->
+        send_resp(conn, :failed_dependency, "Can't handle preview.")
+    end
+  end
+
+  defp handle_jpeg_preview(%{params: params} = conn, url) do
     quality = Config.get!([:media_preview_proxy, :image_quality])
 
     with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
-- 
cgit v1.2.3


From 4ef210a587113313cd6887b7499832d0c0798f7f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Sun, 30 Aug 2020 09:32:22 -0500
Subject: Credo

---
 lib/pleroma/helpers/media_helper.ex  | 2 +-
 lib/pleroma/helpers/qt_fast_start.ex | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index d8a6db4e1..9bd815c26 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -102,7 +102,7 @@ defmodule Pleroma.Helpers.MediaHelper do
     File.rm(fifo_path)
   end
 
-  defp mkfifo() do
+  defp mkfifo do
     path = "#{@tmp_base}#{to_charlist(:erlang.phash2(self()))}"
 
     case System.cmd("mkfifo", [path]) do
diff --git a/lib/pleroma/helpers/qt_fast_start.ex b/lib/pleroma/helpers/qt_fast_start.ex
index 8cba06e54..bb93224b5 100644
--- a/lib/pleroma/helpers/qt_fast_start.ex
+++ b/lib/pleroma/helpers/qt_fast_start.ex
@@ -13,7 +13,7 @@ defmodule Pleroma.Helpers.QtFastStart do
   #               ISO/IEC 14496-12:2015, ISO/IEC 15444-12:2015
   #               Paracetamol
 
-  def fix(binary = <<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::bits>>) do
+  def fix(<<0x00, 0x00, 0x00, _, 0x66, 0x74, 0x79, 0x70, _::bits>> = binary) do
     index = fix(binary, 0, nil, nil, [])
 
     case index do
@@ -59,7 +59,8 @@ defmodule Pleroma.Helpers.QtFastStart do
   defp faststart(index) do
     {{_ftyp, _, _, _, ftyp}, index} = List.keytake(index, "ftyp", 0)
 
-    # Skip re-writing the free fourcc as it's kind of useless. Why stream useless bytes when you can do without?
+    # Skip re-writing the free fourcc as it's kind of useless.
+    # Why stream useless bytes when you can do without?
     {free_size, index} =
       case List.keytake(index, "free", 0) do
         {{_, _, _, size, _}, index} -> {size, index}
-- 
cgit v1.2.3


From d91c4feebeb199f7c584f0a4292ce6f9cc331798 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 31 Aug 2020 11:02:54 +0200
Subject: Notification: Small refactor.

---
 lib/pleroma/notification.ex              | 4 ++--
 lib/pleroma/web/common_api/common_api.ex | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index b952e81fa..8868a910e 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -649,8 +649,8 @@ defmodule Pleroma.Notification do
     |> Repo.one()
   end
 
-  @spec mark_as_read(User.t(), Activity.t()) :: {integer(), nil | [term()]}
-  def mark_as_read(%User{id: id}, %Activity{data: %{"context" => context}}) do
+  @spec mark_context_as_read(User.t(), String.t()) :: {integer(), nil | [term()]}
+  def mark_context_as_read(%User{id: id}, context) do
     from(
       n in Notification,
       join: a in assoc(n, :activity),
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 43e9e39a8..4ab533658 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -453,7 +453,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def add_mute(user, activity) do
     with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]),
-         _ <- Pleroma.Notification.mark_as_read(user, activity) do
+         _ <- Pleroma.Notification.mark_context_as_read(user, activity.data["context"]) do
       {:ok, activity}
     else
       {:error, _} -> {:error, dgettext("errors", "conversation is already muted")}
-- 
cgit v1.2.3


From 0a839d51a7adb034d6514ea647d90546c829813d Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Mon, 31 Aug 2020 13:08:50 +0300
Subject: [#2497] Added Cache-Control response header for media proxy preview
 endpoint.

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 94fae6cac..2afcd861a 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -107,8 +107,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
              }
            ) do
       conn
-      |> put_resp_header("content-type", "image/png")
-      |> put_resp_header("content-disposition", "inline; filename=\"preview.png\"")
+      |> put_preview_response_headers()
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
@@ -126,8 +125,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
              %{max_width: thumbnail_max_width, max_height: thumbnail_max_height, quality: quality}
            ) do
       conn
-      |> put_resp_header("content-type", "image/jpeg")
-      |> put_resp_header("content-disposition", "inline; filename=\"preview.jpg\"")
+      |> put_preview_response_headers()
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
@@ -139,8 +137,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     with {:ok, thumbnail_binary} <-
            MediaHelper.video_framegrab(url) do
       conn
-      |> put_resp_header("content-type", "image/jpeg")
-      |> put_resp_header("content-disposition", "inline; filename=\"preview.jpg\"")
+      |> put_preview_response_headers()
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
@@ -148,6 +145,13 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
+  defp put_preview_response_headers(conn) do
+    conn
+    |> put_resp_header("content-type", "image/jpeg")
+    |> put_resp_header("content-disposition", "inline; filename=\"preview.jpg\"")
+    |> put_resp_header("cache-control", "max-age=0, private, must-revalidate")
+  end
+
   defp thumbnail_max_dimensions(params) do
     config = Config.get([:media_preview_proxy], [])
 
-- 
cgit v1.2.3


From 0b621a834acf751332f4d202bd50d4ff3e789176 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 31 Aug 2020 16:48:17 +0200
Subject: Chats: Add cascading delete on both referenced users.

Also remove the now-superfluous join in the chat controller,
which was only used to filter out these cases.
---
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 1f2e953f7..e8a1746d4 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -149,9 +149,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
       from(c in Chat,
         where: c.user_id == ^user_id,
         where: c.recipient not in ^blocked_ap_ids,
-        order_by: [desc: c.updated_at],
-        inner_join: u in User,
-        on: u.ap_id == c.recipient
+        order_by: [desc: c.updated_at]
       )
       |> Repo.all()
 
-- 
cgit v1.2.3


From dc3a418c270e48c6849159097f8bba57d2a2578e Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 1 Sep 2020 09:08:54 +0300
Subject: application.ex: disable warnings_as_errors at runtime

see changed files for rationale
---
 lib/pleroma/application.ex | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index c0b5db9f1..005aba50a 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -39,6 +39,9 @@ defmodule Pleroma.Application do
     # every time the application is restarted, so we disable module
     # conflicts at runtime
     Code.compiler_options(ignore_module_conflict: true)
+    # Disable warnings_as_errors at runtime, it breaks Phoenix live reload
+    # due to protocol consolidation warnings
+    Code.compiler_options(warnings_as_errors: false)
     Pleroma.Telemetry.Logger.attach()
     Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
-- 
cgit v1.2.3


From 2ecc7d92308d624dc9edb50665d752a71f55f608 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 25 Aug 2020 00:38:10 +0200
Subject: transmogrifier: Remove mastodon emoji-format from emoji field

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 3 ---
 1 file changed, 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 76298c4a0..0831efadc 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -318,9 +318,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         Map.put(mapping, name, data["icon"]["url"])
       end)
 
-    # we merge mastodon and pleroma emoji into a single mapping, to allow for both wire formats
-    emoji = Map.merge(object["emoji"] || %{}, emoji)
-
     Map.put(object, "emoji", emoji)
   end
 
-- 
cgit v1.2.3


From a142da3e4f03f2bfee7af30cca59b0fdc82da73f Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 25 Aug 2020 01:16:12 +0200
Subject: Add new Emoji Ecto.Type and fix emoji in Question

---
 .../activity_pub/object_validators/emoji.ex        | 34 ++++++++++++++++++++++
 .../object_validators/audio_validator.ex           |  5 ++--
 .../object_validators/chat_message_validator.ex    |  2 +-
 .../object_validators/event_validator.ex           |  5 ++--
 .../object_validators/note_validator.ex            | 11 +++++--
 .../object_validators/question_validator.ex        |  5 ++--
 6 files changed, 53 insertions(+), 9 deletions(-)
 create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex

(limited to 'lib')

diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex
new file mode 100644
index 000000000..4aacc5c88
--- /dev/null
+++ b/lib/pleroma/ecto_type/activity_pub/object_validators/emoji.ex
@@ -0,0 +1,34 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Emoji do
+  use Ecto.Type
+
+  def type, do: :map
+
+  def cast(data) when is_map(data) do
+    has_invalid_emoji? =
+      Enum.find(data, fn
+        {name, uri} when is_binary(name) and is_binary(uri) ->
+          # based on ObjectValidators.Uri.cast()
+          case URI.parse(uri) do
+            %URI{host: nil} -> true
+            %URI{host: ""} -> true
+            %URI{scheme: scheme} when scheme in ["https", "http"] -> false
+            _ -> true
+          end
+
+        {_name, _uri} ->
+          true
+      end)
+
+    if has_invalid_emoji?, do: :error, else: {:ok, data}
+  end
+
+  def cast(_data), do: :error
+
+  def dump(data), do: {:ok, data}
+
+  def load(data), do: {:ok, data}
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
index d1869f188..1a97c504a 100644
--- a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
 
@@ -33,8 +34,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     field(:attributedTo, ObjectValidators.ObjectID)
     field(:summary, :string)
     field(:published, ObjectValidators.DateTime)
-    # TODO: Write type
-    field(:emoji, :map, default: %{})
+    field(:emoji, ObjectValidators.Emoji, default: %{})
     field(:sensitive, :boolean, default: false)
     embeds_many(:attachment, AttachmentValidator)
     field(:replies_count, :integer, default: 0)
@@ -83,6 +83,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
     data
     |> CommonFixes.fix_defaults()
     |> CommonFixes.fix_attribution()
+    |> Transmogrifier.fix_emoji()
     |> fix_url()
   end
 
diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
index 91b475393..6acd4a771 100644
--- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex
@@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do
     field(:content, ObjectValidators.SafeText)
     field(:actor, ObjectValidators.ObjectID)
     field(:published, ObjectValidators.DateTime)
-    field(:emoji, :map, default: %{})
+    field(:emoji, ObjectValidators.Emoji, default: %{})
 
     embeds_one(:attachment, AttachmentValidator)
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex
index 07e4821a4..0b4c99dc0 100644
--- a/lib/pleroma/web/activity_pub/object_validators/event_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/event_validator.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
 
@@ -39,8 +40,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
 
     field(:attributedTo, ObjectValidators.ObjectID)
     field(:published, ObjectValidators.DateTime)
-    # TODO: Write type
-    field(:emoji, :map, default: %{})
+    field(:emoji, ObjectValidators.Emoji, default: %{})
     field(:sensitive, :boolean, default: false)
     embeds_many(:attachment, AttachmentValidator)
     field(:replies_count, :integer, default: 0)
@@ -74,6 +74,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EventValidator do
     data
     |> CommonFixes.fix_defaults()
     |> CommonFixes.fix_attribution()
+    |> Transmogrifier.fix_emoji()
   end
 
   def changeset(struct, data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
index 20e735619..ab4469a59 100644
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
@@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
   use Ecto.Schema
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
 
@@ -32,8 +33,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:actor, ObjectValidators.ObjectID)
     field(:attributedTo, ObjectValidators.ObjectID)
     field(:published, ObjectValidators.DateTime)
-    # TODO: Write type
-    field(:emoji, :map, default: %{})
+    field(:emoji, ObjectValidators.Emoji, default: %{})
     field(:sensitive, :boolean, default: false)
     # TODO: Write type
     field(:attachment, {:array, :map}, default: [])
@@ -53,7 +53,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     |> validate_data()
   end
 
+  defp fix(data) do
+    data
+    |> Transmogrifier.fix_emoji()
+  end
+
   def cast_data(data) do
+    data = fix(data)
+
     %__MODULE__{}
     |> cast(data, __schema__(:fields))
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 712047424..934d3c1ea 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
   alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
   alias Pleroma.Web.ActivityPub.ObjectValidators.QuestionOptionsValidator
+  alias Pleroma.Web.ActivityPub.Transmogrifier
 
   import Ecto.Changeset
 
@@ -35,8 +36,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     field(:attributedTo, ObjectValidators.ObjectID)
     field(:summary, :string)
     field(:published, ObjectValidators.DateTime)
-    # TODO: Write type
-    field(:emoji, :map, default: %{})
+    field(:emoji, ObjectValidators.Emoji, default: %{})
     field(:sensitive, :boolean, default: false)
     embeds_many(:attachment, AttachmentValidator)
     field(:replies_count, :integer, default: 0)
@@ -85,6 +85,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     data
     |> CommonFixes.fix_defaults()
     |> CommonFixes.fix_attribution()
+    |> Transmogrifier.fix_emoji()
     |> fix_closed()
   end
 
-- 
cgit v1.2.3


From b960cede9a3183098ac5eb330fbc4d1c14efc034 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 25 Aug 2020 02:18:33 +0200
Subject: common_fixes: Force inserting context and context_id

---
 lib/pleroma/web/activity_pub/object_validators/common_fixes.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
index 721749de0..720213d73 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
@@ -11,8 +11,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
       Utils.create_context(data["context"] || data["conversation"])
 
     data
-    |> Map.put_new("context", context)
-    |> Map.put_new("context_id", context_id)
+    |> Map.put("context", context)
+    |> Map.put("context_id", context_id)
   end
 
   def fix_attribution(data) do
-- 
cgit v1.2.3


From d9a21e4784a83a0780b353c967520cd203f44f3f Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 25 Aug 2020 02:21:19 +0200
Subject: fetcher: Remove fix_object call for Question activities

---
 lib/pleroma/object/fetcher.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 6fdbc8efd..d26c5adf5 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -36,8 +36,7 @@ defmodule Pleroma.Object.Fetcher do
   defp reinject_object(%Object{data: %{"type" => "Question"}} = object, new_data) do
     Logger.debug("Reinjecting object #{new_data["id"]}")
 
-    with new_data <- Transmogrifier.fix_object(new_data),
-         data <- maybe_reinject_internal_fields(object, new_data),
+    with data <- maybe_reinject_internal_fields(object, new_data),
          {:ok, data, _} <- ObjectValidator.validate(data, %{}),
          changeset <- Object.change(object, %{data: data}),
          changeset <- touch_changeset(changeset),
-- 
cgit v1.2.3


From 126461942b63bbb74900f296ebcee72d2a33f3d2 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 1 Sep 2020 09:25:32 +0300
Subject: User table: ensure bio is always a string

Gets rid of '|| ""' in multiple places and fixes #2067
---
 lib/pleroma/user.ex                                | 4 ++--
 lib/pleroma/web/activity_pub/activity_pub.ex       | 2 +-
 lib/pleroma/web/auth/pleroma_authenticator.ex      | 2 +-
 lib/pleroma/web/mastodon_api/views/account_view.ex | 2 +-
 lib/pleroma/web/metadata/opengraph.ex              | 2 +-
 lib/pleroma/web/metadata/twitter_card.ex           | 2 +-
 6 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d2ad9516f..94c96de8d 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -83,7 +83,7 @@ defmodule Pleroma.User do
   ]
 
   schema "users" do
-    field(:bio, :string)
+    field(:bio, :string, default: "")
     field(:raw_bio, :string)
     field(:email, :string)
     field(:name, :string)
@@ -1587,7 +1587,7 @@ defmodule Pleroma.User do
     # "Right to be forgotten"
     # https://gdpr.eu/right-to-be-forgotten/
     change(user, %{
-      bio: nil,
+      bio: "",
       raw_bio: nil,
       email: nil,
       name: nil,
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 624a508ae..333621413 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1224,7 +1224,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       name: data["name"],
       follower_address: data["followers"],
       following_address: data["following"],
-      bio: data["summary"],
+      bio: data["summary"] || "",
       actor_type: actor_type,
       also_known_as: Map.get(data, "alsoKnownAs", []),
       public_key: public_key,
diff --git a/lib/pleroma/web/auth/pleroma_authenticator.ex b/lib/pleroma/web/auth/pleroma_authenticator.ex
index 200ca03dc..c611b3e09 100644
--- a/lib/pleroma/web/auth/pleroma_authenticator.ex
+++ b/lib/pleroma/web/auth/pleroma_authenticator.ex
@@ -68,7 +68,7 @@ defmodule Pleroma.Web.Auth.PleromaAuthenticator do
     nickname = value([registration_attrs["nickname"], Registration.nickname(registration)])
     email = value([registration_attrs["email"], Registration.email(registration)])
     name = value([registration_attrs["name"], Registration.name(registration)]) || nickname
-    bio = value([registration_attrs["bio"], Registration.description(registration)])
+    bio = value([registration_attrs["bio"], Registration.description(registration)]) || ""
 
     random_password = :crypto.strong_rand_bytes(64) |> Base.encode64()
 
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 864c0417f..d2a30a548 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -245,7 +245,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
       followers_count: followers_count,
       following_count: following_count,
       statuses_count: user.note_count,
-      note: user.bio || "",
+      note: user.bio,
       url: user.uri || user.ap_id,
       avatar: image,
       avatar_static: image,
diff --git a/lib/pleroma/web/metadata/opengraph.ex b/lib/pleroma/web/metadata/opengraph.ex
index 68c871e71..bb1b23208 100644
--- a/lib/pleroma/web/metadata/opengraph.ex
+++ b/lib/pleroma/web/metadata/opengraph.ex
@@ -61,7 +61,7 @@ defmodule Pleroma.Web.Metadata.Providers.OpenGraph do
 
   @impl Provider
   def build_tags(%{user: user}) do
-    with truncated_bio = Utils.scrub_html_and_truncate(user.bio || "") do
+    with truncated_bio = Utils.scrub_html_and_truncate(user.bio) do
       [
         {:meta,
          [
diff --git a/lib/pleroma/web/metadata/twitter_card.ex b/lib/pleroma/web/metadata/twitter_card.ex
index 5d08ce422..df34b033f 100644
--- a/lib/pleroma/web/metadata/twitter_card.ex
+++ b/lib/pleroma/web/metadata/twitter_card.ex
@@ -40,7 +40,7 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do
 
   @impl Provider
   def build_tags(%{user: user}) do
-    with truncated_bio = Utils.scrub_html_and_truncate(user.bio || "") do
+    with truncated_bio = Utils.scrub_html_and_truncate(user.bio) do
       [
         title_tag(user),
         {:meta, [property: "twitter:description", content: truncated_bio], []},
-- 
cgit v1.2.3


From d8728580468ecf876e531440fa31aef6a3e33f7b Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 1 Sep 2020 12:48:39 +0200
Subject: Fix removing an account from a list

Mastodon (Frontend) changed a different method for deletes,
keeping old format as mastodon documentation is too loose
---
 .../web/api_spec/operations/list_operation.ex        | 20 ++++++++++++++------
 .../web/mastodon_api/controllers/list_controller.ex  | 13 +++++++++++++
 2 files changed, 27 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/operations/list_operation.ex b/lib/pleroma/web/api_spec/operations/list_operation.ex
index c88ed5dd0..15039052e 100644
--- a/lib/pleroma/web/api_spec/operations/list_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/list_operation.ex
@@ -114,7 +114,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
       description: "Add accounts to the given list.",
       operationId: "ListController.add_to_list",
       parameters: [id_param()],
-      requestBody: add_remove_accounts_request(),
+      requestBody: add_remove_accounts_request(true),
       security: [%{"oAuth" => ["write:lists"]}],
       responses: %{
         200 => Operation.response("Empty object", "application/json", %Schema{type: :object})
@@ -127,8 +127,16 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
       tags: ["Lists"],
       summary: "Remove accounts from list",
       operationId: "ListController.remove_from_list",
-      parameters: [id_param()],
-      requestBody: add_remove_accounts_request(),
+      parameters: [
+        id_param(),
+        Operation.parameter(
+          :account_ids,
+          :query,
+          %Schema{type: :array, items: %Schema{type: :string}},
+          "Array of account IDs"
+        )
+      ],
+      requestBody: add_remove_accounts_request(false),
       security: [%{"oAuth" => ["write:lists"]}],
       responses: %{
         200 => Operation.response("Empty object", "application/json", %Schema{type: :object})
@@ -171,7 +179,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
     )
   end
 
-  defp add_remove_accounts_request do
+  defp add_remove_accounts_request(required) when is_boolean(required) do
     request_body(
       "Parameters",
       %Schema{
@@ -180,9 +188,9 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
         properties: %{
           account_ids: %Schema{type: :array, description: "Array of account IDs", items: FlakeID}
         },
-        required: [:account_ids]
+        required: required && [:account_ids]
       },
-      required: true
+      required: required
     )
   end
 end
diff --git a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex
index acdc76fd2..4df13cb81 100644
--- a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex
@@ -86,6 +86,19 @@ defmodule Pleroma.Web.MastodonAPI.ListController do
     json(conn, %{})
   end
 
+  def remove_from_list(
+        %{assigns: %{list: list}, params: %{account_ids: account_ids}} = conn,
+        _
+      ) do
+    Enum.each(account_ids, fn account_id ->
+      with %User{} = followed <- User.get_cached_by_id(account_id) do
+        Pleroma.List.unfollow(list, followed)
+      end
+    end)
+
+    json(conn, %{})
+  end
+
   defp list_by_id_and_user(%{assigns: %{user: user}, params: %{id: id}} = conn, _) do
     case Pleroma.List.get(id, user) do
       %Pleroma.List{} = list -> assign(conn, :list, list)
-- 
cgit v1.2.3


From 03d06062ab8feffbf7b03ecb7ff86c4a42af79ef Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 1 Sep 2020 19:12:45 +0300
Subject: don't fail on url fetch

---
 lib/pleroma/web/rich_media/parser.ex               | 44 ++++++++++++----------
 .../web/rich_media/parsers/ttl/aws_signed_url.ex   | 15 +++-----
 2 files changed, 29 insertions(+), 30 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index ca592833f..e9aa2dd03 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser do
+  require Logger
+
   defp parsers do
     Pleroma.Config.get([:rich_media, :parsers])
   end
@@ -10,18 +12,19 @@ defmodule Pleroma.Web.RichMedia.Parser do
   def parse(nil), do: {:error, "No URL provided"}
 
   if Pleroma.Config.get(:env) == :test do
+    @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
     def parse(url), do: parse_url(url)
   else
+    @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
     def parse(url) do
-      try do
-        Cachex.fetch!(:rich_media_cache, url, fn _ ->
-          {:commit, parse_url(url)}
-        end)
-        |> set_ttl_based_on_image(url)
-      rescue
-        e ->
-          {:error, "Cachex error: #{inspect(e)}"}
-      end
+      Cachex.fetch!(:rich_media_cache, url, fn _ ->
+        with {:ok, data} <- parse_url(url) do
+          {:commit, {:ok, data}}
+        else
+          error -> {:ignore, error}
+        end
+      end)
+      |> set_ttl_based_on_image(url)
     end
   end
 
@@ -47,9 +50,11 @@ defmodule Pleroma.Web.RichMedia.Parser do
       config :pleroma, :rich_media,
         ttl_setters: [MyModule]
   """
+  @spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) ::
+          {:ok, map()} | {:error, any()}
   def set_ttl_based_on_image({:ok, data}, url) do
     with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
-         ttl when is_number(ttl) <- get_ttl_from_image(data, url) do
+         {:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do
       Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
       {:ok, data}
     else
@@ -58,8 +63,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
     end
   end
 
+  def set_ttl_based_on_image({:error, _} = error, _) do
+    Logger.error("parsing error: #{inspect(error)}")
+    error
+  end
+
   defp get_ttl_from_image(data, url) do
-    Pleroma.Config.get([:rich_media, :ttl_setters])
+    [:rich_media, :ttl_setters]
+    |> Pleroma.Config.get()
     |> Enum.reduce({:ok, nil}, fn
       module, {:ok, _ttl} ->
         module.ttl(data, url)
@@ -70,23 +81,16 @@ defmodule Pleroma.Web.RichMedia.Parser do
   end
 
   defp parse_url(url) do
-    try do
-      {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
-
+    with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
+         {:ok, html} <- Floki.parse_document(html) do
       html
-      |> parse_html()
       |> maybe_parse()
       |> Map.put("url", url)
       |> clean_parsed_data()
       |> check_parsed_data()
-    rescue
-      e ->
-        {:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
     end
   end
 
-  defp parse_html(html), do: Floki.parse_document!(html)
-
   defp maybe_parse(html) do
     Enum.reduce_while(parsers(), %{}, fn parser, acc ->
       case parser.parse(html, acc) do
diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
index 0dc1efdaf..c5aaea2d4 100644
--- a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
+++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
@@ -10,20 +10,15 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
       |> parse_query_params()
       |> format_query_params()
       |> get_expiration_timestamp()
+    else
+      {:error, "Not aws signed url #{inspect(image)}"}
     end
   end
 
-  defp is_aws_signed_url(""), do: nil
-  defp is_aws_signed_url(nil), do: nil
-
-  defp is_aws_signed_url(image) when is_binary(image) do
+  defp is_aws_signed_url(image) when is_binary(image) and image != "" do
     %URI{host: host, query: query} = URI.parse(image)
 
-    if String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") do
-      image
-    else
-      nil
-    end
+    String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires")
   end
 
   defp is_aws_signed_url(_), do: nil
@@ -46,6 +41,6 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
       |> Map.get("X-Amz-Date")
       |> Timex.parse("{ISO:Basic:Z}")
 
-    Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))
+    {:ok, Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))}
   end
 end
-- 
cgit v1.2.3


From 0a9c63fb4351ed29a521697f2c584b0ae007696c Mon Sep 17 00:00:00 2001
From: Sean King <seanking2919@protonmail.com>
Date: Tue, 1 Sep 2020 12:20:32 -0600
Subject: Fix frontend install mix task bug

---
 lib/mix/tasks/pleroma/frontend.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
index 2adbf8d72..0a48be1fe 100644
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -69,7 +69,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
 
     fe_label = "#{frontend} (#{ref})"
 
-    tmp_dir = Path.join(dest, "tmp")
+    tmp_dir = Path.join([instance_static_dir, "frontends", "tmp"])
 
     with {_, :ok} <-
            {:download_or_unzip, download_or_unzip(frontend_info, tmp_dir, options[:file])},
@@ -133,6 +133,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
 
   defp install_frontend(frontend_info, source, dest) do
     from = frontend_info["build_dir"] || "dist"
+    File.rm_rf!(dest)
     File.mkdir_p!(dest)
     File.cp_r!(Path.join([source, from]), dest)
     :ok
-- 
cgit v1.2.3


From 6ce28c409137972ee9b105b9d7ab4a0fd2a0d08b Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Tue, 1 Sep 2020 21:21:58 +0300
Subject: [#2497] Fix for png media proxy preview response headers
 (content-type & content-disposition).

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 2afcd861a..961c73666 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -67,7 +67,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  # TODO: find a workaround so avatar_static and banner_static can work.
+  # TODO: find a workaround so avatar_static and header_static can work.
   # Those only permit GIFs for animation, so we have to permit a way to
   # allow those to get real static variants.
   defp handle_preview("image/gif" = _content_type, conn, url) do
@@ -107,7 +107,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
              }
            ) do
       conn
-      |> put_preview_response_headers()
+      |> put_preview_response_headers("image/png", "preview.png")
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
@@ -145,10 +145,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp put_preview_response_headers(conn) do
+  defp put_preview_response_headers(conn, content_type \\ "image/jpeg", filename \\ "preview.jpg") do
     conn
-    |> put_resp_header("content-type", "image/jpeg")
-    |> put_resp_header("content-disposition", "inline; filename=\"preview.jpg\"")
+    |> put_resp_header("content-type", content_type)
+    |> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"")
     |> put_resp_header("cache-control", "max-age=0, private, must-revalidate")
   end
 
-- 
cgit v1.2.3


From 868057871ac041346d8367181f00f0b127b33945 Mon Sep 17 00:00:00 2001
From: Karol Kosek <krkk@krkk.ct8.pl>
Date: Tue, 1 Sep 2020 19:56:32 +0200
Subject: search: fix 'following' query parameter

The parameter included the accounts that are following you (followers)
instead of those you are actually following.

Co-Authored-By: Haelwenn (lanodan) Monnier <contact@hacktivis.me>
---
 lib/pleroma/user/search.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index d4fd31069..adbef7fb8 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -116,7 +116,7 @@ defmodule Pleroma.User.Search do
   end
 
   defp base_query(_user, false), do: User
-  defp base_query(user, true), do: User.get_followers_query(user)
+  defp base_query(user, true), do: User.get_friends_query(user)
 
   defp filter_invisible_users(query) do
     from(q in query, where: q.invisible == false)
-- 
cgit v1.2.3


From c17d83cd7330d5c874f647a5224a3a130ed88fb0 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 19 Aug 2020 10:22:59 +0300
Subject: improvements and fixes for http requests

- fix for gun worker termination in some circumstances
- pool for http clients (ex_aws, tzdata)
- default pool timeouts for gun
- gun retries on gun_down messages
- s3 upload timeout if streaming enabled
---
 lib/pleroma/gun/connection_pool/worker.ex | 35 +++++++++++++++++++++----------
 lib/pleroma/http/adapter_helper.ex        | 16 ++++++++------
 lib/pleroma/http/adapter_helper/gun.ex    | 18 ++++++++--------
 lib/pleroma/http/ex_aws.ex                |  2 ++
 lib/pleroma/http/tzdata.ex                |  4 ++++
 lib/pleroma/uploaders/s3.ex               | 19 +++++++++++------
 6 files changed, 62 insertions(+), 32 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index fec9d0efa..c36332817 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -83,17 +83,25 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
       end)
 
     {ref, state} = pop_in(state.client_monitors[client_pid])
-    Process.demonitor(ref)
-
-    timer =
-      if used_by == [] do
-        max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
-        Process.send_after(self(), :idle_close, max_idle)
+    # DOWN message can receive right after `remove_client` call and cause worker to terminate
+    state =
+      if is_nil(ref) do
+        state
       else
-        nil
+        Process.demonitor(ref)
+
+        timer =
+          if used_by == [] do
+            max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
+            Process.send_after(self(), :idle_close, max_idle)
+          else
+            nil
+          end
+
+        %{state | timer: timer}
       end
 
-    {:reply, :ok, %{state | timer: timer}, :hibernate}
+    {:reply, :ok, state, :hibernate}
   end
 
   @impl true
@@ -103,16 +111,21 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
     {:stop, :normal, state}
   end
 
+  @impl true
+  def handle_info({:gun_up, _pid, _protocol}, state) do
+    {:noreply, state, :hibernate}
+  end
+
   # Gracefully shutdown if the connection got closed without any streams left
   @impl true
   def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
     {:stop, :normal, state}
   end
 
-  # Otherwise, shutdown with an error
+  # Otherwise, wait for retry
   @impl true
-  def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do
-    {:stop, {:error, down_message}, state}
+  def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
+    {:noreply, state, :hibernate}
   end
 
   @impl true
diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index 9ec3836b0..740e6e9ff 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
 
   @type proxy_type() :: :socks4 | :socks5
   @type host() :: charlist() | :inet.ip_address()
+  @type pool() :: :federation | :upload | :media | :default
 
   alias Pleroma.Config
   alias Pleroma.HTTP.AdapterHelper
@@ -44,14 +45,13 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @spec options(URI.t(), keyword()) :: keyword()
   def options(%URI{} = uri, opts \\ []) do
     @defaults
-    |> put_timeout()
     |> Keyword.merge(opts)
+    |> put_timeout()
     |> adapter_helper().options(uri)
   end
 
-  # For Hackney, this is the time a connection can stay idle in the pool.
-  # For Gun, this is the timeout to receive a message from Gun.
-  defp put_timeout(opts) do
+  @spec pool_timeout(pool()) :: non_neg_integer()
+  def pool_timeout(pool) do
     {config_key, default} =
       if adapter() == Tesla.Adapter.Gun do
         {:pools, Config.get([:pools, :default, :timeout], 5_000)}
@@ -59,9 +59,13 @@ defmodule Pleroma.HTTP.AdapterHelper do
         {:hackney_pools, 10_000}
       end
 
-    timeout = Config.get([config_key, opts[:pool], :timeout], default)
+    Config.get([config_key, pool, :timeout], default)
+  end
 
-    Keyword.merge(opts, timeout: timeout)
+  # For Hackney, this is the time a connection can stay idle in the pool.
+  # For Gun, this is the timeout to receive a message from Gun.
+  defp put_timeout(opts) do
+    Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
   end
 
   def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index b4ff8306c..db0a298b3 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.HTTP.AdapterHelper.Gun do
   @behaviour Pleroma.HTTP.AdapterHelper
 
+  alias Pleroma.Config
   alias Pleroma.Gun.ConnectionPool
   alias Pleroma.HTTP.AdapterHelper
 
@@ -14,7 +15,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     connect_timeout: 5_000,
     domain_lookup_timeout: 5_000,
     tls_handshake_timeout: 5_000,
-    retry: 0,
+    retry: 1,
     retry_timeout: 1000,
     await_up_timeout: 5_000
   ]
@@ -22,10 +23,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   @spec options(keyword(), URI.t()) :: keyword()
   def options(incoming_opts \\ [], %URI{} = uri) do
     proxy =
-      Pleroma.Config.get([:http, :proxy_url])
+      [:http, :proxy_url]
+      |> Config.get()
       |> AdapterHelper.format_proxy()
 
-    config_opts = Pleroma.Config.get([:http, :adapter], [])
+    config_opts = Config.get([:http, :adapter], [])
 
     @defaults
     |> Keyword.merge(config_opts)
@@ -37,8 +39,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
 
   defp add_scheme_opts(opts, %{scheme: "https"}) do
-    opts
-    |> Keyword.put(:certificates_verification, true)
+    Keyword.put(opts, :certificates_verification, true)
   end
 
   @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
@@ -51,11 +52,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
 
   @prefix Pleroma.Gun.ConnectionPool
   def limiter_setup do
-    wait = Pleroma.Config.get([:connections_pool, :connection_acquisition_wait])
-    retries = Pleroma.Config.get([:connections_pool, :connection_acquisition_retries])
+    wait = Config.get([:connections_pool, :connection_acquisition_wait])
+    retries = Config.get([:connections_pool, :connection_acquisition_retries])
 
     :pools
-    |> Pleroma.Config.get([])
+    |> Config.get([])
     |> Enum.each(fn {name, opts} ->
       max_running = Keyword.get(opts, :size, 50)
       max_waiting = Keyword.get(opts, :max_waiting, 10)
@@ -69,7 +70,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
       case result do
         :ok -> :ok
         {:error, :existing} -> :ok
-        e -> raise e
       end
     end)
 
diff --git a/lib/pleroma/http/ex_aws.ex b/lib/pleroma/http/ex_aws.ex
index e53e64077..2fe8beafc 100644
--- a/lib/pleroma/http/ex_aws.ex
+++ b/lib/pleroma/http/ex_aws.ex
@@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.ExAws do
 
   @impl true
   def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
+    http_opts = Keyword.put(http_opts, :adapter, pool: :upload)
+
     case HTTP.request(method, url, body, headers, http_opts) do
       {:ok, env} ->
         {:ok, %{status_code: env.status, headers: env.headers, body: env.body}}
diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex
index 34bb253a7..7dd3352ff 100644
--- a/lib/pleroma/http/tzdata.ex
+++ b/lib/pleroma/http/tzdata.ex
@@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def get(url, headers, options) do
+    options = Keyword.put(options, :adapter, pool: :upload)
+
     with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
       {:ok, {env.status, env.headers, env.body}}
     end
@@ -18,6 +20,8 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def head(url, headers, options) do
+    options = Keyword.put(options, :adapter, pool: :upload)
+
     with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
       {:ok, {env.status, env.headers}}
     end
diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex
index a13ff23b6..ed9794ca2 100644
--- a/lib/pleroma/uploaders/s3.ex
+++ b/lib/pleroma/uploaders/s3.ex
@@ -46,12 +46,19 @@ defmodule Pleroma.Uploaders.S3 do
 
     op =
       if streaming do
-        upload.tempfile
-        |> ExAws.S3.Upload.stream_file()
-        |> ExAws.S3.upload(bucket, s3_name, [
-          {:acl, :public_read},
-          {:content_type, upload.content_type}
-        ])
+        op =
+          upload.tempfile
+          |> ExAws.S3.Upload.stream_file()
+          |> ExAws.S3.upload(bucket, s3_name, [
+            {:acl, :public_read},
+            {:content_type, upload.content_type}
+          ])
+
+        # set s3 upload timeout to respect :upload pool timeout
+        # timeout should be slightly larger, so s3 can retry upload on fail
+        timeout = Pleroma.HTTP.AdapterHelper.pool_timeout(:upload) + 1_000
+        opts = Keyword.put(op.opts, :timeout, timeout)
+        Map.put(op, :opts, opts)
       else
         {:ok, file_data} = File.read(upload.tempfile)
 
-- 
cgit v1.2.3


From 5e8adf91b46c3f23ec423d53afccbb062df4a241 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 1 Sep 2020 11:30:56 +0300
Subject: don't overwrite passed pool option in http clients

---
 lib/pleroma/http/ex_aws.ex | 2 +-
 lib/pleroma/http/tzdata.ex | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/http/ex_aws.ex b/lib/pleroma/http/ex_aws.ex
index 2fe8beafc..c3f335c73 100644
--- a/lib/pleroma/http/ex_aws.ex
+++ b/lib/pleroma/http/ex_aws.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.ExAws do
 
   @impl true
   def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
-    http_opts = Keyword.put(http_opts, :adapter, pool: :upload)
+    http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload)
 
     case HTTP.request(method, url, body, headers, http_opts) do
       {:ok, env} ->
diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex
index 7dd3352ff..356799aab 100644
--- a/lib/pleroma/http/tzdata.ex
+++ b/lib/pleroma/http/tzdata.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def get(url, headers, options) do
-    options = Keyword.put(options, :adapter, pool: :upload)
+    options = Keyword.put_new(options, :adapter, pool: :upload)
 
     with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
       {:ok, {env.status, env.headers, env.body}}
@@ -20,7 +20,7 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def head(url, headers, options) do
-    options = Keyword.put(options, :adapter, pool: :upload)
+    options = Keyword.put_new(options, :adapter, pool: :upload)
 
     with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
       {:ok, {env.status, env.headers}}
-- 
cgit v1.2.3


From 79f65b4374908a32ebf39db176a30a01152a9141 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 2 Sep 2020 09:16:51 +0300
Subject: correct pool and uniform headers format

---
 lib/mix/tasks/pleroma/frontend.ex        | 4 +++-
 lib/pleroma/instances/instance.ex        | 4 +++-
 lib/pleroma/object/fetcher.ex            | 6 +++---
 lib/pleroma/web/rich_media/helpers.ex    | 2 +-
 lib/pleroma/web/web_finger/web_finger.ex | 4 ++--
 5 files changed, 12 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
index 2adbf8d72..484af6da7 100644
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -124,7 +124,9 @@ defmodule Mix.Tasks.Pleroma.Frontend do
     url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
 
     with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [], timeout: 120_000, recv_timeout: 120_000) do
+           Pleroma.HTTP.get(url, [],
+             adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000]
+           ) do
       unzip(zip_body, dest)
     else
       e -> {:error, e}
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index a1f935232..711c42158 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -150,7 +150,9 @@ defmodule Pleroma.Instances.Instance do
   defp scrape_favicon(%URI{} = instance_uri) do
     try do
       with {:ok, %Tesla.Env{body: html}} <-
-             Pleroma.HTTP.get(to_string(instance_uri), [{:Accept, "text/html"}]),
+             Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
+               adapter: [pool: :media]
+             ),
            favicon_rel <-
              html
              |> Floki.parse_document!()
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 6fdbc8efd..374d8704a 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -164,12 +164,12 @@ defmodule Pleroma.Object.Fetcher do
         date: date
       })
 
-    [{"signature", signature}]
+    {"signature", signature}
   end
 
   defp sign_fetch(headers, id, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ make_signature(id, date)
+      [make_signature(id, date) | headers]
     else
       headers
     end
@@ -177,7 +177,7 @@ defmodule Pleroma.Object.Fetcher do
 
   defp maybe_date_fetch(headers, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ [{"date", date}]
+      [{"date", date} | headers]
     else
       headers
     end
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 6210f2c5a..2fb482b51 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -96,6 +96,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
         @rich_media_options
       end
 
-    Pleroma.HTTP.get(url, headers, options)
+    Pleroma.HTTP.get(url, headers, adapter: options)
   end
 end
diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index c4051e63e..6629f5356 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -136,12 +136,12 @@ defmodule Pleroma.Web.WebFinger do
 
   def find_lrdd_template(domain) do
     with {:ok, %{status: status, body: body}} when status in 200..299 <-
-           HTTP.get("http://#{domain}/.well-known/host-meta", []) do
+           HTTP.get("http://#{domain}/.well-known/host-meta") do
       get_template_from_xml(body)
     else
       _ ->
         with {:ok, %{body: body, status: status}} when status in 200..299 <-
-               HTTP.get("https://#{domain}/.well-known/host-meta", []) do
+               HTTP.get("https://#{domain}/.well-known/host-meta") do
           get_template_from_xml(body)
         else
           e -> {:error, "Can't find LRDD template: #{inspect(e)}"}
-- 
cgit v1.2.3


From 1c57ef44983e150f3cc016290fe99f159eb79cb0 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 2 Sep 2020 10:33:43 +0300
Subject: default pool for tz_data client

---
 lib/pleroma/http/tzdata.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex
index 356799aab..4539ac359 100644
--- a/lib/pleroma/http/tzdata.ex
+++ b/lib/pleroma/http/tzdata.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def get(url, headers, options) do
-    options = Keyword.put_new(options, :adapter, pool: :upload)
+    options = Keyword.put_new(options, :adapter, pool: :default)
 
     with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
       {:ok, {env.status, env.headers, env.body}}
@@ -20,7 +20,7 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def head(url, headers, options) do
-    options = Keyword.put_new(options, :adapter, pool: :upload)
+    options = Keyword.put_new(options, :adapter, pool: :default)
 
     with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
       {:ok, {env.status, env.headers}}
-- 
cgit v1.2.3


From 84fbf1616104c09e0f4f5442d86ca2c573ae4056 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 2 Sep 2020 10:50:51 +0300
Subject: timeout option moved to gun adapter helper

---
 lib/pleroma/http/adapter_helper.ex     | 23 ++---------------------
 lib/pleroma/http/adapter_helper/gun.ex | 15 +++++++++++++++
 lib/pleroma/uploaders/s3.ex            | 14 +++++++++-----
 3 files changed, 26 insertions(+), 26 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index 740e6e9ff..0728cbaa2 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -10,9 +10,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
 
   @type proxy_type() :: :socks4 | :socks5
   @type host() :: charlist() | :inet.ip_address()
-  @type pool() :: :federation | :upload | :media | :default
 
-  alias Pleroma.Config
   alias Pleroma.HTTP.AdapterHelper
   require Logger
 
@@ -46,29 +44,12 @@ defmodule Pleroma.HTTP.AdapterHelper do
   def options(%URI{} = uri, opts \\ []) do
     @defaults
     |> Keyword.merge(opts)
-    |> put_timeout()
     |> adapter_helper().options(uri)
   end
 
-  @spec pool_timeout(pool()) :: non_neg_integer()
-  def pool_timeout(pool) do
-    {config_key, default} =
-      if adapter() == Tesla.Adapter.Gun do
-        {:pools, Config.get([:pools, :default, :timeout], 5_000)}
-      else
-        {:hackney_pools, 10_000}
-      end
-
-    Config.get([config_key, pool, :timeout], default)
-  end
-
-  # For Hackney, this is the time a connection can stay idle in the pool.
-  # For Gun, this is the timeout to receive a message from Gun.
-  defp put_timeout(opts) do
-    Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
-  end
-
+  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
   def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
+
   defp adapter, do: Application.get_env(:tesla, :adapter)
 
   defp adapter_helper do
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index db0a298b3..02e20f2d1 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -20,6 +20,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     await_up_timeout: 5_000
   ]
 
+  @type pool() :: :federation | :upload | :media | :default
+
   @spec options(keyword(), URI.t()) :: keyword()
   def options(incoming_opts \\ [], %URI{} = uri) do
     proxy =
@@ -34,6 +36,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     |> add_scheme_opts(uri)
     |> AdapterHelper.maybe_add_proxy(proxy)
     |> Keyword.merge(incoming_opts)
+    |> put_timeout()
   end
 
   defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
@@ -42,6 +45,18 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     Keyword.put(opts, :certificates_verification, true)
   end
 
+  defp put_timeout(opts) do
+    # this is the timeout to receive a message from Gun
+    Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
+  end
+
+  @spec pool_timeout(pool()) :: non_neg_integer()
+  def pool_timeout(pool) do
+    default = Config.get([:pools, :default, :timeout], 5_000)
+
+    Config.get([:pools, pool, :timeout], default)
+  end
+
   @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
   def get_conn(uri, opts) do
     case ConnectionPool.get_conn(uri, opts) do
diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex
index ed9794ca2..6dbef9085 100644
--- a/lib/pleroma/uploaders/s3.ex
+++ b/lib/pleroma/uploaders/s3.ex
@@ -54,11 +54,15 @@ defmodule Pleroma.Uploaders.S3 do
             {:content_type, upload.content_type}
           ])
 
-        # set s3 upload timeout to respect :upload pool timeout
-        # timeout should be slightly larger, so s3 can retry upload on fail
-        timeout = Pleroma.HTTP.AdapterHelper.pool_timeout(:upload) + 1_000
-        opts = Keyword.put(op.opts, :timeout, timeout)
-        Map.put(op, :opts, opts)
+        if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do
+          # set s3 upload timeout to respect :upload pool timeout
+          # timeout should be slightly larger, so s3 can retry upload on fail
+          timeout = Pleroma.HTTP.AdapterHelper.Gun.pool_timeout(:upload) + 1_000
+          opts = Keyword.put(op.opts, :timeout, timeout)
+          Map.put(op, :opts, opts)
+        else
+          op
+        end
       else
         {:ok, file_data} = File.read(upload.tempfile)
 
-- 
cgit v1.2.3


From 46236d1d873473d95b11cd7bfdcaa284ea55a9ad Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 2 Sep 2020 12:42:25 +0300
Subject: html.ex: optimize external url extraction

By using a :not() selector and only extracting attributes from the
first match.
---
 lib/pleroma/html.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index dc1b9b840..20b02f091 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -109,8 +109,9 @@ defmodule Pleroma.HTML do
       result =
         content
         |> Floki.parse_fragment!()
-        |> Floki.filter_out("a.mention,a.hashtag,a.attachment,a[rel~=\"tag\"]")
-        |> Floki.attribute("a", "href")
+        |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
+        |> Enum.take(1)
+        |> Floki.attribute("href")
         |> Enum.at(0)
 
       {:commit, {:ok, result}}
-- 
cgit v1.2.3


From 19691389b92e617f1edad7d4e3168fe917d0a675 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 2 Sep 2020 14:21:28 +0300
Subject: Rich media: Add failure tracking

---
 lib/pleroma/web/rich_media/parser.ex | 54 +++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 22 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index e9aa2dd03..e98c743ca 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -17,14 +17,25 @@ defmodule Pleroma.Web.RichMedia.Parser do
   else
     @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
     def parse(url) do
-      Cachex.fetch!(:rich_media_cache, url, fn _ ->
-        with {:ok, data} <- parse_url(url) do
-          {:commit, {:ok, data}}
-        else
-          error -> {:ignore, error}
-        end
-      end)
-      |> set_ttl_based_on_image(url)
+      with {:ok, data} <- get_cached_or_parse(url),
+           {:ok, _} <- set_ttl_based_on_image(data, url) do
+        {:ok, data}
+      else
+        error ->
+          Logger.error(fn -> "Rich media error: #{inspect(error)}" end)
+      end
+    end
+
+    defp get_cached_or_parse(url) do
+      case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do
+        {:ok, _data} = res ->
+          res
+
+        {:error, _} = e ->
+          ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
+          Cachex.expire(:rich_media_cache, url, ttl)
+          e
+      end
     end
   end
 
@@ -50,24 +61,23 @@ defmodule Pleroma.Web.RichMedia.Parser do
       config :pleroma, :rich_media,
         ttl_setters: [MyModule]
   """
-  @spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) ::
-          {:ok, map()} | {:error, any()}
-  def set_ttl_based_on_image({:ok, data}, url) do
-    with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
-         {:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do
-      Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
-      {:ok, data}
-    else
+  @spec set_ttl_based_on_image(map(), String.t()) ::
+          {:ok, Integer.t() | :noop} | {:error, :no_key}
+  def set_ttl_based_on_image(data, url) do
+    case get_ttl_from_image(data, url) do
+      {:ok, ttl} when is_number(ttl) ->
+        ttl = ttl * 1000
+
+        case Cachex.expire_at(:rich_media_cache, url, ttl) do
+          {:ok, true} -> {:ok, ttl}
+          {:ok, false} -> {:error, :no_key}
+        end
+
       _ ->
-        {:ok, data}
+        {:ok, :noop}
     end
   end
 
-  def set_ttl_based_on_image({:error, _} = error, _) do
-    Logger.error("parsing error: #{inspect(error)}")
-    error
-  end
-
   defp get_ttl_from_image(data, url) do
     [:rich_media, :ttl_setters]
     |> Pleroma.Config.get()
-- 
cgit v1.2.3


From a11f23c130331d98db941c3edcc4d2dcf139bbc6 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Wed, 2 Sep 2020 15:45:47 +0300
Subject: user agent if Endpoint is not started yet

---
 lib/pleroma/application.ex | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 005aba50a..33b1e3872 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -22,13 +22,18 @@ defmodule Pleroma.Application do
   def repository, do: @repository
 
   def user_agent do
-    case Config.get([:http, :user_agent], :default) do
-      :default ->
-        info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
-        named_version() <> "; " <> info
-
-      custom ->
-        custom
+    if Process.whereis(Pleroma.Web.Endpoint) do
+      case Config.get([:http, :user_agent], :default) do
+        :default ->
+          info = "#{Pleroma.Web.base_url()} <#{Config.get([:instance, :email], "")}>"
+          named_version() <> "; " <> info
+
+        custom ->
+          custom
+      end
+    else
+      # fallback, if endpoint is not started yet
+      "Pleroma Data Loader"
     end
   end
 
-- 
cgit v1.2.3


From d48fc90978eee46c8fba00a80082d14fc31a0eec Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 2 Sep 2020 13:44:08 +0300
Subject: StatusView: Start fetching rich media cards as soon as possible

---
 lib/pleroma/web/mastodon_api/views/status_view.ex | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 01b8bb6bb..3fe1967be 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -23,6 +23,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
   import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2]
 
+  # This is a naive way to do this, just spawning a process per activity
+  # to fetch the preview. However it should be fine considering
+  # pagination is restricted to 40 activities at a time
+  defp fetch_rich_media_for_activities(activities) do
+    Enum.each(activities, fn activity ->
+      spawn(fn ->
+        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+      end)
+    end)
+  end
+
   # TODO: Add cached version.
   defp get_replied_to_activities([]), do: %{}
 
@@ -80,6 +91,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list
     activities = Enum.filter(opts.activities, & &1)
+
+    # Start fetching rich media before doing anything else, so that later calls to get the cards
+    # only block for timeout in the worst case, as opposed to
+    # length(activities_with_links) * timeout
+    fetch_rich_media_for_activities(activities)
     replied_to_activities = get_replied_to_activities(activities)
 
     parent_activities =
-- 
cgit v1.2.3


From cbf7f0e02943f44a73f4418b8c6a8bada06331d8 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 2 Sep 2020 09:09:13 -0500
Subject: Disallow password resets for deactivated accounts.

Ensure all responses to password reset events are identical.
---
 .../web/mastodon_api/controllers/auth_controller.ex      | 16 +++++-----------
 lib/pleroma/web/twitter_api/twitter_api.ex               | 13 ++-----------
 2 files changed, 7 insertions(+), 22 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
index 753b3db3e..9f09550e1 100644
--- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
@@ -59,17 +59,11 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do
   def password_reset(conn, params) do
     nickname_or_email = params["email"] || params["nickname"]
 
-    with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do
-      conn
-      |> put_status(:no_content)
-      |> json("")
-    else
-      {:error, "unknown user"} ->
-        send_resp(conn, :not_found, "")
-
-      {:error, _} ->
-        send_resp(conn, :bad_request, "")
-    end
+    TwitterAPI.password_reset(nickname_or_email)
+
+    conn
+    |> put_status(:no_content)
+    |> json("")
   end
 
   defp local_mastodon_root_path(conn) do
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index 2294d9d0d..5d7948507 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -72,7 +72,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
   def password_reset(nickname_or_email) do
     with true <- is_binary(nickname_or_email),
-         %User{local: true, email: email} = user when is_binary(email) <-
+         %User{local: true, email: email, deactivated: false} = user when is_binary(email) <-
            User.get_by_nickname_or_email(nickname_or_email),
          {:ok, token_record} <- Pleroma.PasswordResetToken.create_token(user) do
       user
@@ -81,17 +81,8 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
       {:ok, :enqueued}
     else
-      false ->
-        {:error, "bad user identifier"}
-
-      %User{local: true, email: nil} ->
+      _ ->
         {:ok, :noop}
-
-      %User{local: false} ->
-        {:error, "remote user"}
-
-      nil ->
-        {:error, "unknown user"}
     end
   end
 
-- 
cgit v1.2.3


From 581f382e712dc50fa51d1ab211f13b8843dcb448 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 2 Sep 2020 18:32:00 +0200
Subject: ListController: DRY up stuff.

---
 .../web/mastodon_api/controllers/list_controller.ex       | 15 +++------------
 1 file changed, 3 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex
index 4df13cb81..5daeaa780 100644
--- a/lib/pleroma/web/mastodon_api/controllers/list_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/list_controller.ex
@@ -74,7 +74,7 @@ defmodule Pleroma.Web.MastodonAPI.ListController do
 
   # DELETE /api/v1/lists/:id/accounts
   def remove_from_list(
-        %{assigns: %{list: list}, body_params: %{account_ids: account_ids}} = conn,
+        %{assigns: %{list: list}, params: %{account_ids: account_ids}} = conn,
         _
       ) do
     Enum.each(account_ids, fn account_id ->
@@ -86,17 +86,8 @@ defmodule Pleroma.Web.MastodonAPI.ListController do
     json(conn, %{})
   end
 
-  def remove_from_list(
-        %{assigns: %{list: list}, params: %{account_ids: account_ids}} = conn,
-        _
-      ) do
-    Enum.each(account_ids, fn account_id ->
-      with %User{} = followed <- User.get_cached_by_id(account_id) do
-        Pleroma.List.unfollow(list, followed)
-      end
-    end)
-
-    json(conn, %{})
+  def remove_from_list(%{body_params: params} = conn, _) do
+    remove_from_list(%{conn | params: params}, %{})
   end
 
   defp list_by_id_and_user(%{assigns: %{user: user}, params: %{id: id}} = conn, _) do
-- 
cgit v1.2.3


From 60c925380da644866836fa4a275f4d57eaaada04 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 3 Sep 2020 20:13:29 +0300
Subject: [#2497] Added support for enforcing output format for media proxy
 preview, used for avatar_static & header_static (AccountView).

---
 lib/pleroma/helpers/uri_helper.ex                     |  1 +
 lib/pleroma/web/mastodon_api/views/account_view.ex    |  4 ++--
 lib/pleroma/web/media_proxy/media_proxy.ex            | 15 +++++++++------
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 11 ++++++++---
 4 files changed, 20 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/uri_helper.ex b/lib/pleroma/helpers/uri_helper.ex
index 6d205a636..9c9e53447 100644
--- a/lib/pleroma/helpers/uri_helper.ex
+++ b/lib/pleroma/helpers/uri_helper.ex
@@ -15,6 +15,7 @@ defmodule Pleroma.Helpers.UriHelper do
     uri
     |> Map.put(:query, URI.encode_query(updated_params))
     |> URI.to_string()
+    |> String.replace_suffix("?", "")
   end
 
   def maybe_add_base("/" <> uri, base), do: Path.join([base, uri])
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index 7eb4e86fe..a811f81c2 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -182,9 +182,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     display_name = user.name || user.nickname
 
     avatar = User.avatar_url(user) |> MediaProxy.url()
-    avatar_static = User.avatar_url(user) |> MediaProxy.preview_url()
+    avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(output_format: "jpeg")
     header = User.banner_url(user) |> MediaProxy.url()
-    header_static = User.banner_url(user) |> MediaProxy.preview_url()
+    header_static = User.banner_url(user) |> MediaProxy.preview_url(output_format: "jpeg")
 
     following_count =
       if !user.hide_follows_count or !user.hide_follows or opts[:for] == user do
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index 6695d49ce..4cbe1cf89 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.MediaProxy do
   alias Pleroma.Config
+  alias Pleroma.Helpers.UriHelper
   alias Pleroma.Upload
   alias Pleroma.Web
   alias Pleroma.Web.MediaProxy.Invalidation
@@ -58,9 +59,9 @@ defmodule Pleroma.Web.MediaProxy do
 
   # Note: routing all URLs to preview handler (even local and whitelisted).
   #   Preview handler will call url/1 on decoded URLs, and applicable ones will detour media proxy.
-  def preview_url(url) do
+  def preview_url(url, preview_params \\ []) do
     if preview_enabled?() do
-      encode_preview_url(url)
+      encode_preview_url(url, preview_params)
     else
       url
     end
@@ -116,10 +117,10 @@ defmodule Pleroma.Web.MediaProxy do
     build_url(sig64, base64, filename(url))
   end
 
-  def encode_preview_url(url) do
+  def encode_preview_url(url, preview_params \\ []) do
     {base64, sig64} = base64_sig64(url)
 
-    build_preview_url(sig64, base64, filename(url))
+    build_preview_url(sig64, base64, filename(url), preview_params)
   end
 
   def decode_url(sig, url) do
@@ -155,8 +156,10 @@ defmodule Pleroma.Web.MediaProxy do
     proxy_url("proxy", sig_base64, url_base64, filename)
   end
 
-  def build_preview_url(sig_base64, url_base64, filename \\ nil) do
-    proxy_url("proxy/preview", sig_base64, url_base64, filename)
+  def build_preview_url(sig_base64, url_base64, filename \\ nil, preview_params \\ []) do
+    uri = proxy_url("proxy/preview", sig_base64, url_base64, filename)
+
+    UriHelper.append_uri_params(uri, preview_params)
   end
 
   def verify_request_path_and_url(
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 961c73666..9dc76e928 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -67,9 +67,14 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  # TODO: find a workaround so avatar_static and header_static can work.
-  # Those only permit GIFs for animation, so we have to permit a way to
-  # allow those to get real static variants.
+  defp handle_preview(
+         "image/" <> _ = _content_type,
+         %{params: %{"output_format" => "jpeg"}} = conn,
+         url
+       ) do
+    handle_jpeg_preview(conn, url)
+  end
+
   defp handle_preview("image/gif" = _content_type, conn, url) do
     mediaproxy_url = url |> MediaProxy.url()
 
-- 
cgit v1.2.3


From 6141eb94ab034b5141a5c60b2814fb45b829c1ac Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 3 Sep 2020 12:40:42 -0500
Subject: Fetch preview requests through the MediaProxy. Separate connection
 options are not needed.

Use a separate pool for preview requests
---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 961c73666..b1f00fa0c 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -33,8 +33,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   def preview(conn, %{"sig" => sig64, "url" => url64}) do
     with {_, true} <- {:enabled, MediaProxy.preview_enabled?()},
-         {:ok, url} <- MediaProxy.decode_url(sig64, url64),
-         :ok <- MediaProxy.verify_request_path_and_url(conn, url) do
+         {:ok, url} <- MediaProxy.decode_url(sig64, url64) do
       handle_preview(conn, url)
     else
       {:enabled, false} ->
@@ -50,9 +49,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   defp handle_preview(conn, url) do
     with {:ok, %{status: status} = head_response} when status in 200..299 <-
-           Tesla.head(url,
-             opts: [adapter: [timeout: preview_head_request_timeout(), follow_redirect: true]]
-           ) do
+           Pleroma.HTTP.request("head", MediaProxy.url(url), [], [], [adapter: [pool: :preview]]) do
       content_type = Tesla.get_header(head_response, "content-type")
       handle_preview(content_type, conn, url)
     else
@@ -172,17 +169,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     {thumbnail_max_width, thumbnail_max_height}
   end
 
-  defp preview_head_request_timeout do
-    Keyword.get(media_preview_proxy_opts(), :head_request_max_read_duration) ||
-      Keyword.get(media_proxy_opts(), :max_read_duration) ||
-      ReverseProxy.max_read_duration_default()
-  end
-
   defp media_proxy_opts do
     Config.get([:media_proxy, :proxy_opts], [])
   end
-
-  defp media_preview_proxy_opts do
-    Config.get([:media_preview_proxy, :proxy_opts], [])
-  end
 end
-- 
cgit v1.2.3


From f25b0e87f3dd73e02c954c5baab3c52becdd9c9e Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 3 Sep 2020 15:28:57 -0500
Subject: URL passed to helper is already MediaProxy

Set :preview pool on the request
---
 lib/pleroma/helpers/media_helper.ex | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 9bd815c26..cfb091f82 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -12,8 +12,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   def image_resize(url, options) do
     with executable when is_binary(executable) <- System.find_executable("convert"),
          {:ok, args} <- prepare_image_resize_args(options),
-         url = Pleroma.Web.MediaProxy.url(url),
-         {:ok, env} <- Pleroma.HTTP.get(url),
+         {:ok, env} <- Pleroma.HTTP.get(url, [], [adapter: [pool: :preview]]),
          {:ok, fifo_path} <- mkfifo() do
       args = List.flatten([fifo_path, args])
       run_fifo(fifo_path, env, executable, args)
@@ -61,8 +60,7 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
-         url = Pleroma.Web.MediaProxy.url(url),
-         {:ok, env} <- Pleroma.HTTP.get(url),
+         {:ok, env} <- Pleroma.HTTP.get(url, [], [adapter: [pool: :preview]]),
          {:ok, fifo_path} <- mkfifo(),
          args = [
            "-y",
-- 
cgit v1.2.3


From d34fe2840d969c30b393cfb73e34b6301027c776 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 3 Sep 2020 23:15:22 +0300
Subject: HTTP: radically simplify pool checkin/checkout

Use a custom tesla middleware instead of adapter helper function +
custom redirect middleware.

This will also fix "Client died before releasing the connection"
messages when the request pool is overloaded. Since the checkout is
now done after passing ConcurrentLimiter.

This is technically less efficient, since the connection needs to be
checked in/out every time the middleware is left or entered respectively.
But I don't think the nanoseconds we might lose on redirects
to the same host are worth the complexity.
---
 lib/pleroma/http/adapter_helper.ex               |   4 -
 lib/pleroma/http/adapter_helper/gun.ex           |   9 --
 lib/pleroma/http/adapter_helper/hackney.ex       |   3 -
 lib/pleroma/http/http.ex                         |  39 ++++----
 lib/pleroma/tesla/middleware/connection_pool.ex  |  35 ++++++++
 lib/pleroma/tesla/middleware/follow_redirects.ex | 110 -----------------------
 6 files changed, 51 insertions(+), 149 deletions(-)
 create mode 100644 lib/pleroma/tesla/middleware/connection_pool.ex
 delete mode 100644 lib/pleroma/tesla/middleware/follow_redirects.ex

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index 0728cbaa2..d72297323 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -19,7 +19,6 @@ defmodule Pleroma.HTTP.AdapterHelper do
           | {Connection.proxy_type(), Connection.host(), pos_integer()}
 
   @callback options(keyword(), URI.t()) :: keyword()
-  @callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()}
 
   @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
   def format_proxy(nil), do: nil
@@ -47,9 +46,6 @@ defmodule Pleroma.HTTP.AdapterHelper do
     |> adapter_helper().options(uri)
   end
 
-  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
-  def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
-
   defp adapter, do: Application.get_env(:tesla, :adapter)
 
   defp adapter_helper do
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index 02e20f2d1..4a967d8f2 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -6,7 +6,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   @behaviour Pleroma.HTTP.AdapterHelper
 
   alias Pleroma.Config
-  alias Pleroma.Gun.ConnectionPool
   alias Pleroma.HTTP.AdapterHelper
 
   require Logger
@@ -57,14 +56,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     Config.get([:pools, pool, :timeout], default)
   end
 
-  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
-  def get_conn(uri, opts) do
-    case ConnectionPool.get_conn(uri, opts) do
-      {:ok, conn_pid} -> {:ok, Keyword.merge(opts, conn: conn_pid, close_conn: false)}
-      err -> err
-    end
-  end
-
   @prefix Pleroma.Gun.ConnectionPool
   def limiter_setup do
     wait = Config.get([:connections_pool, :connection_acquisition_wait])
diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex
index cd569422b..f47a671ad 100644
--- a/lib/pleroma/http/adapter_helper/hackney.ex
+++ b/lib/pleroma/http/adapter_helper/hackney.ex
@@ -23,7 +23,4 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
   end
 
   defp add_scheme_opts(opts, _), do: opts
-
-  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()}
-  def get_conn(_uri, opts), do: {:ok, opts}
 end
diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
index b37b3fa89..7bc73f4a0 100644
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@@ -62,28 +62,21 @@ defmodule Pleroma.HTTP do
     uri = URI.parse(url)
     adapter_opts = AdapterHelper.options(uri, options[:adapter] || [])
 
-    case AdapterHelper.get_conn(uri, adapter_opts) do
-      {:ok, adapter_opts} ->
-        options = put_in(options[:adapter], adapter_opts)
-        params = options[:params] || []
-        request = build_request(method, headers, options, url, body, params)
-
-        adapter = Application.get_env(:tesla, :adapter)
-
-        client = Tesla.client(adapter_middlewares(adapter), adapter)
-
-        maybe_limit(
-          fn ->
-            request(client, request)
-          end,
-          adapter,
-          adapter_opts
-        )
-
-      # Connection release is handled in a custom FollowRedirects middleware
-      err ->
-        err
-    end
+    options = put_in(options[:adapter], adapter_opts)
+    params = options[:params] || []
+    request = build_request(method, headers, options, url, body, params)
+
+    adapter = Application.get_env(:tesla, :adapter)
+
+    client = Tesla.client(adapter_middlewares(adapter), adapter)
+
+    maybe_limit(
+      fn ->
+        request(client, request)
+      end,
+      adapter,
+      adapter_opts
+    )
   end
 
   @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
@@ -110,7 +103,7 @@ defmodule Pleroma.HTTP do
   end
 
   defp adapter_middlewares(Tesla.Adapter.Gun) do
-    [Pleroma.HTTP.Middleware.FollowRedirects]
+    [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool]
   end
 
   defp adapter_middlewares(_), do: []
diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex
new file mode 100644
index 000000000..a435ab4cc
--- /dev/null
+++ b/lib/pleroma/tesla/middleware/connection_pool.ex
@@ -0,0 +1,35 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Tesla.Middleware.ConnectionPool do
+  @moduledoc """
+  Middleware to get/release connections from `Pleroma.Gun.ConnectionPool`
+  """
+
+  @behaviour Tesla.Middleware
+
+  alias Pleroma.Gun.ConnectionPool
+
+  @impl Tesla.Middleware
+  def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
+    uri = URI.parse(url)
+
+    case ConnectionPool.get_conn(uri, opts[:adapter]) do
+      {:ok, conn_pid} ->
+        adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false)
+        opts = Keyword.put(opts, :adapter, adapter_opts)
+        env = %{env | opts: opts}
+        res = Tesla.run(env, next)
+
+        unless opts[:adapter][:body_as] == :chunks do
+          ConnectionPool.release_conn(conn_pid)
+        end
+
+        res
+
+      err ->
+        err
+    end
+  end
+end
diff --git a/lib/pleroma/tesla/middleware/follow_redirects.ex b/lib/pleroma/tesla/middleware/follow_redirects.ex
deleted file mode 100644
index 5a7032215..000000000
--- a/lib/pleroma/tesla/middleware/follow_redirects.ex
+++ /dev/null
@@ -1,110 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2015-2020 Tymon Tobolski <https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex>
-# Copyright © 2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.HTTP.Middleware.FollowRedirects do
-  @moduledoc """
-  Pool-aware version of https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex
-
-  Follow 3xx redirects
-  ## Options
-  - `:max_redirects` - limit number of redirects (default: `5`)
-  """
-
-  alias Pleroma.Gun.ConnectionPool
-
-  @behaviour Tesla.Middleware
-
-  @max_redirects 5
-  @redirect_statuses [301, 302, 303, 307, 308]
-
-  @impl Tesla.Middleware
-  def call(env, next, opts \\ []) do
-    max = Keyword.get(opts, :max_redirects, @max_redirects)
-
-    redirect(env, next, max)
-  end
-
-  defp redirect(env, next, left) do
-    opts = env.opts[:adapter]
-
-    case Tesla.run(env, next) do
-      {:ok, %{status: status} = res} when status in @redirect_statuses and left > 0 ->
-        release_conn(opts)
-
-        case Tesla.get_header(res, "location") do
-          nil ->
-            {:ok, res}
-
-          location ->
-            location = parse_location(location, res)
-
-            case get_conn(location, opts) do
-              {:ok, opts} ->
-                %{env | opts: Keyword.put(env.opts, :adapter, opts)}
-                |> new_request(res.status, location)
-                |> redirect(next, left - 1)
-
-              e ->
-                e
-            end
-        end
-
-      {:ok, %{status: status}} when status in @redirect_statuses ->
-        release_conn(opts)
-        {:error, {__MODULE__, :too_many_redirects}}
-
-      {:error, _} = e ->
-        release_conn(opts)
-        e
-
-      other ->
-        unless opts[:body_as] == :chunks do
-          release_conn(opts)
-        end
-
-        other
-    end
-  end
-
-  defp get_conn(location, opts) do
-    uri = URI.parse(location)
-
-    case ConnectionPool.get_conn(uri, opts) do
-      {:ok, conn} ->
-        {:ok, Keyword.merge(opts, conn: conn)}
-
-      e ->
-        e
-    end
-  end
-
-  defp release_conn(opts) do
-    ConnectionPool.release_conn(opts[:conn])
-  end
-
-  # The 303 (See Other) redirect was added in HTTP/1.1 to indicate that the originally
-  # requested resource is not available, however a related resource (or another redirect)
-  # available via GET is available at the specified location.
-  # https://tools.ietf.org/html/rfc7231#section-6.4.4
-  defp new_request(env, 303, location), do: %{env | url: location, method: :get, query: []}
-
-  # The 307 (Temporary Redirect) status code indicates that the target
-  # resource resides temporarily under a different URI and the user agent
-  # MUST NOT change the request method (...)
-  # https://tools.ietf.org/html/rfc7231#section-6.4.7
-  defp new_request(env, 307, location), do: %{env | url: location}
-
-  defp new_request(env, _, location), do: %{env | url: location, query: []}
-
-  defp parse_location("https://" <> _rest = location, _env), do: location
-  defp parse_location("http://" <> _rest = location, _env), do: location
-
-  defp parse_location(location, env) do
-    env.url
-    |> URI.parse()
-    |> URI.merge(location)
-    |> URI.to_string()
-  end
-end
-- 
cgit v1.2.3


From 8bd2b6eb138ace3408a03c78ecc339fc35b19f10 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 4 Sep 2020 14:24:15 +0300
Subject: temp hackney fix

---
 lib/pleroma/http/adapter_helper/hackney.ex | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex
index f47a671ad..42e3acfec 100644
--- a/lib/pleroma/http/adapter_helper/hackney.ex
+++ b/lib/pleroma/http/adapter_helper/hackney.ex
@@ -22,5 +22,9 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
   end
 
+  defp add_scheme_opts(opts, %URI{scheme: "https"}) do
+    Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
+  end
+
   defp add_scheme_opts(opts, _), do: opts
 end
-- 
cgit v1.2.3


From 473458b0fbecb05121b235f525aefcef34f0409e Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 4 Sep 2020 14:45:30 +0300
Subject: fix for ReverseProxy

---
 lib/pleroma/reverse_proxy/client/hackney.ex | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex
index e84118a90..ad988fac3 100644
--- a/lib/pleroma/reverse_proxy/client/hackney.ex
+++ b/lib/pleroma/reverse_proxy/client/hackney.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
 
   @impl true
   def request(method, url, headers, body, opts \\ []) do
+    opts = Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
     :hackney.request(method, url, headers, body, opts)
   end
 
-- 
cgit v1.2.3


From 10da13c71343623a5e52beebdc6abc1f400bc40d Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 4 Sep 2020 22:10:40 +0300
Subject: ConnectionPool middleware: Fix connection leak on ReverseProxy
 redirects

Requires a patched Tesla due to upstream not saving opts between
redirects, patch submitted at https://github.com/teamon/tesla/pull/414
---
 lib/pleroma/tesla/middleware/connection_pool.ex | 27 +++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex
index a435ab4cc..5909e98d6 100644
--- a/lib/pleroma/tesla/middleware/connection_pool.ex
+++ b/lib/pleroma/tesla/middleware/connection_pool.ex
@@ -15,21 +15,32 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
   def call(%Tesla.Env{url: url, opts: opts} = env, next, _) do
     uri = URI.parse(url)
 
+    # Avoid leaking connections when the middleware is called twice
+    # with body_as: :chunks. We assume only the middleware can set
+    # opts[:adapter][:conn]
+    if opts[:adapter][:conn] do
+      ConnectionPool.release_conn(opts[:adapter][:conn])
+    end
+
     case ConnectionPool.get_conn(uri, opts[:adapter]) do
       {:ok, conn_pid} ->
         adapter_opts = Keyword.merge(opts[:adapter], conn: conn_pid, close_conn: false)
         opts = Keyword.put(opts, :adapter, adapter_opts)
         env = %{env | opts: opts}
-        res = Tesla.run(env, next)
 
-        unless opts[:adapter][:body_as] == :chunks do
-          ConnectionPool.release_conn(conn_pid)
+        case Tesla.run(env, next) do
+          {:ok, env} ->
+            unless opts[:adapter][:body_as] == :chunks do
+              ConnectionPool.release_conn(conn_pid)
+              {:ok, pop_in(env[:opts][:adapter][:conn])}
+            else
+              {:ok, env}
+            end
+
+          err ->
+            ConnectionPool.release_conn(conn_pid)
+            err
         end
-
-        res
-
-      err ->
-        err
     end
   end
 end
-- 
cgit v1.2.3


From 0d91f65284f99bded89c0400e976e0ffa5bc202f Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 4 Sep 2020 07:52:22 +0200
Subject: Prevent AccountView and instance.get_or_update_favicon fails

---
 lib/pleroma/instances/instance.ex | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 711c42158..ef5d17de4 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -145,6 +145,8 @@ defmodule Pleroma.Instances.Instance do
 
       favicon
     end
+  rescue
+    _ -> nil
   end
 
   defp scrape_favicon(%URI{} = instance_uri) do
@@ -159,7 +161,8 @@ defmodule Pleroma.Instances.Instance do
              |> Floki.attribute("link[rel=icon]", "href")
              |> List.first(),
            favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(),
-           true <- is_binary(favicon) do
+           true <- is_binary(favicon),
+           true <- String.length(favicon) <= 255 do
         favicon
       else
         _ -> nil
-- 
cgit v1.2.3


From c3b02341bf4ab610e9425d6811dca057e9f811a4 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 5 Sep 2020 16:16:35 +0300
Subject: [#2497] Made media preview proxy fall back to media proxy instead of
 to source url. Adjusted tests. Refactoring.

---
 lib/pleroma/helpers/media_helper.ex                |  6 ++-
 lib/pleroma/web/media_proxy/media_proxy.ex         |  4 +-
 .../web/media_proxy/media_proxy_controller.ex      | 50 ++++++++++++----------
 3 files changed, 32 insertions(+), 28 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index cfb091f82..bb93d4915 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -7,12 +7,14 @@ defmodule Pleroma.Helpers.MediaHelper do
   Handles common media-related operations.
   """
 
+  alias Pleroma.HTTP
+
   @tmp_base "/tmp/pleroma-media_preview-pipe"
 
   def image_resize(url, options) do
     with executable when is_binary(executable) <- System.find_executable("convert"),
          {:ok, args} <- prepare_image_resize_args(options),
-         {:ok, env} <- Pleroma.HTTP.get(url, [], [adapter: [pool: :preview]]),
+         {:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]),
          {:ok, fifo_path} <- mkfifo() do
       args = List.flatten([fifo_path, args])
       run_fifo(fifo_path, env, executable, args)
@@ -60,7 +62,7 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
-         {:ok, env} <- Pleroma.HTTP.get(url, [], [adapter: [pool: :preview]]),
+         {:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]),
          {:ok, fifo_path} <- mkfifo(),
          args = [
            "-y",
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index 4cbe1cf89..80017cde1 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -57,13 +57,11 @@ defmodule Pleroma.Web.MediaProxy do
     end
   end
 
-  # Note: routing all URLs to preview handler (even local and whitelisted).
-  #   Preview handler will call url/1 on decoded URLs, and applicable ones will detour media proxy.
   def preview_url(url, preview_params \\ []) do
     if preview_enabled?() do
       encode_preview_url(url, preview_params)
     else
-      url
+      url(url)
     end
   end
 
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 33daa1e05..469fbae59 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -48,10 +48,12 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   end
 
   defp handle_preview(conn, url) do
+    media_proxy_url = MediaProxy.url(url)
+
     with {:ok, %{status: status} = head_response} when status in 200..299 <-
-           Pleroma.HTTP.request("head", MediaProxy.url(url), [], [], [adapter: [pool: :preview]]) do
+           Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :preview]) do
       content_type = Tesla.get_header(head_response, "content-type")
-      handle_preview(content_type, conn, url)
+      handle_preview(content_type, conn, media_proxy_url)
     else
       {_, %{status: status}} ->
         send_resp(conn, :failed_dependency, "Can't fetch HTTP headers (HTTP #{status}).")
@@ -67,40 +69,38 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   defp handle_preview(
          "image/" <> _ = _content_type,
          %{params: %{"output_format" => "jpeg"}} = conn,
-         url
+         media_proxy_url
        ) do
-    handle_jpeg_preview(conn, url)
+    handle_jpeg_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview("image/gif" = _content_type, conn, url) do
-    mediaproxy_url = url |> MediaProxy.url()
-
-    redirect(conn, external: mediaproxy_url)
+  defp handle_preview("image/gif" = _content_type, conn, media_proxy_url) do
+    redirect(conn, external: media_proxy_url)
   end
 
-  defp handle_preview("image/png" <> _ = _content_type, conn, url) do
-    handle_png_preview(conn, url)
+  defp handle_preview("image/png" <> _ = _content_type, conn, media_proxy_url) do
+    handle_png_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview("image/" <> _ = _content_type, conn, url) do
-    handle_jpeg_preview(conn, url)
+  defp handle_preview("image/" <> _ = _content_type, conn, media_proxy_url) do
+    handle_jpeg_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview("video/" <> _ = _content_type, conn, url) do
-    handle_video_preview(conn, url)
+  defp handle_preview("video/" <> _ = _content_type, conn, media_proxy_url) do
+    handle_video_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview(content_type, conn, _url) do
+  defp handle_preview(content_type, conn, _media_proxy_url) do
     send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
   end
 
-  defp handle_png_preview(%{params: params} = conn, url) do
+  defp handle_png_preview(%{params: params} = conn, media_proxy_url) do
     quality = Config.get!([:media_preview_proxy, :image_quality])
 
     with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
          {:ok, thumbnail_binary} <-
            MediaHelper.image_resize(
-             url,
+             media_proxy_url,
              %{
                max_width: thumbnail_max_width,
                max_height: thumbnail_max_height,
@@ -109,7 +109,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
              }
            ) do
       conn
-      |> put_preview_response_headers("image/png", "preview.png")
+      |> put_preview_response_headers(["image/png", "preview.png"])
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
@@ -117,13 +117,13 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp handle_jpeg_preview(%{params: params} = conn, url) do
+  defp handle_jpeg_preview(%{params: params} = conn, media_proxy_url) do
     quality = Config.get!([:media_preview_proxy, :image_quality])
 
     with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
          {:ok, thumbnail_binary} <-
            MediaHelper.image_resize(
-             url,
+             media_proxy_url,
              %{max_width: thumbnail_max_width, max_height: thumbnail_max_height, quality: quality}
            ) do
       conn
@@ -135,9 +135,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp handle_video_preview(conn, url) do
+  defp handle_video_preview(conn, media_proxy_url) do
     with {:ok, thumbnail_binary} <-
-           MediaHelper.video_framegrab(url) do
+           MediaHelper.video_framegrab(media_proxy_url) do
       conn
       |> put_preview_response_headers()
       |> send_resp(200, thumbnail_binary)
@@ -147,10 +147,14 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp put_preview_response_headers(conn, content_type \\ "image/jpeg", filename \\ "preview.jpg") do
+  defp put_preview_response_headers(
+         conn,
+         [content_type, filename] = _content_info \\ ["image/jpeg", "preview.jpg"]
+       ) do
     conn
     |> put_resp_header("content-type", content_type)
     |> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"")
+    # TODO: enable caching
     |> put_resp_header("cache-control", "max-age=0, private, must-revalidate")
   end
 
-- 
cgit v1.2.3


From f170d471307ba0082b98351190b3d6b808bdfe1a Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 5 Sep 2020 20:19:09 +0300
Subject: [#2497] Adjusted media proxy preview invalidation. Allowed
 client-side caching for media preview. Adjusted prewarmer to fetch only
 proxiable URIs. Removed :preview pool in favor of existing :media one. Misc.
 refactoring.

---
 lib/pleroma/helpers/media_helper.ex                |  4 ++--
 lib/pleroma/reverse_proxy/reverse_proxy.ex         |  1 +
 .../activity_pub/mrf/media_proxy_warming_policy.ex | 27 +++++++++++++---------
 lib/pleroma/web/media_proxy/invalidation.ex        |  4 +++-
 lib/pleroma/web/media_proxy/media_proxy.ex         | 20 ++++++++--------
 .../web/media_proxy/media_proxy_controller.ex      |  5 ++--
 6 files changed, 34 insertions(+), 27 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index bb93d4915..a1205e10d 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   def image_resize(url, options) do
     with executable when is_binary(executable) <- System.find_executable("convert"),
          {:ok, args} <- prepare_image_resize_args(options),
-         {:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]),
+         {:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]),
          {:ok, fifo_path} <- mkfifo() do
       args = List.flatten([fifo_path, args])
       run_fifo(fifo_path, env, executable, args)
@@ -62,7 +62,7 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
-         {:ok, env} <- HTTP.get(url, [], adapter: [pool: :preview]),
+         {:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]),
          {:ok, fifo_path} <- mkfifo(),
          args = [
            "-y",
diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex
index 35637e934..8ae1157df 100644
--- a/lib/pleroma/reverse_proxy/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex
@@ -18,6 +18,7 @@ defmodule Pleroma.ReverseProxy do
   @methods ~w(GET HEAD)
 
   def max_read_duration_default, do: @max_read_duration
+  def default_cache_control_header, do: @default_cache_control_header
 
   @moduledoc """
   A reverse proxy.
diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index 5d8bb72aa..1050b74ba 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -12,23 +12,28 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
   require Logger
 
-  @options [
+  @adapter_options [
     pool: :media
   ]
 
   def perform(:prefetch, url) do
-    Logger.debug("Prefetching #{inspect(url)}")
+    # Fetching only proxiable resources
+    if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
+      # If preview proxy is enabled, it'll also hit media proxy (so we're caching both requests)
+      prefetch_url = MediaProxy.preview_url(url)
 
-    opts =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.put(@options, :recv_timeout, 10_000)
-      else
-        @options
-      end
+      Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
 
-    url
-    |> MediaProxy.preview_url()
-    |> HTTP.get([], adapter: opts)
+      HTTP.get(prefetch_url, [], adapter: adapter_options())
+    end
+  end
+
+  defp adapter_options do
+    if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+      Keyword.put(@adapter_options, :recv_timeout, 10_000)
+    else
+      @adapter_options
+    end
   end
 
   def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex
index 5808861e6..4f4340478 100644
--- a/lib/pleroma/web/media_proxy/invalidation.ex
+++ b/lib/pleroma/web/media_proxy/invalidation.ex
@@ -33,6 +33,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do
   def prepare_urls(urls) do
     urls
     |> List.wrap()
-    |> Enum.map(&MediaProxy.url/1)
+    |> Enum.map(fn url -> [MediaProxy.url(url), MediaProxy.preview_url(url)] end)
+    |> List.flatten()
+    |> Enum.uniq()
   end
 end
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index 80017cde1..ba553998b 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -41,20 +41,16 @@ defmodule Pleroma.Web.MediaProxy do
   def url("/" <> _ = url), do: url
 
   def url(url) do
-    if not enabled?() or not url_proxiable?(url) do
-      url
-    else
+    if enabled?() and url_proxiable?(url) do
       encode_url(url)
+    else
+      url
     end
   end
 
   @spec url_proxiable?(String.t()) :: boolean()
   def url_proxiable?(url) do
-    if local?(url) or whitelisted?(url) do
-      false
-    else
-      true
-    end
+    not local?(url) and not whitelisted?(url)
   end
 
   def preview_url(url, preview_params \\ []) do
@@ -69,7 +65,7 @@ defmodule Pleroma.Web.MediaProxy do
 
   # Note: media proxy must be enabled for media preview proxy in order to load all
   #   non-local non-whitelisted URLs through it and be sure that body size constraint is preserved.
-  def preview_enabled?, do: enabled?() and Config.get([:media_preview_proxy, :enabled], false)
+  def preview_enabled?, do: enabled?() and !!Config.get([:media_preview_proxy, :enabled])
 
   def local?(url), do: String.starts_with?(url, Pleroma.Web.base_url())
 
@@ -138,9 +134,13 @@ defmodule Pleroma.Web.MediaProxy do
     if path = URI.parse(url_or_path).path, do: Path.basename(path)
   end
 
+  def base_url do
+    Config.get([:media_proxy, :base_url], Web.base_url())
+  end
+
   defp proxy_url(path, sig_base64, url_base64, filename) do
     [
-      Config.get([:media_proxy, :base_url], Web.base_url()),
+      base_url(),
       path,
       sig_base64,
       url_base64,
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 469fbae59..89f4a23bd 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -51,7 +51,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     media_proxy_url = MediaProxy.url(url)
 
     with {:ok, %{status: status} = head_response} when status in 200..299 <-
-           Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :preview]) do
+           Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :media]) do
       content_type = Tesla.get_header(head_response, "content-type")
       handle_preview(content_type, conn, media_proxy_url)
     else
@@ -154,8 +154,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     conn
     |> put_resp_header("content-type", content_type)
     |> put_resp_header("content-disposition", "inline; filename=\"#{filename}\"")
-    # TODO: enable caching
-    |> put_resp_header("cache-control", "max-age=0, private, must-revalidate")
+    |> put_resp_header("cache-control", ReverseProxy.default_cache_control_header())
   end
 
   defp thumbnail_max_dimensions(params) do
-- 
cgit v1.2.3


From 88a6ee4a5989036de5c1e82c6111291887597d98 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 5 Sep 2020 20:23:18 +0300
Subject: [#2497] Func defs grouping fix.

---
 .../web/activity_pub/mrf/media_proxy_warming_policy.ex   | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index 1050b74ba..6c63fe15c 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -16,6 +16,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
     pool: :media
   ]
 
+  defp adapter_options do
+    if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
+      Keyword.put(@adapter_options, :recv_timeout, 10_000)
+    else
+      @adapter_options
+    end
+  end
+
   def perform(:prefetch, url) do
     # Fetching only proxiable resources
     if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
@@ -28,14 +36,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
     end
   end
 
-  defp adapter_options do
-    if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-      Keyword.put(@adapter_options, :recv_timeout, 10_000)
-    else
-      @adapter_options
-    end
-  end
-
   def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
     Enum.each(attachments, fn
       %{"url" => url} when is_list(url) ->
-- 
cgit v1.2.3


From e198ba492e5cb1b6ff81775db08298bfcdf1454a Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 5 Sep 2020 12:37:27 +0300
Subject: Rich Media: Do not cache URLs for preview statuses

Closes #1987
---
 lib/pleroma/html.ex                   | 34 ++++++++++++++++++++--------------
 lib/pleroma/web/rich_media/helpers.ex |  2 +-
 2 files changed, 21 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index 20b02f091..43e9145be 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -100,21 +100,27 @@ defmodule Pleroma.HTML do
     end)
   end
 
-  def extract_first_external_url(_, nil), do: {:error, "No content"}
+  def extract_first_external_url_from_object(%{data: %{"content" => content}} = object)
+      when is_binary(content) do
+    unless object.data["fake"] do
+      key = "URL|#{object.id}"
+
+      Cachex.fetch!(:scrubber_cache, key, fn _key ->
+        {:commit, {:ok, extract_first_external_url(content)}}
+      end)
+    else
+      {:ok, extract_first_external_url(content)}
+    end
+  end
 
-  def extract_first_external_url(object, content) do
-    key = "URL|#{object.id}"
+  def extract_first_external_url_from_object(_), do: {:error, :no_content}
 
-    Cachex.fetch!(:scrubber_cache, key, fn _key ->
-      result =
-        content
-        |> Floki.parse_fragment!()
-        |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
-        |> Enum.take(1)
-        |> Floki.attribute("href")
-        |> Enum.at(0)
-
-      {:commit, {:ok, result}}
-    end)
+  def extract_first_external_url(content) do
+    content
+    |> Floki.parse_fragment!()
+    |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
+    |> Enum.take(1)
+    |> Floki.attribute("href")
+    |> Enum.at(0)
   end
 end
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 2fb482b51..752ca9f81 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -58,7 +58,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
     with true <- Config.get([:rich_media, :enabled]),
          false <- object.data["sensitive"] || false,
          {:ok, page_url} <-
-           HTML.extract_first_external_url(object, object.data["content"]),
+           HTML.extract_first_external_url_from_object(object),
          :ok <- validate_page_url(page_url),
          {:ok, rich_media} <- Parser.parse(page_url) do
       %{page_url: page_url, rich_media: rich_media}
-- 
cgit v1.2.3


From 5298de3be6683022fa53cc011dd567e8b2a706b9 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 5 Sep 2020 21:17:03 +0300
Subject: ConnectionPool middleware: fix a crash due to unimplemented behaviour

Structs don't implement Access behaviour, so this crashed. Tests didn't
catch it and I didn't test that part of the codepath. Very sorry
---
 lib/pleroma/tesla/middleware/connection_pool.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex
index 5909e98d6..049db6eb3 100644
--- a/lib/pleroma/tesla/middleware/connection_pool.ex
+++ b/lib/pleroma/tesla/middleware/connection_pool.ex
@@ -32,7 +32,7 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
           {:ok, env} ->
             unless opts[:adapter][:body_as] == :chunks do
               ConnectionPool.release_conn(conn_pid)
-              {:ok, pop_in(env[:opts][:adapter][:conn])}
+              {:ok, pop_in(env.opts[:adapter][:conn])}
             else
               {:ok, env}
             end
-- 
cgit v1.2.3


From 9d6aca5bee6f90f3c0af5a5353f052108c9def62 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 5 Sep 2020 21:27:06 +0300
Subject: ConnectionPool: fix the previous hotfix

I rushed the hotfix and forgot how `pop_in` actually works,
I want to die. We need some integration tests for the HTTP client
---
 lib/pleroma/tesla/middleware/connection_pool.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex
index 049db6eb3..2c5a2b53b 100644
--- a/lib/pleroma/tesla/middleware/connection_pool.ex
+++ b/lib/pleroma/tesla/middleware/connection_pool.ex
@@ -32,7 +32,8 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
           {:ok, env} ->
             unless opts[:adapter][:body_as] == :chunks do
               ConnectionPool.release_conn(conn_pid)
-              {:ok, pop_in(env.opts[:adapter][:conn])}
+              {_, res} = pop_in(env.opts[:adapter][:conn])
+              {:ok, res}
             else
               {:ok, env}
             end
-- 
cgit v1.2.3


From 129a2f48df95ddd85fceee741a9991a6e092ed3d Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 5 Sep 2020 21:36:17 +0300
Subject: ConnectionPool middleware: handle connection opening errors

---
 lib/pleroma/tesla/middleware/connection_pool.ex | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/tesla/middleware/connection_pool.ex b/lib/pleroma/tesla/middleware/connection_pool.ex
index 2c5a2b53b..056e736ce 100644
--- a/lib/pleroma/tesla/middleware/connection_pool.ex
+++ b/lib/pleroma/tesla/middleware/connection_pool.ex
@@ -42,6 +42,9 @@ defmodule Pleroma.Tesla.Middleware.ConnectionPool do
             ConnectionPool.release_conn(conn_pid)
             err
         end
+
+      err ->
+        err
     end
   end
 end
-- 
cgit v1.2.3


From 170599c390e7c82bdff0d4180d04b2f0f3906f35 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 5 Sep 2020 22:00:51 +0300
Subject: RichMedia: do not log webpages missing metadata as errors

Also fixes the return value of Parser.parse on errors, previously
was just `:ok` due to the logger call in the end
---
 lib/pleroma/web/rich_media/parser.ex | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index e98c743ca..5727fda18 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -21,8 +21,13 @@ defmodule Pleroma.Web.RichMedia.Parser do
            {:ok, _} <- set_ttl_based_on_image(data, url) do
         {:ok, data}
       else
+        {:error, {:invalid_metadata, data}} = e ->
+          Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
+          e
+
         error ->
-          Logger.error(fn -> "Rich media error: #{inspect(error)}" end)
+          Logger.error(fn -> "Rich media error for #{url}: #{inspect(error)}" end)
+          error
       end
     end
 
@@ -90,7 +95,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
     end)
   end
 
-  defp parse_url(url) do
+  def parse_url(url) do
     with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
          {:ok, html} <- Floki.parse_document(html) do
       html
@@ -116,7 +121,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
   end
 
   defp check_parsed_data(data) do
-    {:error, "Found metadata was invalid or incomplete: #{inspect(data)}"}
+    {:error, {:invalid_metadata, data}}
   end
 
   defp clean_parsed_data(data) do
-- 
cgit v1.2.3


From 5ae56aafb2edc737f7e9fb36e00377815f028ce6 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Sun, 6 Sep 2020 21:42:51 +0300
Subject: added import mutes

---
 lib/pleroma/user.ex                                | 51 ------------
 lib/pleroma/user/import.ex                         | 91 ++++++++++++++++++++++
 .../controllers/user_import_controller.ex          | 57 ++++++++++++++
 lib/pleroma/web/router.ex                          |  7 +-
 .../web/twitter_api/controllers/util_controller.ex | 35 ---------
 lib/pleroma/workers/background_worker.ex           | 11 ++-
 6 files changed, 161 insertions(+), 91 deletions(-)
 create mode 100644 lib/pleroma/user/import.ex
 create mode 100644 lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 94c96de8d..be2ef0d1b 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1686,42 +1686,6 @@ defmodule Pleroma.User do
 
   def perform(:deactivate_async, user, status), do: deactivate(user, status)
 
-  @spec perform(atom(), User.t(), list()) :: list() | {:error, any()}
-  def perform(:blocks_import, %User{} = blocker, blocked_identifiers)
-      when is_list(blocked_identifiers) do
-    Enum.map(
-      blocked_identifiers,
-      fn blocked_identifier ->
-        with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier),
-             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
-          blocked
-        else
-          err ->
-            Logger.debug("blocks_import failed for #{blocked_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
-  def perform(:follow_import, %User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    Enum.map(
-      followed_identifiers,
-      fn followed_identifier ->
-        with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier),
-             {:ok, follower} <- maybe_direct_follow(follower, followed),
-             {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
-          followed
-        else
-          err ->
-            Logger.debug("follow_import failed for #{followed_identifier} with: #{inspect(err)}")
-            err
-        end
-      end
-    )
-  end
-
   @spec external_users_query() :: Ecto.Query.t()
   def external_users_query do
     User.Query.build(%{
@@ -1750,21 +1714,6 @@ defmodule Pleroma.User do
     Repo.all(query)
   end
 
-  def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do
-    BackgroundWorker.enqueue("blocks_import", %{
-      "blocker_id" => blocker.id,
-      "blocked_identifiers" => blocked_identifiers
-    })
-  end
-
-  def follow_import(%User{} = follower, followed_identifiers)
-      when is_list(followed_identifiers) do
-    BackgroundWorker.enqueue("follow_import", %{
-      "follower_id" => follower.id,
-      "followed_identifiers" => followed_identifiers
-    })
-  end
-
   def delete_notifications_from_user_activities(%User{ap_id: ap_id}) do
     Notification
     |> join(:inner, [n], activity in assoc(n, :activity))
diff --git a/lib/pleroma/user/import.ex b/lib/pleroma/user/import.ex
new file mode 100644
index 000000000..de27bdc4c
--- /dev/null
+++ b/lib/pleroma/user/import.ex
@@ -0,0 +1,91 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.Import do
+  use Ecto.Schema
+
+  alias Pleroma.User
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Workers.BackgroundWorker
+
+  require Logger
+
+  @spec perform(atom(), User.t(), list()) :: :ok | list() | {:error, any()}
+  def perform(:mutes_import, %User{} = user, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = muted_user} <- User.get_or_fetch(identifier),
+             {:ok, _} <- User.mute(user, muted_user) do
+          muted_user
+        else
+          error -> handle_error(:mutes_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(:blocks_import, %User{} = blocker, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = blocked} <- User.get_or_fetch(identifier),
+             {:ok, _block} <- CommonAPI.block(blocker, blocked) do
+          blocked
+        else
+          error -> handle_error(:blocks_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(:follow_import, %User{} = follower, [_ | _] = identifiers) do
+    Enum.map(
+      identifiers,
+      fn identifier ->
+        with {:ok, %User{} = followed} <- User.get_or_fetch(identifier),
+             {:ok, follower} <- User.maybe_direct_follow(follower, followed),
+             {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do
+          followed
+        else
+          error -> handle_error(:follow_import, identifier, error)
+        end
+      end
+    )
+  end
+
+  def perform(_, _, _), do: :ok
+
+  defp handle_error(op, user_id, error) do
+    Logger.debug("#{op} failed for #{user_id} with: #{inspect(error)}")
+    error
+  end
+
+  def blocks_import(%User{} = blocker, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "blocks_import",
+      %{
+        "blocker_id" => blocker.id,
+        "blocked_identifiers" => identifiers
+      }
+    )
+  end
+
+  def follow_import(%User{} = follower, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "follow_import",
+      %{
+        "follower_id" => follower.id,
+        "followed_identifiers" => identifiers
+      }
+    )
+  end
+
+  def mutes_import(%User{} = user, [_ | _] = identifiers) do
+    BackgroundWorker.enqueue(
+      "mutes_import",
+      %{"user_id" => user.id, "identifiers" => identifiers}
+    )
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex
new file mode 100644
index 000000000..df6a0f131
--- /dev/null
+++ b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex
@@ -0,0 +1,57 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.UserImportController do
+  use Pleroma.Web, :controller
+
+  require Logger
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.User
+
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:follows"]} when action == :follow)
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks)
+  plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action == :mutes)
+
+  def follow(conn, %{"list" => %Plug.Upload{path: path}}) do
+    follow(conn, %{"list" => File.read!(path)})
+  end
+
+  def follow(%{assigns: %{user: follower}} = conn, %{"list" => list}) do
+    identifiers =
+      list
+      |> String.split("\n")
+      |> Enum.map(&(&1 |> String.split(",") |> List.first()))
+      |> List.delete("Account address")
+      |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@")))
+      |> Enum.reject(&(&1 == ""))
+
+    User.Import.follow_import(follower, identifiers)
+    json(conn, "job started")
+  end
+
+  def blocks(conn, %{"list" => %Plug.Upload{path: path}}) do
+    blocks(conn, %{"list" => File.read!(path)})
+  end
+
+  def blocks(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do
+    User.Import.blocks_import(blocker, prepare_user_identifiers(list))
+    json(conn, "job started")
+  end
+
+  def mutes(conn, %{"list" => %Plug.Upload{path: path}}) do
+    mutes(conn, %{"list" => File.read!(path)})
+  end
+
+  def mutes(%{assigns: %{user: user}} = conn, %{"list" => list}) do
+    User.Import.mutes_import(user, prepare_user_identifiers(list))
+    json(conn, "job started")
+  end
+
+  defp prepare_user_identifiers(list) do
+    list
+    |> String.split()
+    |> Enum.map(&String.trim_leading(&1, "@"))
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index c6433cc53..f69b1545f 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -260,14 +260,15 @@ defmodule Pleroma.Web.Router do
     post("/delete_account", UtilController, :delete_account)
     put("/notification_settings", UtilController, :update_notificaton_settings)
     post("/disable_account", UtilController, :disable_account)
-
-    post("/blocks_import", UtilController, :blocks_import)
-    post("/follow_import", UtilController, :follow_import)
   end
 
   scope "/api/pleroma", Pleroma.Web.PleromaAPI do
     pipe_through(:authenticated_api)
 
+    post("/mutes_import", UserImportController, :mutes)
+    post("/blocks_import", UserImportController, :blocks)
+    post("/follow_import", UserImportController, :follow)
+
     get("/accounts/mfa", TwoFactorAuthenticationController, :settings)
     get("/accounts/mfa/backup_codes", TwoFactorAuthenticationController, :backup_codes)
     get("/accounts/mfa/setup/:method", TwoFactorAuthenticationController, :setup)
diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
index f02c4075c..70b0fbd54 100644
--- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex
@@ -18,14 +18,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
 
   plug(Pleroma.Web.FederatingPlug when action == :remote_subscribe)
 
-  plug(
-    OAuthScopesPlug,
-    %{scopes: ["follow", "write:follows"]}
-    when action == :follow_import
-  )
-
-  plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks_import)
-
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:accounts"]}
@@ -104,33 +96,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do
     end
   end
 
-  def follow_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
-    follow_import(conn, %{"list" => File.read!(listfile.path)})
-  end
-
-  def follow_import(%{assigns: %{user: follower}} = conn, %{"list" => list}) do
-    followed_identifiers =
-      list
-      |> String.split("\n")
-      |> Enum.map(&(&1 |> String.split(",") |> List.first()))
-      |> List.delete("Account address")
-      |> Enum.map(&(&1 |> String.trim() |> String.trim_leading("@")))
-      |> Enum.reject(&(&1 == ""))
-
-    User.follow_import(follower, followed_identifiers)
-    json(conn, "job started")
-  end
-
-  def blocks_import(conn, %{"list" => %Plug.Upload{} = listfile}) do
-    blocks_import(conn, %{"list" => File.read!(listfile.path)})
-  end
-
-  def blocks_import(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do
-    blocked_identifiers = list |> String.split() |> Enum.map(&String.trim_leading(&1, "@"))
-    User.blocks_import(blocker, blocked_identifiers)
-    json(conn, "job started")
-  end
-
   def change_password(%{assigns: %{user: user}} = conn, params) do
     case CommonAPI.Utils.confirm_current_password(user, params["password"]) do
       {:ok, user} ->
diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex
index cec5a7462..f9c767ee0 100644
--- a/lib/pleroma/workers/background_worker.ex
+++ b/lib/pleroma/workers/background_worker.ex
@@ -34,7 +34,7 @@ defmodule Pleroma.Workers.BackgroundWorker do
         }
       }) do
     blocker = User.get_cached_by_id(blocker_id)
-    {:ok, User.perform(:blocks_import, blocker, blocked_identifiers)}
+    {:ok, User.Import.perform(:blocks_import, blocker, blocked_identifiers)}
   end
 
   def perform(%Job{
@@ -45,7 +45,14 @@ defmodule Pleroma.Workers.BackgroundWorker do
         }
       }) do
     follower = User.get_cached_by_id(follower_id)
-    {:ok, User.perform(:follow_import, follower, followed_identifiers)}
+    {:ok, User.Import.perform(:follow_import, follower, followed_identifiers)}
+  end
+
+  def perform(%Job{
+        args: %{"op" => "mutes_import", "user_id" => user_id, "identifiers" => identifiers}
+      }) do
+    user = User.get_cached_by_id(user_id)
+    {:ok, User.Import.perform(:mutes_import, user, identifiers)}
   end
 
   def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do
-- 
cgit v1.2.3


From 917d325972e3aeb367583c61aaa109d62fcba837 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 7 Sep 2020 07:17:30 +0300
Subject: added api spec

---
 .../api_spec/operations/user_import_operation.ex   | 80 ++++++++++++++++++++++
 .../controllers/user_import_controller.ex          | 22 +++---
 2 files changed, 93 insertions(+), 9 deletions(-)
 create mode 100644 lib/pleroma/web/api_spec/operations/user_import_operation.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/operations/user_import_operation.ex b/lib/pleroma/web/api_spec/operations/user_import_operation.ex
new file mode 100644
index 000000000..a50314fb7
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/user_import_operation.ex
@@ -0,0 +1,80 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.UserImportOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  @spec open_api_operation(atom) :: Operation.t()
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def follow_operation do
+    %Operation{
+      tags: ["follow_import"],
+      summary: "Imports your follows.",
+      operationId: "UserImportController.follow",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:follow"]}]
+    }
+  end
+
+  def blocks_operation do
+    %Operation{
+      tags: ["blocks_import"],
+      summary: "Imports your blocks.",
+      operationId: "UserImportController.blocks",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:blocks"]}]
+    }
+  end
+
+  def mutes_operation do
+    %Operation{
+      tags: ["mutes_import"],
+      summary: "Imports your mutes.",
+      operationId: "UserImportController.mutes",
+      requestBody: request_body("Parameters", import_request(), required: true),
+      responses: %{
+        200 => ok_response(),
+        500 => Operation.response("Error", "application/json", ApiError)
+      },
+      security: [%{"oAuth" => ["write:mutes"]}]
+    }
+  end
+
+  defp import_request do
+    %Schema{
+      type: :object,
+      required: [:list],
+      properties: %{
+        list: %Schema{
+          description:
+            "STRING or FILE containing a whitespace-separated list of accounts to import.",
+          anyOf: [
+            %Schema{type: :string, format: :binary},
+            %Schema{type: :string}
+          ]
+        }
+      }
+    }
+  end
+
+  defp ok_response do
+    Operation.response("Ok", "application/json", %Schema{type: :string, example: "ok"})
+  end
+end
diff --git a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex
index df6a0f131..f10c45750 100644
--- a/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/user_import_controller.ex
@@ -9,16 +9,20 @@ defmodule Pleroma.Web.PleromaAPI.UserImportController do
 
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
+  alias Pleroma.Web.ApiSpec
 
   plug(OAuthScopesPlug, %{scopes: ["follow", "write:follows"]} when action == :follow)
   plug(OAuthScopesPlug, %{scopes: ["follow", "write:blocks"]} when action == :blocks)
   plug(OAuthScopesPlug, %{scopes: ["follow", "write:mutes"]} when action == :mutes)
 
-  def follow(conn, %{"list" => %Plug.Upload{path: path}}) do
-    follow(conn, %{"list" => File.read!(path)})
+  plug(OpenApiSpex.Plug.CastAndValidate)
+  defdelegate open_api_operation(action), to: ApiSpec.UserImportOperation
+
+  def follow(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    follow(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
   end
 
-  def follow(%{assigns: %{user: follower}} = conn, %{"list" => list}) do
+  def follow(%{assigns: %{user: follower}, body_params: %{list: list}} = conn, _) do
     identifiers =
       list
       |> String.split("\n")
@@ -31,20 +35,20 @@ defmodule Pleroma.Web.PleromaAPI.UserImportController do
     json(conn, "job started")
   end
 
-  def blocks(conn, %{"list" => %Plug.Upload{path: path}}) do
-    blocks(conn, %{"list" => File.read!(path)})
+  def blocks(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    blocks(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
   end
 
-  def blocks(%{assigns: %{user: blocker}} = conn, %{"list" => list}) do
+  def blocks(%{assigns: %{user: blocker}, body_params: %{list: list}} = conn, _) do
     User.Import.blocks_import(blocker, prepare_user_identifiers(list))
     json(conn, "job started")
   end
 
-  def mutes(conn, %{"list" => %Plug.Upload{path: path}}) do
-    mutes(conn, %{"list" => File.read!(path)})
+  def mutes(%{body_params: %{list: %Plug.Upload{path: path}}} = conn, _) do
+    mutes(%Plug.Conn{conn | body_params: %{list: File.read!(path)}}, %{})
   end
 
-  def mutes(%{assigns: %{user: user}} = conn, %{"list" => list}) do
+  def mutes(%{assigns: %{user: user}, body_params: %{list: list}} = conn, _) do
     User.Import.mutes_import(user, prepare_user_identifiers(list))
     json(conn, "job started")
   end
-- 
cgit v1.2.3


From 08aef7dd4e054c5ed02e359b61fe57daad97fbde Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 5 Sep 2020 06:38:07 +0200
Subject: instance: Log catch favicon errors as warnings

---
 lib/pleroma/instances/instance.ex | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index ef5d17de4..8bf53c090 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -14,6 +14,8 @@ defmodule Pleroma.Instances.Instance do
   import Ecto.Query
   import Ecto.Changeset
 
+  require Logger
+
   schema "instances" do
     field(:host, :string)
     field(:unreachable_since, :naive_datetime_usec)
@@ -146,7 +148,9 @@ defmodule Pleroma.Instances.Instance do
       favicon
     end
   rescue
-    _ -> nil
+    e ->
+      Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
+      nil
   end
 
   defp scrape_favicon(%URI{} = instance_uri) do
@@ -161,14 +165,18 @@ defmodule Pleroma.Instances.Instance do
              |> Floki.attribute("link[rel=icon]", "href")
              |> List.first(),
            favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(),
-           true <- is_binary(favicon),
-           true <- String.length(favicon) <= 255 do
+           true <- is_binary(favicon) do
         favicon
       else
         _ -> nil
       end
     rescue
-      _ -> nil
+      e ->
+        Logger.warn(
+          "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
+        )
+
+        nil
     end
   end
 end
-- 
cgit v1.2.3


From ee67c98e550310813cfdb9242e5fab2e566e1e2a Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sun, 6 Sep 2020 12:13:26 +0300
Subject: removing Stats worker from Oban cron jobs

---
 lib/mix/pleroma.ex                       |  1 +
 lib/pleroma/application.ex               |  1 +
 lib/pleroma/config/oban.ex               | 30 +++++++++++++
 lib/pleroma/stats.ex                     | 76 +++++++++++++++++++++++---------
 lib/pleroma/workers/cron/stats_worker.ex | 17 -------
 5 files changed, 87 insertions(+), 38 deletions(-)
 create mode 100644 lib/pleroma/config/oban.ex
 delete mode 100644 lib/pleroma/workers/cron/stats_worker.ex

(limited to 'lib')

diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index fe9b0d16c..49ba2aae4 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -18,6 +18,7 @@ defmodule Mix.Pleroma do
   @doc "Common functions to be reused in mix tasks"
   def start_pleroma do
     Pleroma.Config.Holder.save_default()
+    Pleroma.Config.Oban.warn()
     Application.put_env(:phoenix, :serve_endpoints, false, persistent: true)
 
     if Pleroma.Config.get(:env) != :test do
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 33b1e3872..c39e24919 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -50,6 +50,7 @@ defmodule Pleroma.Application do
     Pleroma.Telemetry.Logger.attach()
     Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
+    Pleroma.Config.Oban.warn()
     Config.DeprecationWarnings.warn()
     Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled()
     Pleroma.ApplicationRequirements.verify!()
diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex
new file mode 100644
index 000000000..c2d56ebab
--- /dev/null
+++ b/lib/pleroma/config/oban.ex
@@ -0,0 +1,30 @@
+defmodule Pleroma.Config.Oban do
+  require Logger
+
+  def warn do
+    oban_config = Pleroma.Config.get(Oban)
+
+    crontab =
+      [Pleroma.Workers.Cron.StatsWorker]
+      |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
+        with acc when is_list(acc) <- acc,
+             setting when is_tuple(setting) <-
+               Enum.find(acc, fn {_, worker} -> worker == removed_worker end) do
+          """
+          !!!OBAN CONFIG WARNING!!!
+          You are using old workers in Oban crontab settings, which were removed.
+          Please, remove setting from crontab in your config file (prod.secret.exs): #{
+            inspect(setting)
+          }
+          """
+          |> Logger.warn()
+
+          List.delete(acc, setting)
+        else
+          _ -> acc
+        end
+      end)
+
+    Pleroma.Config.put(Oban, Keyword.put(oban_config, :crontab, crontab))
+  end
+end
diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex
index 9a03f01db..e7f8d272c 100644
--- a/lib/pleroma/stats.ex
+++ b/lib/pleroma/stats.ex
@@ -3,12 +3,15 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Stats do
+  use GenServer
+
   import Ecto.Query
+
   alias Pleroma.CounterCache
   alias Pleroma.Repo
   alias Pleroma.User
 
-  use GenServer
+  @interval :timer.seconds(60)
 
   def start_link(_) do
     GenServer.start_link(
@@ -18,6 +21,11 @@ defmodule Pleroma.Stats do
     )
   end
 
+  @impl true
+  def init(_args) do
+    {:ok, nil, {:continue, :calculate_stats}}
+  end
+
   @doc "Performs update stats"
   def force_update do
     GenServer.call(__MODULE__, :force_update)
@@ -29,7 +37,11 @@ defmodule Pleroma.Stats do
   end
 
   @doc "Returns stats data"
-  @spec get_stats() :: %{domain_count: integer(), status_count: integer(), user_count: integer()}
+  @spec get_stats() :: %{
+          domain_count: non_neg_integer(),
+          status_count: non_neg_integer(),
+          user_count: non_neg_integer()
+        }
   def get_stats do
     %{stats: stats} = GenServer.call(__MODULE__, :get_state)
 
@@ -44,25 +56,14 @@ defmodule Pleroma.Stats do
     peers
   end
 
-  def init(_args) do
-    {:ok, calculate_stat_data()}
-  end
-
-  def handle_call(:force_update, _from, _state) do
-    new_stats = calculate_stat_data()
-    {:reply, new_stats, new_stats}
-  end
-
-  def handle_call(:get_state, _from, state) do
-    {:reply, state, state}
-  end
-
-  def handle_cast(:run_update, _state) do
-    new_stats = calculate_stat_data()
-
-    {:noreply, new_stats}
-  end
-
+  @spec calculate_stat_data() :: %{
+          peers: list(),
+          stats: %{
+            domain_count: non_neg_integer(),
+            status_count: non_neg_integer(),
+            user_count: non_neg_integer()
+          }
+        }
   def calculate_stat_data do
     peers =
       from(
@@ -97,6 +98,7 @@ defmodule Pleroma.Stats do
     }
   end
 
+  @spec get_status_visibility_count(String.t() | nil) :: map()
   def get_status_visibility_count(instance \\ nil) do
     if is_nil(instance) do
       CounterCache.get_sum()
@@ -104,4 +106,36 @@ defmodule Pleroma.Stats do
       CounterCache.get_by_instance(instance)
     end
   end
+
+  @impl true
+  def handle_continue(:calculate_stats, _) do
+    stats = calculate_stat_data()
+    Process.send_after(self(), :run_update, @interval)
+    {:noreply, stats}
+  end
+
+  @impl true
+  def handle_call(:force_update, _from, _state) do
+    new_stats = calculate_stat_data()
+    {:reply, new_stats, new_stats}
+  end
+
+  @impl true
+  def handle_call(:get_state, _from, state) do
+    {:reply, state, state}
+  end
+
+  @impl true
+  def handle_cast(:run_update, _state) do
+    new_stats = calculate_stat_data()
+
+    {:noreply, new_stats}
+  end
+
+  @impl true
+  def handle_info(:run_update, _) do
+    new_stats = calculate_stat_data()
+    Process.send_after(self(), :run_update, @interval)
+    {:noreply, new_stats}
+  end
 end
diff --git a/lib/pleroma/workers/cron/stats_worker.ex b/lib/pleroma/workers/cron/stats_worker.ex
deleted file mode 100644
index 6a79540bc..000000000
--- a/lib/pleroma/workers/cron/stats_worker.ex
+++ /dev/null
@@ -1,17 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.Cron.StatsWorker do
-  @moduledoc """
-  The worker to update peers statistics.
-  """
-
-  use Oban.Worker, queue: "background"
-
-  @impl Oban.Worker
-  def perform(_job) do
-    Pleroma.Stats.do_collect()
-    :ok
-  end
-end
-- 
cgit v1.2.3


From a83916fdacac7b11ca478ef9a61b32dd269c8fd2 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 4 Sep 2020 19:05:08 +0300
Subject: adapter options unification

not needed options deletion
---
 lib/mix/tasks/pleroma/frontend.ex                     |  4 +---
 lib/pleroma/gun/conn.ex                               |  8 ++++----
 lib/pleroma/http/adapter_helper.ex                    |  2 +-
 lib/pleroma/http/adapter_helper/gun.ex                | 14 ++++++--------
 lib/pleroma/http/adapter_helper/hackney.ex            | 14 ++++++++++----
 .../activity_pub/mrf/media_proxy_warming_policy.ex    | 12 +++---------
 lib/pleroma/web/rel_me.ex                             | 15 +++------------
 lib/pleroma/web/rich_media/helpers.ex                 | 19 +++++--------------
 8 files changed, 33 insertions(+), 55 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
index 1957b1d84..73df67439 100644
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -124,9 +124,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
     url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
 
     with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [],
-             adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000]
-           ) do
+           Pleroma.HTTP.get(url, [], adapter: [pool: :media, recv_timeout: 120_000]) do
       unzip(zip_body, dest)
     else
       e -> {:error, e}
diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index a3f75a4bb..75b1ffc0a 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -13,7 +13,7 @@ defmodule Pleroma.Gun.Conn do
     opts =
       opts
       |> Enum.into(%{})
-      |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
+      |> Map.put_new(:connect_timeout, pool_opts[:connect_timeout] || 5_000)
       |> Map.put_new(:supervise, false)
       |> maybe_add_tls_opts(uri)
 
@@ -50,7 +50,7 @@ defmodule Pleroma.Gun.Conn do
 
     with open_opts <- Map.delete(opts, :tls_opts),
          {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]),
          stream <- Gun.connect(conn, connect_opts),
          {:response, :fin, 200, _} <- Gun.await(conn, stream) do
       {:ok, conn}
@@ -88,7 +88,7 @@ defmodule Pleroma.Gun.Conn do
       |> Map.put(:socks_opts, socks_opts)
 
     with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
       {:ok, conn}
     else
       error ->
@@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
 
     with {:ok, conn} <- Gun.open(host, port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
+         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
       {:ok, conn}
     else
       error ->
diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index d72297323..08b51578a 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @moduledoc """
   Configure Tesla.Client with default and customized adapter options.
   """
-  @defaults [pool: :federation]
+  @defaults [pool: :federation, connect_timeout: 5_000, recv_timeout: 5_000]
 
   @type proxy_type() :: :socks4 | :socks5
   @type host() :: charlist() | :inet.ip_address()
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index 4a967d8f2..1dbb71362 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -11,12 +11,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   require Logger
 
   @defaults [
-    connect_timeout: 5_000,
-    domain_lookup_timeout: 5_000,
-    tls_handshake_timeout: 5_000,
     retry: 1,
-    retry_timeout: 1000,
-    await_up_timeout: 5_000
+    retry_timeout: 1_000
   ]
 
   @type pool() :: :federation | :upload | :media | :default
@@ -45,15 +41,17 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   end
 
   defp put_timeout(opts) do
+    {recv_timeout, opts} = Keyword.pop(opts, :recv_timeout, pool_timeout(opts[:pool]))
     # this is the timeout to receive a message from Gun
-    Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
+    # `:timeout` key is used in Tesla
+    Keyword.put(opts, :timeout, recv_timeout)
   end
 
   @spec pool_timeout(pool()) :: non_neg_integer()
   def pool_timeout(pool) do
-    default = Config.get([:pools, :default, :timeout], 5_000)
+    default = Config.get([:pools, :default, :recv_timeout], 5_000)
 
-    Config.get([:pools, pool, :timeout], default)
+    Config.get([:pools, pool, :recv_timeout], default)
   end
 
   @prefix Pleroma.Gun.ConnectionPool
diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex
index 42e3acfec..ef84553c1 100644
--- a/lib/pleroma/http/adapter_helper/hackney.ex
+++ b/lib/pleroma/http/adapter_helper/hackney.ex
@@ -2,11 +2,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
   @behaviour Pleroma.HTTP.AdapterHelper
 
   @defaults [
-    connect_timeout: 10_000,
-    recv_timeout: 20_000,
     follow_redirect: true,
-    force_redirect: true,
-    pool: :federation
+    force_redirect: true
   ]
 
   @spec options(keyword(), URI.t()) :: keyword()
@@ -19,6 +16,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
     |> Keyword.merge(config_opts)
     |> Keyword.merge(connection_opts)
     |> add_scheme_opts(uri)
+    |> maybe_add_with_body()
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
   end
 
@@ -27,4 +25,12 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
   end
 
   defp add_scheme_opts(opts, _), do: opts
+
+  defp maybe_add_with_body(opts) do
+    if opts[:max_body] do
+      Keyword.put(opts, :with_body, true)
+    else
+      opts
+    end
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index dfab105a3..a203405a0 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -13,22 +13,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
   require Logger
 
   @options [
-    pool: :media
+    pool: :media,
+    recv_timeout: 10_000
   ]
 
   def perform(:prefetch, url) do
     Logger.debug("Prefetching #{inspect(url)}")
 
-    opts =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.put(@options, :recv_timeout, 10_000)
-      else
-        @options
-      end
-
     url
     |> MediaProxy.url()
-    |> HTTP.get([], adapter: opts)
+    |> HTTP.get([], adapter: @options)
   end
 
   def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex
index 8e2b51508..32bce3c1b 100644
--- a/lib/pleroma/web/rel_me.ex
+++ b/lib/pleroma/web/rel_me.ex
@@ -5,7 +5,8 @@
 defmodule Pleroma.Web.RelMe do
   @options [
     pool: :media,
-    max_body: 2_000_000
+    max_body: 2_000_000,
+    recv_timeout: 2_000
   ]
 
   if Pleroma.Config.get(:env) == :test do
@@ -23,18 +24,8 @@ defmodule Pleroma.Web.RelMe do
   def parse(_), do: {:error, "No URL provided"}
 
   defp parse_url(url) do
-    opts =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.merge(@options,
-          recv_timeout: 2_000,
-          with_body: true
-        )
-      else
-        @options
-      end
-
     with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
-           Pleroma.HTTP.get(url, [], adapter: opts),
+           Pleroma.HTTP.get(url, [], adapter: @options),
          {:ok, html_tree} <- Floki.parse_document(html),
          data <-
            Floki.attribute(html_tree, "link[rel~=me]", "href") ++
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 752ca9f81..084a66466 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -9,14 +9,15 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   alias Pleroma.Object
   alias Pleroma.Web.RichMedia.Parser
 
-  @rich_media_options [
+  @options [
     pool: :media,
-    max_body: 2_000_000
+    max_body: 2_000_000,
+    recv_timeout: 2_000
   ]
 
   @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
-    validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
+    validate_tld = Config.get([Pleroma.Formatter, :validate_tld])
 
     page_url
     |> Linkify.Parser.url?(validate_tld: validate_tld)
@@ -86,16 +87,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   def rich_media_get(url) do
     headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
 
-    options =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-        Keyword.merge(@rich_media_options,
-          recv_timeout: 2_000,
-          with_body: true
-        )
-      else
-        @rich_media_options
-      end
-
-    Pleroma.HTTP.get(url, headers, adapter: options)
+    Pleroma.HTTP.get(url, headers, adapter: @options)
   end
 end
-- 
cgit v1.2.3


From 696bf09433aa7f33cf580c71cb7f1f3367d4c124 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Mon, 7 Sep 2020 16:57:42 +0300
Subject: passing adapter options directly without adapter key

---
 lib/mix/tasks/pleroma/benchmark.ex                            | 11 ++++-------
 lib/mix/tasks/pleroma/frontend.ex                             |  2 +-
 lib/pleroma/http/ex_aws.ex                                    |  2 +-
 lib/pleroma/http/http.ex                                      |  2 +-
 lib/pleroma/http/tzdata.ex                                    |  4 ++--
 .../web/activity_pub/mrf/media_proxy_warming_policy.ex        |  2 +-
 lib/pleroma/web/rel_me.ex                                     |  2 +-
 lib/pleroma/web/rich_media/helpers.ex                         |  2 +-
 8 files changed, 12 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex
index dd2b9c8f2..a607d5d4f 100644
--- a/lib/mix/tasks/pleroma/benchmark.ex
+++ b/lib/mix/tasks/pleroma/benchmark.ex
@@ -91,20 +91,17 @@ defmodule Mix.Tasks.Pleroma.Benchmark do
         "Without conn and without pool" => fn ->
           {:ok, %Tesla.Env{}} =
             Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool, receive_conn: false]
+              pool: :no_pool,
+              receive_conn: false
             )
         end,
         "Without conn and with pool" => fn ->
           {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [receive_conn: false]
-            )
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false)
         end,
         "With reused conn and without pool" => fn ->
           {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              adapter: [pool: :no_pool]
-            )
+            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool)
         end,
         "With reused conn and with pool" => fn ->
           {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
index 73df67439..cbce81ab9 100644
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -124,7 +124,7 @@ defmodule Mix.Tasks.Pleroma.Frontend do
     url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
 
     with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [], adapter: [pool: :media, recv_timeout: 120_000]) do
+           Pleroma.HTTP.get(url, [], pool: :media, recv_timeout: 120_000) do
       unzip(zip_body, dest)
     else
       e -> {:error, e}
diff --git a/lib/pleroma/http/ex_aws.ex b/lib/pleroma/http/ex_aws.ex
index c3f335c73..5cac3532f 100644
--- a/lib/pleroma/http/ex_aws.ex
+++ b/lib/pleroma/http/ex_aws.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.ExAws do
 
   @impl true
   def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
-    http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload)
+    http_opts = Keyword.put_new(http_opts, :pool, :upload)
 
     case HTTP.request(method, url, body, headers, http_opts) do
       {:ok, env} ->
diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
index 7bc73f4a0..052597191 100644
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@@ -60,7 +60,7 @@ defmodule Pleroma.HTTP do
           {:ok, Env.t()} | {:error, any()}
   def request(method, url, body, headers, options) when is_binary(url) do
     uri = URI.parse(url)
-    adapter_opts = AdapterHelper.options(uri, options[:adapter] || [])
+    adapter_opts = AdapterHelper.options(uri, options || [])
 
     options = put_in(options[:adapter], adapter_opts)
     params = options[:params] || []
diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex
index 4539ac359..09cfdadf7 100644
--- a/lib/pleroma/http/tzdata.ex
+++ b/lib/pleroma/http/tzdata.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def get(url, headers, options) do
-    options = Keyword.put_new(options, :adapter, pool: :default)
+    options = Keyword.put_new(options, :pool, :default)
 
     with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
       {:ok, {env.status, env.headers, env.body}}
@@ -20,7 +20,7 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def head(url, headers, options) do
-    options = Keyword.put_new(options, :adapter, pool: :default)
+    options = Keyword.put_new(options, :pool, :default)
 
     with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
       {:ok, {env.status, env.headers}}
diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index a203405a0..98d595469 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
     url
     |> MediaProxy.url()
-    |> HTTP.get([], adapter: @options)
+    |> HTTP.get([], @options)
   end
 
   def perform(:preload, %{"object" => %{"attachment" => attachments}} = _message) do
diff --git a/lib/pleroma/web/rel_me.ex b/lib/pleroma/web/rel_me.ex
index 32bce3c1b..28f75b18d 100644
--- a/lib/pleroma/web/rel_me.ex
+++ b/lib/pleroma/web/rel_me.ex
@@ -25,7 +25,7 @@ defmodule Pleroma.Web.RelMe do
 
   defp parse_url(url) do
     with {:ok, %Tesla.Env{body: html, status: status}} when status in 200..299 <-
-           Pleroma.HTTP.get(url, [], adapter: @options),
+           Pleroma.HTTP.get(url, [], @options),
          {:ok, html_tree} <- Floki.parse_document(html),
          data <-
            Floki.attribute(html_tree, "link[rel~=me]", "href") ++
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 084a66466..bd7f03cbe 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -87,6 +87,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   def rich_media_get(url) do
     headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
 
-    Pleroma.HTTP.get(url, headers, adapter: @options)
+    Pleroma.HTTP.get(url, headers, @options)
   end
 end
-- 
cgit v1.2.3


From 18d21aed00dcbdaabd7db25b8b7d0c88141ec98a Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Mon, 7 Sep 2020 19:04:16 +0300
Subject: deprecation warnings

---
 lib/pleroma/config/deprecation_warnings.ex | 43 ++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex
index 0f52eb210..2bfe4ddba 100644
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@@ -56,6 +56,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     check_old_mrf_config()
     check_media_proxy_whitelist_config()
     check_welcome_message_config()
+    check_gun_pool_options()
   end
 
   def check_welcome_message_config do
@@ -115,4 +116,46 @@ defmodule Pleroma.Config.DeprecationWarnings do
       """)
     end
   end
+
+  def check_gun_pool_options do
+    pool_config = Config.get(:connections_pool)
+
+    if timeout = pool_config[:await_up_timeout] do
+      Logger.warn("""
+      !!!DEPRECATION WARNING!!!
+      Your config is using old setting name `await_up_timeout` instead of `connect_timeout`. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+      """)
+
+      Config.put(:connections_pool, Keyword.put_new(pool_config, :connect_timeout, timeout))
+    end
+
+    pools_configs = Config.get(:pools)
+
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+    Your config is using old setting name `timeout` instead of `recv_timeout` in pool settings. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
+    """
+
+    updated_config =
+      Enum.reduce(pools_configs, [], fn {pool_name, config}, acc ->
+        if timeout = config[:timeout] do
+          Keyword.put(acc, pool_name, Keyword.put_new(config, :recv_timeout, timeout))
+        else
+          acc
+        end
+      end)
+
+    if updated_config != [] do
+      pool_warnings =
+        updated_config
+        |> Keyword.keys()
+        |> Enum.map(fn pool_name ->
+          "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`"
+        end)
+
+      Logger.warn(Enum.join([warning_preface | pool_warnings]))
+
+      Config.put(:pools, updated_config)
+    end
+  end
 end
-- 
cgit v1.2.3


From 699224a900d54b6d32e0bd3f2abd9eccc523df11 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Mon, 7 Sep 2020 22:14:40 +0200
Subject: ForceBotUnlistedPolicy: initial add, tiny clean up from my previous
 version

---
 .../activity_pub/mrf/force_bot_unlisted_policy.ex  | 61 ++++++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
new file mode 100644
index 000000000..31fd90586
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
@@ -0,0 +1,61 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
+  alias Pleroma.User
+  @behaviour Pleroma.Web.ActivityPub.MRF
+  @moduledoc "Remove bot posts from federated timeline"
+
+  require Pleroma.Constants
+
+  defp check_by_actor_type(user) do
+    if user.actor_type in ["Application", "Service"], do: 1.0, else: 0.0
+  end
+
+  defp check_by_nickname(user) do
+    if Regex.match?(~r/bot@|ebooks@/i, user.nickname), do: 1.0, else: 0.0
+  end
+
+  defp botness_score(user), do: check_by_actor_type(user) + check_by_nickname(user)
+
+  @impl true
+  def filter(
+         %{
+           "type" => "Create",
+           "to" => to,
+           "cc" => cc,
+           "actor" => actor,
+           "object" => object
+         } = message
+       ) do
+    user = User.get_cached_by_ap_id(actor)
+    isbot = 0.8 < botness_score(user)
+
+    if isbot and Enum.member?(to, Pleroma.Constants.as_public()) do
+      to = List.delete(to, Pleroma.Constants.as_public()) ++ [user.follower_address]
+      cc = List.delete(cc, user.follower_address) ++ [Pleroma.Constants.as_public()]
+
+      object =
+        object
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+
+      message =
+        message
+        |> Map.put("to", to)
+        |> Map.put("cc", cc)
+        |> Map.put("object", object)
+
+      {:ok, message}
+    else
+      {:ok, message}
+    end
+  end
+
+  @impl true
+  def filter(message), do: {:ok, message}
+
+  @impl true
+  def describe, do: {:ok, %{}}
+end
-- 
cgit v1.2.3


From 8b695c3eeb6ee7a91fc5a8a4293fb3cb53212818 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Mon, 7 Sep 2020 22:53:45 +0200
Subject: ForceBotUnlistedPolicy: format

---
 .../web/activity_pub/mrf/force_bot_unlisted_policy.ex    | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
index 31fd90586..7290f444b 100644
--- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
@@ -21,14 +21,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
 
   @impl true
   def filter(
-         %{
-           "type" => "Create",
-           "to" => to,
-           "cc" => cc,
-           "actor" => actor,
-           "object" => object
-         } = message
-       ) do
+        %{
+          "type" => "Create",
+          "to" => to,
+          "cc" => cc,
+          "actor" => actor,
+          "object" => object
+        } = message
+      ) do
     user = User.get_cached_by_ap_id(actor)
     isbot = 0.8 < botness_score(user)
 
-- 
cgit v1.2.3


From 95688c90ad9cd6438a764b4ea6e0f2e3b594b5c8 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Tue, 8 Sep 2020 01:13:49 +0200
Subject: ForceBotUnlistedPolicy: simplify code

---
 .../web/activity_pub/mrf/force_bot_unlisted_policy.ex       | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
index 7290f444b..ea9c3d3f5 100644
--- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex
@@ -9,15 +9,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
 
   require Pleroma.Constants
 
-  defp check_by_actor_type(user) do
-    if user.actor_type in ["Application", "Service"], do: 1.0, else: 0.0
-  end
-
-  defp check_by_nickname(user) do
-    if Regex.match?(~r/bot@|ebooks@/i, user.nickname), do: 1.0, else: 0.0
-  end
+  defp check_by_actor_type(user), do: user.actor_type in ["Application", "Service"]
+  defp check_by_nickname(user), do: Regex.match?(~r/bot@|ebooks@/i, user.nickname)
 
-  defp botness_score(user), do: check_by_actor_type(user) + check_by_nickname(user)
+  defp check_if_bot(user), do: check_by_actor_type(user) or check_by_nickname(user)
 
   @impl true
   def filter(
@@ -30,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do
         } = message
       ) do
     user = User.get_cached_by_ap_id(actor)
-    isbot = 0.8 < botness_score(user)
+    isbot = check_if_bot(user)
 
     if isbot and Enum.member?(to, Pleroma.Constants.as_public()) do
       to = List.delete(to, Pleroma.Constants.as_public()) ++ [user.follower_address]
-- 
cgit v1.2.3


From fa347b9c2f416cd2c402e3e6eebb561dfc0ee8a8 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Wed, 26 Aug 2020 13:32:03 -0500
Subject: Fix uploading webp image files when Exiftool Upload Filter is enabled

---
 lib/pleroma/upload/filter/exiftool.ex | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex
index ea8798fe3..a91bd5e24 100644
--- a/lib/pleroma/upload/filter/exiftool.ex
+++ b/lib/pleroma/upload/filter/exiftool.ex
@@ -10,9 +10,20 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   @behaviour Pleroma.Upload.Filter
 
   @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
-  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
+  def filter(%Pleroma.Upload{name: file, tempfile: path, content_type: "image" <> _}) do
+    # webp is not compatible with exiftool at this time
+    if Regex.match?(~r/\.(webp)$/i, file) do
+      :ok
+    else
+      strip_exif(path)
+    end
+  end
+
+  def filter(_), do: :ok
+
+  defp strip_exif(path) do
     try do
-      case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
+      case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", path], parallelism: true) do
         {_response, 0} -> :ok
         {error, 1} -> {:error, error}
       end
@@ -21,6 +32,4 @@ defmodule Pleroma.Upload.Filter.Exiftool do
         {:error, "exiftool command not found"}
     end
   end
-
-  def filter(_), do: :ok
 end
-- 
cgit v1.2.3


From 2165a249744a1ad4386a9d237871abe88e298942 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 4 Sep 2020 17:40:59 -0500
Subject: Improve upload filter return values so we can identify when filters
 make no changes to the input

---
 lib/pleroma/upload/filter.ex                    | 13 ++++++++++---
 lib/pleroma/upload/filter/anonymize_filename.ex |  4 +++-
 lib/pleroma/upload/filter/dedupe.ex             |  4 ++--
 lib/pleroma/upload/filter/exiftool.ex           |  8 ++++----
 lib/pleroma/upload/filter/mogrifun.ex           |  6 +++---
 lib/pleroma/upload/filter/mogrify.ex            |  6 +++---
 6 files changed, 25 insertions(+), 16 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/upload/filter.ex b/lib/pleroma/upload/filter.ex
index dbdadc97f..661135634 100644
--- a/lib/pleroma/upload/filter.ex
+++ b/lib/pleroma/upload/filter.ex
@@ -15,7 +15,11 @@ defmodule Pleroma.Upload.Filter do
 
   require Logger
 
-  @callback filter(Pleroma.Upload.t()) :: :ok | {:ok, Pleroma.Upload.t()} | {:error, any()}
+  @callback filter(Pleroma.Upload.t()) ::
+              {:ok, :filtered}
+              | {:ok, :noop}
+              | {:ok, :filtered, Pleroma.Upload.t()}
+              | {:error, any()}
 
   @spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()}
 
@@ -25,10 +29,13 @@ defmodule Pleroma.Upload.Filter do
 
   def filter([filter | rest], upload) do
     case filter.filter(upload) do
-      :ok ->
+      {:ok, :filtered} ->
         filter(rest, upload)
 
-      {:ok, upload} ->
+      {:ok, :filtered, upload} ->
+        filter(rest, upload)
+
+      {:ok, :noop} ->
         filter(rest, upload)
 
       error ->
diff --git a/lib/pleroma/upload/filter/anonymize_filename.ex b/lib/pleroma/upload/filter/anonymize_filename.ex
index 07ead8203..fc456e4f4 100644
--- a/lib/pleroma/upload/filter/anonymize_filename.ex
+++ b/lib/pleroma/upload/filter/anonymize_filename.ex
@@ -16,9 +16,11 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
   def filter(%Upload{name: name} = upload) do
     extension = List.last(String.split(name, "."))
     name = predefined_name(extension) || random(extension)
-    {:ok, %Upload{upload | name: name}}
+    {:ok, :filtered, %Upload{upload | name: name}}
   end
 
+  def filter(_), do: {:ok, :noop}
+
   @spec predefined_name(String.t()) :: String.t() | nil
   defp predefined_name(extension) do
     with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
diff --git a/lib/pleroma/upload/filter/dedupe.ex b/lib/pleroma/upload/filter/dedupe.ex
index 41218a918..86cbc8996 100644
--- a/lib/pleroma/upload/filter/dedupe.ex
+++ b/lib/pleroma/upload/filter/dedupe.ex
@@ -17,8 +17,8 @@ defmodule Pleroma.Upload.Filter.Dedupe do
       |> Base.encode16(case: :lower)
 
     filename = shasum <> "." <> extension
-    {:ok, %Upload{upload | id: shasum, path: filename}}
+    {:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end
diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex
index a91bd5e24..94d12c01b 100644
--- a/lib/pleroma/upload/filter/exiftool.ex
+++ b/lib/pleroma/upload/filter/exiftool.ex
@@ -9,22 +9,22 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   """
   @behaviour Pleroma.Upload.Filter
 
-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
   def filter(%Pleroma.Upload{name: file, tempfile: path, content_type: "image" <> _}) do
     # webp is not compatible with exiftool at this time
     if Regex.match?(~r/\.(webp)$/i, file) do
-      :ok
+      {:ok, :noop}
     else
       strip_exif(path)
     end
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 
   defp strip_exif(path) do
     try do
       case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", path], parallelism: true) do
-        {_response, 0} -> :ok
+        {_response, 0} -> {:ok, :filtered}
         {error, 1} -> {:error, error}
       end
     rescue
diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex
index c8fa7b190..363e5cf0f 100644
--- a/lib/pleroma/upload/filter/mogrifun.ex
+++ b/lib/pleroma/upload/filter/mogrifun.ex
@@ -38,16 +38,16 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
     [{"fill", "yellow"}, {"tint", "40"}]
   ]
 
-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do
       Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
-      :ok
+      {:ok, :filtered}
     rescue
       _e in ErlangError ->
         {:error, "mogrify command not found"}
     end
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end
diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex
index 7a45add5a..71968fd9c 100644
--- a/lib/pleroma/upload/filter/mogrify.ex
+++ b/lib/pleroma/upload/filter/mogrify.ex
@@ -8,18 +8,18 @@ defmodule Pleroma.Upload.Filter.Mogrify do
   @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
   @type conversions :: conversion() | [conversion()]
 
-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do
       do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
-      :ok
+      {:ok, :filtered}
     rescue
       _e in ErlangError ->
         {:error, "mogrify command not found"}
     end
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 
   def do_filter(file, filters) do
     file
-- 
cgit v1.2.3


From 216c84a8f4d82649110ffaa2bc9d02b879805c5f Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 4 Sep 2020 17:56:05 -0500
Subject: Bypass the filter based on content-type as well in case a webp image
 is uploaded with the wrong file extension.

---
 lib/pleroma/upload/filter/exiftool.ex | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex
index 94d12c01b..b07a671ac 100644
--- a/lib/pleroma/upload/filter/exiftool.ex
+++ b/lib/pleroma/upload/filter/exiftool.ex
@@ -10,8 +10,11 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   @behaviour Pleroma.Upload.Filter
 
   @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
+
+  # webp is not compatible with exiftool at this time
+  def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
+
   def filter(%Pleroma.Upload{name: file, tempfile: path, content_type: "image" <> _}) do
-    # webp is not compatible with exiftool at this time
     if Regex.match?(~r/\.(webp)$/i, file) do
       {:ok, :noop}
     else
-- 
cgit v1.2.3


From 4ea07f74e9416da8f97a12cfdc24da82e1c00d91 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 4 Sep 2020 22:18:01 -0500
Subject: Revert/simplify.

We only need to check the content-type. There's no chance a webp file
will get mismatched as another image type.
---
 lib/pleroma/upload/filter/exiftool.ex | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex
index b07a671ac..1fd0cfdaa 100644
--- a/lib/pleroma/upload/filter/exiftool.ex
+++ b/lib/pleroma/upload/filter/exiftool.ex
@@ -14,19 +14,9 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   # webp is not compatible with exiftool at this time
   def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
 
-  def filter(%Pleroma.Upload{name: file, tempfile: path, content_type: "image" <> _}) do
-    if Regex.match?(~r/\.(webp)$/i, file) do
-      {:ok, :noop}
-    else
-      strip_exif(path)
-    end
-  end
-
-  def filter(_), do: {:ok, :noop}
-
-  defp strip_exif(path) do
+  def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do
-      case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", path], parallelism: true) do
+      case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
         {_response, 0} -> {:ok, :filtered}
         {error, 1} -> {:error, error}
       end
@@ -35,4 +25,6 @@ defmodule Pleroma.Upload.Filter.Exiftool do
         {:error, "exiftool command not found"}
     end
   end
+
+  def filter(_), do: {:ok, :noop}
 end
-- 
cgit v1.2.3


From 718c7cc8473282a9239cc125e10b54df7ed5e0fe Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 7 Sep 2020 16:38:16 +0000
Subject: Merge branch 'fix/apc2s-limits' into 'develop'

AP C2S: Restrict character limit on Note

Closes #2

See merge request pleroma/secteam/pleroma!9
---
 .../web/activity_pub/activity_pub_controller.ex    | 37 ++++++++++++++--------
 1 file changed, 23 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 220c4fe52..732c44271 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -399,21 +399,30 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   defp handle_user_activity(
          %User{} = user,
-         %{"type" => "Create", "object" => %{"type" => "Note"}} = params
+         %{"type" => "Create", "object" => %{"type" => "Note"} = object} = params
        ) do
-    object =
-      params["object"]
-      |> Map.merge(Map.take(params, ["to", "cc"]))
-      |> Map.put("attributedTo", user.ap_id())
-      |> Transmogrifier.fix_object()
-
-    ActivityPub.create(%{
-      to: params["to"],
-      actor: user,
-      context: object["context"],
-      object: object,
-      additional: Map.take(params, ["cc"])
-    })
+    content = if is_binary(object["content"]), do: object["content"], else: ""
+    name = if is_binary(object["name"]), do: object["name"], else: ""
+    summary = if is_binary(object["summary"]), do: object["summary"], else: ""
+    length = String.length(content <> name <> summary)
+
+    if length > Pleroma.Config.get([:instance, :limit]) do
+      {:error, dgettext("errors", "Note is over the character limit")}
+    else
+      object =
+        object
+        |> Map.merge(Map.take(params, ["to", "cc"]))
+        |> Map.put("attributedTo", user.ap_id())
+        |> Transmogrifier.fix_object()
+
+      ActivityPub.create(%{
+        to: params["to"],
+        actor: user,
+        context: object["context"],
+        object: object,
+        additional: Map.take(params, ["cc"])
+      })
+    end
   end
 
   defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do
-- 
cgit v1.2.3


From 84acfab15659a1e09bfd5ce75f1ba0d60d2ab7b2 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 2 Sep 2020 06:55:58 +0000
Subject: Merge branch 'fix-searching-following' into 'develop'

search: fix 'following' query parameter

See merge request pleroma/pleroma!2943
---
 lib/pleroma/user/search.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index d4fd31069..adbef7fb8 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -116,7 +116,7 @@ defmodule Pleroma.User.Search do
   end
 
   defp base_query(_user, false), do: User
-  defp base_query(user, true), do: User.get_followers_query(user)
+  defp base_query(user, true), do: User.get_friends_query(user)
 
   defp filter_invisible_users(query) do
     from(q in query, where: q.invisible == false)
-- 
cgit v1.2.3


From 74d46a1b0945380c2254943043c5935787876288 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 8 Sep 2020 09:47:16 +0000
Subject: Merge branch 'secfix/search-crash' into 'develop'

Fix crash done via search

See merge request pleroma/secteam/pleroma!12
---
 lib/pleroma/user.ex        | 20 ++++++++++----------
 lib/pleroma/user/search.ex |  4 ++--
 2 files changed, 12 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d2ad9516f..ce5f4bc4a 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1125,31 +1125,31 @@ defmodule Pleroma.User do
     User.Query.build(%{followers: user, deactivated: false})
   end
 
-  def get_followers_query(user, page) do
+  def get_followers_query(%User{} = user, page) do
     user
     |> get_followers_query(nil)
     |> User.Query.paginate(page, 20)
   end
 
   @spec get_followers_query(User.t()) :: Ecto.Query.t()
-  def get_followers_query(user), do: get_followers_query(user, nil)
+  def get_followers_query(%User{} = user), do: get_followers_query(user, nil)
 
   @spec get_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
-  def get_followers(user, page \\ nil) do
+  def get_followers(%User{} = user, page \\ nil) do
     user
     |> get_followers_query(page)
     |> Repo.all()
   end
 
   @spec get_external_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
-  def get_external_followers(user, page \\ nil) do
+  def get_external_followers(%User{} = user, page \\ nil) do
     user
     |> get_followers_query(page)
     |> User.Query.build(%{external: true})
     |> Repo.all()
   end
 
-  def get_followers_ids(user, page \\ nil) do
+  def get_followers_ids(%User{} = user, page \\ nil) do
     user
     |> get_followers_query(page)
     |> select([u], u.id)
@@ -1161,29 +1161,29 @@ defmodule Pleroma.User do
     User.Query.build(%{friends: user, deactivated: false})
   end
 
-  def get_friends_query(user, page) do
+  def get_friends_query(%User{} = user, page) do
     user
     |> get_friends_query(nil)
     |> User.Query.paginate(page, 20)
   end
 
   @spec get_friends_query(User.t()) :: Ecto.Query.t()
-  def get_friends_query(user), do: get_friends_query(user, nil)
+  def get_friends_query(%User{} = user), do: get_friends_query(user, nil)
 
-  def get_friends(user, page \\ nil) do
+  def get_friends(%User{} = user, page \\ nil) do
     user
     |> get_friends_query(page)
     |> Repo.all()
   end
 
-  def get_friends_ap_ids(user) do
+  def get_friends_ap_ids(%User{} = user) do
     user
     |> get_friends_query(nil)
     |> select([u], u.ap_id)
     |> Repo.all()
   end
 
-  def get_friends_ids(user, page \\ nil) do
+  def get_friends_ids(%User{} = user, page \\ nil) do
     user
     |> get_friends_query(page)
     |> select([u], u.id)
diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index adbef7fb8..7babd47ea 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -115,8 +115,8 @@ defmodule Pleroma.User.Search do
     )
   end
 
-  defp base_query(_user, false), do: User
-  defp base_query(user, true), do: User.get_friends_query(user)
+  defp base_query(%User{} = user, true), do: User.get_friends_query(user)
+  defp base_query(_user, _following), do: User
 
   defp filter_invisible_users(query) do
     from(q in query, where: q.invisible == false)
-- 
cgit v1.2.3


From 13e606941c6ff5de64b0b104e2b0c1f2a66fda59 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 8 Sep 2020 09:49:27 +0000
Subject: Merge branch 'fix/2087-metadata' into 'develop'

Fix/2087 metadata

See merge request pleroma/secteam/pleroma!11
---
 lib/pleroma/web/feed/tag_controller.ex  | 10 +++++++++-
 lib/pleroma/web/feed/user_controller.ex | 10 +++++++++-
 lib/pleroma/web/metadata.ex             | 11 ++++++++++-
 3 files changed, 28 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex
index 39b2a766a..93a8294b7 100644
--- a/lib/pleroma/web/feed/tag_controller.ex
+++ b/lib/pleroma/web/feed/tag_controller.ex
@@ -9,7 +9,15 @@ defmodule Pleroma.Web.Feed.TagController do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.Feed.FeedView
 
-  def feed(conn, %{"tag" => raw_tag} = params) do
+  def feed(conn, params) do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
+      render_feed(conn, params)
+    else
+      render_error(conn, :not_found, "Not found")
+    end
+  end
+
+  def render_feed(conn, %{"tag" => raw_tag} = params) do
     {format, tag} = parse_tag(raw_tag)
 
     activities =
diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 9cd334a33..71eb1ea7e 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -37,7 +37,15 @@ defmodule Pleroma.Web.Feed.UserController do
     end
   end
 
-  def feed(conn, %{"nickname" => nickname} = params) do
+  def feed(conn, params) do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:profiles, :local) do
+      render_feed(conn, params)
+    else
+      errors(conn, {:error, :not_found})
+    end
+  end
+
+  def render_feed(conn, %{"nickname" => nickname} = params) do
     format = get_format(conn)
 
     format =
diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex
index a9f70c43e..0f2d8d1e7 100644
--- a/lib/pleroma/web/metadata.ex
+++ b/lib/pleroma/web/metadata.ex
@@ -7,8 +7,9 @@ defmodule Pleroma.Web.Metadata do
 
   def build_tags(params) do
     providers = [
+      Pleroma.Web.Metadata.Providers.RelMe,
       Pleroma.Web.Metadata.Providers.RestrictIndexing
-      | Pleroma.Config.get([__MODULE__, :providers], [])
+      | activated_providers()
     ]
 
     Enum.reduce(providers, "", fn parser, acc ->
@@ -42,4 +43,12 @@ defmodule Pleroma.Web.Metadata do
   def activity_nsfw?(_) do
     false
   end
+
+  defp activated_providers do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
+      [Pleroma.Web.Metadata.Providers.Feed | Pleroma.Config.get([__MODULE__, :providers], [])]
+    else
+      []
+    end
+  end
 end
-- 
cgit v1.2.3


From 8c3241df449857b408d653306f8c0713ebf3c880 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 2 Sep 2020 09:38:43 +0000
Subject: Merge branch 'fix/2047-rich-media-parser' into 'develop'

RichMedia parser fix

Closes #2047

See merge request pleroma/pleroma!2941
---
 lib/pleroma/web/rich_media/parser.ex               | 44 ++++++++++++----------
 .../web/rich_media/parsers/ttl/aws_signed_url.ex   | 15 +++-----
 2 files changed, 29 insertions(+), 30 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index ca592833f..e9aa2dd03 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -3,6 +3,8 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.RichMedia.Parser do
+  require Logger
+
   defp parsers do
     Pleroma.Config.get([:rich_media, :parsers])
   end
@@ -10,18 +12,19 @@ defmodule Pleroma.Web.RichMedia.Parser do
   def parse(nil), do: {:error, "No URL provided"}
 
   if Pleroma.Config.get(:env) == :test do
+    @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
     def parse(url), do: parse_url(url)
   else
+    @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
     def parse(url) do
-      try do
-        Cachex.fetch!(:rich_media_cache, url, fn _ ->
-          {:commit, parse_url(url)}
-        end)
-        |> set_ttl_based_on_image(url)
-      rescue
-        e ->
-          {:error, "Cachex error: #{inspect(e)}"}
-      end
+      Cachex.fetch!(:rich_media_cache, url, fn _ ->
+        with {:ok, data} <- parse_url(url) do
+          {:commit, {:ok, data}}
+        else
+          error -> {:ignore, error}
+        end
+      end)
+      |> set_ttl_based_on_image(url)
     end
   end
 
@@ -47,9 +50,11 @@ defmodule Pleroma.Web.RichMedia.Parser do
       config :pleroma, :rich_media,
         ttl_setters: [MyModule]
   """
+  @spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) ::
+          {:ok, map()} | {:error, any()}
   def set_ttl_based_on_image({:ok, data}, url) do
     with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
-         ttl when is_number(ttl) <- get_ttl_from_image(data, url) do
+         {:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do
       Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
       {:ok, data}
     else
@@ -58,8 +63,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
     end
   end
 
+  def set_ttl_based_on_image({:error, _} = error, _) do
+    Logger.error("parsing error: #{inspect(error)}")
+    error
+  end
+
   defp get_ttl_from_image(data, url) do
-    Pleroma.Config.get([:rich_media, :ttl_setters])
+    [:rich_media, :ttl_setters]
+    |> Pleroma.Config.get()
     |> Enum.reduce({:ok, nil}, fn
       module, {:ok, _ttl} ->
         module.ttl(data, url)
@@ -70,23 +81,16 @@ defmodule Pleroma.Web.RichMedia.Parser do
   end
 
   defp parse_url(url) do
-    try do
-      {:ok, %Tesla.Env{body: html}} = Pleroma.Web.RichMedia.Helpers.rich_media_get(url)
-
+    with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url),
+         {:ok, html} <- Floki.parse_document(html) do
       html
-      |> parse_html()
       |> maybe_parse()
       |> Map.put("url", url)
       |> clean_parsed_data()
       |> check_parsed_data()
-    rescue
-      e ->
-        {:error, "Parsing error: #{inspect(e)} #{inspect(__STACKTRACE__)}"}
     end
   end
 
-  defp parse_html(html), do: Floki.parse_document!(html)
-
   defp maybe_parse(html) do
     Enum.reduce_while(parsers(), %{}, fn parser, acc ->
       case parser.parse(html, acc) do
diff --git a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
index 0dc1efdaf..c5aaea2d4 100644
--- a/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
+++ b/lib/pleroma/web/rich_media/parsers/ttl/aws_signed_url.ex
@@ -10,20 +10,15 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
       |> parse_query_params()
       |> format_query_params()
       |> get_expiration_timestamp()
+    else
+      {:error, "Not aws signed url #{inspect(image)}"}
     end
   end
 
-  defp is_aws_signed_url(""), do: nil
-  defp is_aws_signed_url(nil), do: nil
-
-  defp is_aws_signed_url(image) when is_binary(image) do
+  defp is_aws_signed_url(image) when is_binary(image) and image != "" do
     %URI{host: host, query: query} = URI.parse(image)
 
-    if String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") do
-      image
-    else
-      nil
-    end
+    String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires")
   end
 
   defp is_aws_signed_url(_), do: nil
@@ -46,6 +41,6 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do
       |> Map.get("X-Amz-Date")
       |> Timex.parse("{ISO:Basic:Z}")
 
-    Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))
+    {:ok, Timex.to_unix(date) + String.to_integer(Map.get(params, "X-Amz-Expires"))}
   end
 end
-- 
cgit v1.2.3


From ea4b6c64d60d1dc3246f5a2f23a2e3a47e8fb476 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 2 Sep 2020 13:46:11 +0000
Subject: Merge branch 'feat/rich-media-improvements' into 'develop'

Rich media improvements

See merge request pleroma/pleroma!2944
---
 lib/pleroma/html.ex                               |  5 ++-
 lib/pleroma/web/mastodon_api/views/status_view.ex | 16 +++++++
 lib/pleroma/web/rich_media/parser.ex              | 54 ++++++++++++++---------
 3 files changed, 51 insertions(+), 24 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index dc1b9b840..20b02f091 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -109,8 +109,9 @@ defmodule Pleroma.HTML do
       result =
         content
         |> Floki.parse_fragment!()
-        |> Floki.filter_out("a.mention,a.hashtag,a.attachment,a[rel~=\"tag\"]")
-        |> Floki.attribute("a", "href")
+        |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
+        |> Enum.take(1)
+        |> Floki.attribute("href")
         |> Enum.at(0)
 
       {:commit, {:ok, result}}
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 01b8bb6bb..3fe1967be 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -23,6 +23,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
   import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2]
 
+  # This is a naive way to do this, just spawning a process per activity
+  # to fetch the preview. However it should be fine considering
+  # pagination is restricted to 40 activities at a time
+  defp fetch_rich_media_for_activities(activities) do
+    Enum.each(activities, fn activity ->
+      spawn(fn ->
+        Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
+      end)
+    end)
+  end
+
   # TODO: Add cached version.
   defp get_replied_to_activities([]), do: %{}
 
@@ -80,6 +91,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list
     activities = Enum.filter(opts.activities, & &1)
+
+    # Start fetching rich media before doing anything else, so that later calls to get the cards
+    # only block for timeout in the worst case, as opposed to
+    # length(activities_with_links) * timeout
+    fetch_rich_media_for_activities(activities)
     replied_to_activities = get_replied_to_activities(activities)
 
     parent_activities =
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index e9aa2dd03..e98c743ca 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -17,14 +17,25 @@ defmodule Pleroma.Web.RichMedia.Parser do
   else
     @spec parse(String.t()) :: {:ok, map()} | {:error, any()}
     def parse(url) do
-      Cachex.fetch!(:rich_media_cache, url, fn _ ->
-        with {:ok, data} <- parse_url(url) do
-          {:commit, {:ok, data}}
-        else
-          error -> {:ignore, error}
-        end
-      end)
-      |> set_ttl_based_on_image(url)
+      with {:ok, data} <- get_cached_or_parse(url),
+           {:ok, _} <- set_ttl_based_on_image(data, url) do
+        {:ok, data}
+      else
+        error ->
+          Logger.error(fn -> "Rich media error: #{inspect(error)}" end)
+      end
+    end
+
+    defp get_cached_or_parse(url) do
+      case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do
+        {:ok, _data} = res ->
+          res
+
+        {:error, _} = e ->
+          ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
+          Cachex.expire(:rich_media_cache, url, ttl)
+          e
+      end
     end
   end
 
@@ -50,24 +61,23 @@ defmodule Pleroma.Web.RichMedia.Parser do
       config :pleroma, :rich_media,
         ttl_setters: [MyModule]
   """
-  @spec set_ttl_based_on_image({:ok, map()} | {:error, any()}, String.t()) ::
-          {:ok, map()} | {:error, any()}
-  def set_ttl_based_on_image({:ok, data}, url) do
-    with {:ok, nil} <- Cachex.ttl(:rich_media_cache, url),
-         {:ok, ttl} when is_number(ttl) <- get_ttl_from_image(data, url) do
-      Cachex.expire_at(:rich_media_cache, url, ttl * 1000)
-      {:ok, data}
-    else
+  @spec set_ttl_based_on_image(map(), String.t()) ::
+          {:ok, Integer.t() | :noop} | {:error, :no_key}
+  def set_ttl_based_on_image(data, url) do
+    case get_ttl_from_image(data, url) do
+      {:ok, ttl} when is_number(ttl) ->
+        ttl = ttl * 1000
+
+        case Cachex.expire_at(:rich_media_cache, url, ttl) do
+          {:ok, true} -> {:ok, ttl}
+          {:ok, false} -> {:error, :no_key}
+        end
+
       _ ->
-        {:ok, data}
+        {:ok, :noop}
     end
   end
 
-  def set_ttl_based_on_image({:error, _} = error, _) do
-    Logger.error("parsing error: #{inspect(error)}")
-    error
-  end
-
   defp get_ttl_from_image(data, url) do
     [:rich_media, :ttl_setters]
     |> Pleroma.Config.get()
-- 
cgit v1.2.3


From dccbed856df131900bf7851fbd11c518f85bce7a Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 2 Sep 2020 09:36:56 +0000
Subject: Merge branch 'http-fixes' into 'develop'

Improvements and fixes for http requests

See merge request pleroma/pleroma!2904
---
 lib/mix/tasks/pleroma/frontend.ex         |  4 +++-
 lib/pleroma/gun/connection_pool/worker.ex | 35 +++++++++++++++++++++----------
 lib/pleroma/http/adapter_helper.ex        | 19 ++---------------
 lib/pleroma/http/adapter_helper/gun.ex    | 33 +++++++++++++++++++++--------
 lib/pleroma/http/ex_aws.ex                |  2 ++
 lib/pleroma/http/tzdata.ex                |  4 ++++
 lib/pleroma/instances/instance.ex         |  4 +++-
 lib/pleroma/object/fetcher.ex             |  6 +++---
 lib/pleroma/uploaders/s3.ex               | 23 ++++++++++++++------
 lib/pleroma/web/rich_media/helpers.ex     |  2 +-
 lib/pleroma/web/web_finger/web_finger.ex  |  4 ++--
 11 files changed, 85 insertions(+), 51 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/frontend.ex b/lib/mix/tasks/pleroma/frontend.ex
index 2adbf8d72..484af6da7 100644
--- a/lib/mix/tasks/pleroma/frontend.ex
+++ b/lib/mix/tasks/pleroma/frontend.ex
@@ -124,7 +124,9 @@ defmodule Mix.Tasks.Pleroma.Frontend do
     url = String.replace(frontend_info["build_url"], "${ref}", frontend_info["ref"])
 
     with {:ok, %{status: 200, body: zip_body}} <-
-           Pleroma.HTTP.get(url, [], timeout: 120_000, recv_timeout: 120_000) do
+           Pleroma.HTTP.get(url, [],
+             adapter: [pool: :media, timeout: 120_000, recv_timeout: 120_000]
+           ) do
       unzip(zip_body, dest)
     else
       e -> {:error, e}
diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index fec9d0efa..c36332817 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -83,17 +83,25 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
       end)
 
     {ref, state} = pop_in(state.client_monitors[client_pid])
-    Process.demonitor(ref)
-
-    timer =
-      if used_by == [] do
-        max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
-        Process.send_after(self(), :idle_close, max_idle)
+    # DOWN message can receive right after `remove_client` call and cause worker to terminate
+    state =
+      if is_nil(ref) do
+        state
       else
-        nil
+        Process.demonitor(ref)
+
+        timer =
+          if used_by == [] do
+            max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
+            Process.send_after(self(), :idle_close, max_idle)
+          else
+            nil
+          end
+
+        %{state | timer: timer}
       end
 
-    {:reply, :ok, %{state | timer: timer}, :hibernate}
+    {:reply, :ok, state, :hibernate}
   end
 
   @impl true
@@ -103,16 +111,21 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
     {:stop, :normal, state}
   end
 
+  @impl true
+  def handle_info({:gun_up, _pid, _protocol}, state) do
+    {:noreply, state, :hibernate}
+  end
+
   # Gracefully shutdown if the connection got closed without any streams left
   @impl true
   def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
     {:stop, :normal, state}
   end
 
-  # Otherwise, shutdown with an error
+  # Otherwise, wait for retry
   @impl true
-  def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do
-    {:stop, {:error, down_message}, state}
+  def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams}, state) do
+    {:noreply, state, :hibernate}
   end
 
   @impl true
diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index 9ec3836b0..0728cbaa2 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -11,7 +11,6 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @type proxy_type() :: :socks4 | :socks5
   @type host() :: charlist() | :inet.ip_address()
 
-  alias Pleroma.Config
   alias Pleroma.HTTP.AdapterHelper
   require Logger
 
@@ -44,27 +43,13 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @spec options(URI.t(), keyword()) :: keyword()
   def options(%URI{} = uri, opts \\ []) do
     @defaults
-    |> put_timeout()
     |> Keyword.merge(opts)
     |> adapter_helper().options(uri)
   end
 
-  # For Hackney, this is the time a connection can stay idle in the pool.
-  # For Gun, this is the timeout to receive a message from Gun.
-  defp put_timeout(opts) do
-    {config_key, default} =
-      if adapter() == Tesla.Adapter.Gun do
-        {:pools, Config.get([:pools, :default, :timeout], 5_000)}
-      else
-        {:hackney_pools, 10_000}
-      end
-
-    timeout = Config.get([config_key, opts[:pool], :timeout], default)
-
-    Keyword.merge(opts, timeout: timeout)
-  end
-
+  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
   def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
+
   defp adapter, do: Application.get_env(:tesla, :adapter)
 
   defp adapter_helper do
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index b4ff8306c..02e20f2d1 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.HTTP.AdapterHelper.Gun do
   @behaviour Pleroma.HTTP.AdapterHelper
 
+  alias Pleroma.Config
   alias Pleroma.Gun.ConnectionPool
   alias Pleroma.HTTP.AdapterHelper
 
@@ -14,31 +15,46 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     connect_timeout: 5_000,
     domain_lookup_timeout: 5_000,
     tls_handshake_timeout: 5_000,
-    retry: 0,
+    retry: 1,
     retry_timeout: 1000,
     await_up_timeout: 5_000
   ]
 
+  @type pool() :: :federation | :upload | :media | :default
+
   @spec options(keyword(), URI.t()) :: keyword()
   def options(incoming_opts \\ [], %URI{} = uri) do
     proxy =
-      Pleroma.Config.get([:http, :proxy_url])
+      [:http, :proxy_url]
+      |> Config.get()
       |> AdapterHelper.format_proxy()
 
-    config_opts = Pleroma.Config.get([:http, :adapter], [])
+    config_opts = Config.get([:http, :adapter], [])
 
     @defaults
     |> Keyword.merge(config_opts)
     |> add_scheme_opts(uri)
     |> AdapterHelper.maybe_add_proxy(proxy)
     |> Keyword.merge(incoming_opts)
+    |> put_timeout()
   end
 
   defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
 
   defp add_scheme_opts(opts, %{scheme: "https"}) do
-    opts
-    |> Keyword.put(:certificates_verification, true)
+    Keyword.put(opts, :certificates_verification, true)
+  end
+
+  defp put_timeout(opts) do
+    # this is the timeout to receive a message from Gun
+    Keyword.put_new(opts, :timeout, pool_timeout(opts[:pool]))
+  end
+
+  @spec pool_timeout(pool()) :: non_neg_integer()
+  def pool_timeout(pool) do
+    default = Config.get([:pools, :default, :timeout], 5_000)
+
+    Config.get([:pools, pool, :timeout], default)
   end
 
   @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
@@ -51,11 +67,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
 
   @prefix Pleroma.Gun.ConnectionPool
   def limiter_setup do
-    wait = Pleroma.Config.get([:connections_pool, :connection_acquisition_wait])
-    retries = Pleroma.Config.get([:connections_pool, :connection_acquisition_retries])
+    wait = Config.get([:connections_pool, :connection_acquisition_wait])
+    retries = Config.get([:connections_pool, :connection_acquisition_retries])
 
     :pools
-    |> Pleroma.Config.get([])
+    |> Config.get([])
     |> Enum.each(fn {name, opts} ->
       max_running = Keyword.get(opts, :size, 50)
       max_waiting = Keyword.get(opts, :max_waiting, 10)
@@ -69,7 +85,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
       case result do
         :ok -> :ok
         {:error, :existing} -> :ok
-        e -> raise e
       end
     end)
 
diff --git a/lib/pleroma/http/ex_aws.ex b/lib/pleroma/http/ex_aws.ex
index e53e64077..c3f335c73 100644
--- a/lib/pleroma/http/ex_aws.ex
+++ b/lib/pleroma/http/ex_aws.ex
@@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.ExAws do
 
   @impl true
   def request(method, url, body \\ "", headers \\ [], http_opts \\ []) do
+    http_opts = Keyword.put_new(http_opts, :adapter, pool: :upload)
+
     case HTTP.request(method, url, body, headers, http_opts) do
       {:ok, env} ->
         {:ok, %{status_code: env.status, headers: env.headers, body: env.body}}
diff --git a/lib/pleroma/http/tzdata.ex b/lib/pleroma/http/tzdata.ex
index 34bb253a7..4539ac359 100644
--- a/lib/pleroma/http/tzdata.ex
+++ b/lib/pleroma/http/tzdata.ex
@@ -11,6 +11,8 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def get(url, headers, options) do
+    options = Keyword.put_new(options, :adapter, pool: :default)
+
     with {:ok, %Tesla.Env{} = env} <- HTTP.get(url, headers, options) do
       {:ok, {env.status, env.headers, env.body}}
     end
@@ -18,6 +20,8 @@ defmodule Pleroma.HTTP.Tzdata do
 
   @impl true
   def head(url, headers, options) do
+    options = Keyword.put_new(options, :adapter, pool: :default)
+
     with {:ok, %Tesla.Env{} = env} <- HTTP.head(url, headers, options) do
       {:ok, {env.status, env.headers}}
     end
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index a1f935232..711c42158 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -150,7 +150,9 @@ defmodule Pleroma.Instances.Instance do
   defp scrape_favicon(%URI{} = instance_uri) do
     try do
       with {:ok, %Tesla.Env{body: html}} <-
-             Pleroma.HTTP.get(to_string(instance_uri), [{:Accept, "text/html"}]),
+             Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
+               adapter: [pool: :media]
+             ),
            favicon_rel <-
              html
              |> Floki.parse_document!()
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 6fdbc8efd..374d8704a 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -164,12 +164,12 @@ defmodule Pleroma.Object.Fetcher do
         date: date
       })
 
-    [{"signature", signature}]
+    {"signature", signature}
   end
 
   defp sign_fetch(headers, id, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ make_signature(id, date)
+      [make_signature(id, date) | headers]
     else
       headers
     end
@@ -177,7 +177,7 @@ defmodule Pleroma.Object.Fetcher do
 
   defp maybe_date_fetch(headers, date) do
     if Pleroma.Config.get([:activitypub, :sign_object_fetches]) do
-      headers ++ [{"date", date}]
+      [{"date", date} | headers]
     else
       headers
     end
diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex
index a13ff23b6..6dbef9085 100644
--- a/lib/pleroma/uploaders/s3.ex
+++ b/lib/pleroma/uploaders/s3.ex
@@ -46,12 +46,23 @@ defmodule Pleroma.Uploaders.S3 do
 
     op =
       if streaming do
-        upload.tempfile
-        |> ExAws.S3.Upload.stream_file()
-        |> ExAws.S3.upload(bucket, s3_name, [
-          {:acl, :public_read},
-          {:content_type, upload.content_type}
-        ])
+        op =
+          upload.tempfile
+          |> ExAws.S3.Upload.stream_file()
+          |> ExAws.S3.upload(bucket, s3_name, [
+            {:acl, :public_read},
+            {:content_type, upload.content_type}
+          ])
+
+        if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do
+          # set s3 upload timeout to respect :upload pool timeout
+          # timeout should be slightly larger, so s3 can retry upload on fail
+          timeout = Pleroma.HTTP.AdapterHelper.Gun.pool_timeout(:upload) + 1_000
+          opts = Keyword.put(op.opts, :timeout, timeout)
+          Map.put(op, :opts, opts)
+        else
+          op
+        end
       else
         {:ok, file_data} = File.read(upload.tempfile)
 
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 6210f2c5a..2fb482b51 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -96,6 +96,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do
         @rich_media_options
       end
 
-    Pleroma.HTTP.get(url, headers, options)
+    Pleroma.HTTP.get(url, headers, adapter: options)
   end
 end
diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index c4051e63e..6629f5356 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -136,12 +136,12 @@ defmodule Pleroma.Web.WebFinger do
 
   def find_lrdd_template(domain) do
     with {:ok, %{status: status, body: body}} when status in 200..299 <-
-           HTTP.get("http://#{domain}/.well-known/host-meta", []) do
+           HTTP.get("http://#{domain}/.well-known/host-meta") do
       get_template_from_xml(body)
     else
       _ ->
         with {:ok, %{body: body, status: status}} when status in 200..299 <-
-               HTTP.get("https://#{domain}/.well-known/host-meta", []) do
+               HTTP.get("https://#{domain}/.well-known/host-meta") do
           get_template_from_xml(body)
         else
           e -> {:error, "Can't find LRDD template: #{inspect(e)}"}
-- 
cgit v1.2.3


From 43b07c31cb88fbb87ff41a8e24303f930a657f44 Mon Sep 17 00:00:00 2001
From: feld <feld@feld.me>
Date: Wed, 2 Sep 2020 17:11:24 +0000
Subject: Merge branch 'fix/2095-deactivated-account-reset' into 'develop'

Disallow password resets for deactivated accounts

Closes #2095

See merge request pleroma/pleroma!2935
---
 .../web/mastodon_api/controllers/auth_controller.ex      | 16 +++++-----------
 lib/pleroma/web/twitter_api/twitter_api.ex               | 13 ++-----------
 2 files changed, 7 insertions(+), 22 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
index 753b3db3e..9f09550e1 100644
--- a/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/auth_controller.ex
@@ -59,17 +59,11 @@ defmodule Pleroma.Web.MastodonAPI.AuthController do
   def password_reset(conn, params) do
     nickname_or_email = params["email"] || params["nickname"]
 
-    with {:ok, _} <- TwitterAPI.password_reset(nickname_or_email) do
-      conn
-      |> put_status(:no_content)
-      |> json("")
-    else
-      {:error, "unknown user"} ->
-        send_resp(conn, :not_found, "")
-
-      {:error, _} ->
-        send_resp(conn, :bad_request, "")
-    end
+    TwitterAPI.password_reset(nickname_or_email)
+
+    conn
+    |> put_status(:no_content)
+    |> json("")
   end
 
   defp local_mastodon_root_path(conn) do
diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index 2294d9d0d..5d7948507 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -72,7 +72,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
   def password_reset(nickname_or_email) do
     with true <- is_binary(nickname_or_email),
-         %User{local: true, email: email} = user when is_binary(email) <-
+         %User{local: true, email: email, deactivated: false} = user when is_binary(email) <-
            User.get_by_nickname_or_email(nickname_or_email),
          {:ok, token_record} <- Pleroma.PasswordResetToken.create_token(user) do
       user
@@ -81,17 +81,8 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
       {:ok, :enqueued}
     else
-      false ->
-        {:error, "bad user identifier"}
-
-      %User{local: true, email: nil} ->
+      _ ->
         {:ok, :noop}
-
-      %User{local: false} ->
-        {:error, "remote user"}
-
-      nil ->
-        {:error, "unknown user"}
     end
   end
 
-- 
cgit v1.2.3


From 964bc8e2e8c9759985abaf9b8a5e26a2c5666fea Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 7 Sep 2020 11:12:30 +0000
Subject: Merge branch 'bugfix/favicon-too-long' into 'develop'

instance: Drop favicon URLs longer than 255 characters

Closes #2116

See merge request pleroma/pleroma!2949
---
 lib/pleroma/instances/instance.ex | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 711c42158..8bf53c090 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -14,6 +14,8 @@ defmodule Pleroma.Instances.Instance do
   import Ecto.Query
   import Ecto.Changeset
 
+  require Logger
+
   schema "instances" do
     field(:host, :string)
     field(:unreachable_since, :naive_datetime_usec)
@@ -145,6 +147,10 @@ defmodule Pleroma.Instances.Instance do
 
       favicon
     end
+  rescue
+    e ->
+      Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}")
+      nil
   end
 
   defp scrape_favicon(%URI{} = instance_uri) do
@@ -165,7 +171,12 @@ defmodule Pleroma.Instances.Instance do
         _ -> nil
       end
     rescue
-      _ -> nil
+      e ->
+        Logger.warn(
+          "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
+        )
+
+        nil
     end
   end
 end
-- 
cgit v1.2.3


From 9b85592b8bb5b61de6a67d087beecbb651ffb397 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 7 Sep 2020 10:19:19 +0000
Subject: Merge branch 'fix/rich-media-fake-statuses' into 'develop'

Rich Media: Do not cache URLs for preview statuses

Closes #1987

See merge request pleroma/pleroma!2956
---
 lib/pleroma/html.ex                   | 34 ++++++++++++++++++++--------------
 lib/pleroma/web/rich_media/helpers.ex |  2 +-
 2 files changed, 21 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex
index 20b02f091..43e9145be 100644
--- a/lib/pleroma/html.ex
+++ b/lib/pleroma/html.ex
@@ -100,21 +100,27 @@ defmodule Pleroma.HTML do
     end)
   end
 
-  def extract_first_external_url(_, nil), do: {:error, "No content"}
+  def extract_first_external_url_from_object(%{data: %{"content" => content}} = object)
+      when is_binary(content) do
+    unless object.data["fake"] do
+      key = "URL|#{object.id}"
+
+      Cachex.fetch!(:scrubber_cache, key, fn _key ->
+        {:commit, {:ok, extract_first_external_url(content)}}
+      end)
+    else
+      {:ok, extract_first_external_url(content)}
+    end
+  end
 
-  def extract_first_external_url(object, content) do
-    key = "URL|#{object.id}"
+  def extract_first_external_url_from_object(_), do: {:error, :no_content}
 
-    Cachex.fetch!(:scrubber_cache, key, fn _key ->
-      result =
-        content
-        |> Floki.parse_fragment!()
-        |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
-        |> Enum.take(1)
-        |> Floki.attribute("href")
-        |> Enum.at(0)
-
-      {:commit, {:ok, result}}
-    end)
+  def extract_first_external_url(content) do
+    content
+    |> Floki.parse_fragment!()
+    |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])")
+    |> Enum.take(1)
+    |> Floki.attribute("href")
+    |> Enum.at(0)
   end
 end
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 2fb482b51..752ca9f81 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -58,7 +58,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do
     with true <- Config.get([:rich_media, :enabled]),
          false <- object.data["sensitive"] || false,
          {:ok, page_url} <-
-           HTML.extract_first_external_url(object, object.data["content"]),
+           HTML.extract_first_external_url_from_object(object),
          :ok <- validate_page_url(page_url),
          {:ok, rich_media} <- Parser.parse(page_url) do
       %{page_url: page_url, rich_media: rich_media}
-- 
cgit v1.2.3


From 4b76d7ce67333fd9e09ce39102c3d5818309009c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 7 Sep 2020 11:38:15 +0000
Subject: Merge branch 'fix/admin-controller-atom-leak' into 'develop'

Fix atom leak in AdminAPIController

See merge request pleroma/secteam/pleroma!6
---
 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index aa2af1ab5..f5e4d49f9 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -379,8 +379,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     filters
     |> String.split(",")
     |> Enum.filter(&Enum.member?(@filters, &1))
-    |> Enum.map(&String.to_atom/1)
-    |> Map.new(&{&1, true})
+    |> Map.new(&{String.to_existing_atom(&1), true})
   end
 
   def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
-- 
cgit v1.2.3


From 87d2805791e1dd6746009e8c1445719e8cbfd31d Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 8 Sep 2020 17:29:28 +0300
Subject: combo fixes

---
 lib/pleroma/stats.ex | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/pleroma/stats.ex b/lib/pleroma/stats.ex
index e7f8d272c..e5c9c668b 100644
--- a/lib/pleroma/stats.ex
+++ b/lib/pleroma/stats.ex
@@ -23,6 +23,7 @@ defmodule Pleroma.Stats do
 
   @impl true
   def init(_args) do
+    if Pleroma.Config.get(:env) == :test, do: :ok = Ecto.Adapters.SQL.Sandbox.checkout(Repo)
     {:ok, nil, {:continue, :calculate_stats}}
   end
 
-- 
cgit v1.2.3


From ee0e05f9301e149c769f36bfd0fc8527ec7b6326 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 8 Sep 2020 10:43:57 +0200
Subject: Drop unused "inReplyToAtomUri" in objects

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 0831efadc..af4384213 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -168,7 +168,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def fix_in_reply_to(%{"inReplyTo" => in_reply_to} = object, options)
       when not is_nil(in_reply_to) do
     in_reply_to_id = prepare_in_reply_to(in_reply_to)
-    object = Map.put(object, "inReplyToAtomUri", in_reply_to_id)
     depth = (options[:depth] || 0) + 1
 
     if Federator.allowed_thread_distance?(depth) do
@@ -176,9 +175,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
            %Activity{} <- Activity.get_create_by_object_ap_id(replied_object.data["id"]) do
         object
         |> Map.put("inReplyTo", replied_object.data["id"])
-        |> Map.put("inReplyToAtomUri", object["inReplyToAtomUri"] || in_reply_to_id)
         |> Map.put("context", replied_object.data["context"] || object["conversation"])
-        |> Map.drop(["conversation"])
+        |> Map.drop(["conversation", "inReplyToAtomUri"])
       else
         e ->
           Logger.warn("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
-- 
cgit v1.2.3


From 921f926e96fd07131d4b79f5a29caed17ae2cc56 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 8 Sep 2020 09:13:11 +0200
Subject: Remove OStatus in testsuite

---
 lib/pleroma/object/containment.ex | 7 -------
 1 file changed, 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex
index bc88e8a0c..29cb3d672 100644
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@@ -44,13 +44,6 @@ defmodule Pleroma.Object.Containment do
     nil
   end
 
-  # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus
-  # objects being present in the test suite environment.  Once these objects are
-  # removed, please also remove this.
-  if Mix.env() == :test do
-    defp compare_uris(_, %URI{scheme: "tag"}), do: :ok
-  end
-
   defp compare_uris(%URI{host: host} = _id_uri, %URI{host: host} = _other_uri), do: :ok
   defp compare_uris(_id_uri, _other_uri), do: :error
 
-- 
cgit v1.2.3


From 7490b76bbfa0e39f7950d5611f00bddb2aac34ad Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 8 Sep 2020 12:21:33 +0000
Subject: Merge branch 'fix/upload-filter-exiftool-webp' into 'develop'

Skip processing webp image files because we can't strip EXIF

Closes #2080

See merge request pleroma/pleroma!2924
---
 lib/pleroma/upload/filter.ex                    | 13 ++++++++++---
 lib/pleroma/upload/filter/anonymize_filename.ex |  4 +++-
 lib/pleroma/upload/filter/dedupe.ex             |  4 ++--
 lib/pleroma/upload/filter/exiftool.ex           | 10 +++++++---
 lib/pleroma/upload/filter/mogrifun.ex           |  6 +++---
 lib/pleroma/upload/filter/mogrify.ex            |  6 +++---
 6 files changed, 28 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/upload/filter.ex b/lib/pleroma/upload/filter.ex
index dbdadc97f..661135634 100644
--- a/lib/pleroma/upload/filter.ex
+++ b/lib/pleroma/upload/filter.ex
@@ -15,7 +15,11 @@ defmodule Pleroma.Upload.Filter do
 
   require Logger
 
-  @callback filter(Pleroma.Upload.t()) :: :ok | {:ok, Pleroma.Upload.t()} | {:error, any()}
+  @callback filter(Pleroma.Upload.t()) ::
+              {:ok, :filtered}
+              | {:ok, :noop}
+              | {:ok, :filtered, Pleroma.Upload.t()}
+              | {:error, any()}
 
   @spec filter([module()], Pleroma.Upload.t()) :: {:ok, Pleroma.Upload.t()} | {:error, any()}
 
@@ -25,10 +29,13 @@ defmodule Pleroma.Upload.Filter do
 
   def filter([filter | rest], upload) do
     case filter.filter(upload) do
-      :ok ->
+      {:ok, :filtered} ->
         filter(rest, upload)
 
-      {:ok, upload} ->
+      {:ok, :filtered, upload} ->
+        filter(rest, upload)
+
+      {:ok, :noop} ->
         filter(rest, upload)
 
       error ->
diff --git a/lib/pleroma/upload/filter/anonymize_filename.ex b/lib/pleroma/upload/filter/anonymize_filename.ex
index 07ead8203..fc456e4f4 100644
--- a/lib/pleroma/upload/filter/anonymize_filename.ex
+++ b/lib/pleroma/upload/filter/anonymize_filename.ex
@@ -16,9 +16,11 @@ defmodule Pleroma.Upload.Filter.AnonymizeFilename do
   def filter(%Upload{name: name} = upload) do
     extension = List.last(String.split(name, "."))
     name = predefined_name(extension) || random(extension)
-    {:ok, %Upload{upload | name: name}}
+    {:ok, :filtered, %Upload{upload | name: name}}
   end
 
+  def filter(_), do: {:ok, :noop}
+
   @spec predefined_name(String.t()) :: String.t() | nil
   defp predefined_name(extension) do
     with name when not is_nil(name) <- Config.get([__MODULE__, :text]),
diff --git a/lib/pleroma/upload/filter/dedupe.ex b/lib/pleroma/upload/filter/dedupe.ex
index 41218a918..86cbc8996 100644
--- a/lib/pleroma/upload/filter/dedupe.ex
+++ b/lib/pleroma/upload/filter/dedupe.ex
@@ -17,8 +17,8 @@ defmodule Pleroma.Upload.Filter.Dedupe do
       |> Base.encode16(case: :lower)
 
     filename = shasum <> "." <> extension
-    {:ok, %Upload{upload | id: shasum, path: filename}}
+    {:ok, :filtered, %Upload{upload | id: shasum, path: filename}}
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end
diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex
index ea8798fe3..1fd0cfdaa 100644
--- a/lib/pleroma/upload/filter/exiftool.ex
+++ b/lib/pleroma/upload/filter/exiftool.ex
@@ -9,11 +9,15 @@ defmodule Pleroma.Upload.Filter.Exiftool do
   """
   @behaviour Pleroma.Upload.Filter
 
-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()}
+
+  # webp is not compatible with exiftool at this time
+  def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop}
+
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do
       case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do
-        {_response, 0} -> :ok
+        {_response, 0} -> {:ok, :filtered}
         {error, 1} -> {:error, error}
       end
     rescue
@@ -22,5 +26,5 @@ defmodule Pleroma.Upload.Filter.Exiftool do
     end
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end
diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex
index c8fa7b190..363e5cf0f 100644
--- a/lib/pleroma/upload/filter/mogrifun.ex
+++ b/lib/pleroma/upload/filter/mogrifun.ex
@@ -38,16 +38,16 @@ defmodule Pleroma.Upload.Filter.Mogrifun do
     [{"fill", "yellow"}, {"tint", "40"}]
   ]
 
-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do
       Filter.Mogrify.do_filter(file, [Enum.random(@filters)])
-      :ok
+      {:ok, :filtered}
     rescue
       _e in ErlangError ->
         {:error, "mogrify command not found"}
     end
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 end
diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex
index 7a45add5a..71968fd9c 100644
--- a/lib/pleroma/upload/filter/mogrify.ex
+++ b/lib/pleroma/upload/filter/mogrify.ex
@@ -8,18 +8,18 @@ defmodule Pleroma.Upload.Filter.Mogrify do
   @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()}
   @type conversions :: conversion() | [conversion()]
 
-  @spec filter(Pleroma.Upload.t()) :: :ok | {:error, String.t()}
+  @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()}
   def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do
     try do
       do_filter(file, Pleroma.Config.get!([__MODULE__, :args]))
-      :ok
+      {:ok, :filtered}
     rescue
       _e in ErlangError ->
         {:error, "mogrify command not found"}
     end
   end
 
-  def filter(_), do: :ok
+  def filter(_), do: {:ok, :noop}
 
   def do_filter(file, filters) do
     file
-- 
cgit v1.2.3


From 27c3973b78b432cef7feb9e2518bf143285c461d Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 7 Sep 2020 12:47:43 +0000
Subject: Merge branch 'temp-hackney-fix' into 'develop'

temp hackney fix

See merge request pleroma/pleroma!2951
---
 lib/pleroma/http/adapter_helper/hackney.ex  | 4 ++++
 lib/pleroma/reverse_proxy/client/hackney.ex | 1 +
 2 files changed, 5 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex
index cd569422b..62bd42485 100644
--- a/lib/pleroma/http/adapter_helper/hackney.ex
+++ b/lib/pleroma/http/adapter_helper/hackney.ex
@@ -22,6 +22,10 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
     |> Pleroma.HTTP.AdapterHelper.maybe_add_proxy(proxy)
   end
 
+  defp add_scheme_opts(opts, %URI{scheme: "https"}) do
+    Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
+  end
+
   defp add_scheme_opts(opts, _), do: opts
 
   @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()}
diff --git a/lib/pleroma/reverse_proxy/client/hackney.ex b/lib/pleroma/reverse_proxy/client/hackney.ex
index e84118a90..ad988fac3 100644
--- a/lib/pleroma/reverse_proxy/client/hackney.ex
+++ b/lib/pleroma/reverse_proxy/client/hackney.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.ReverseProxy.Client.Hackney do
 
   @impl true
   def request(method, url, headers, body, opts \\ []) do
+    opts = Keyword.put(opts, :ssl_options, versions: [:"tlsv1.2", :"tlsv1.1", :tlsv1])
     :hackney.request(method, url, headers, body, opts)
   end
 
-- 
cgit v1.2.3


From a781ac6ca5b7ab23eea795331db0a3fff406630e Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 10 Jun 2020 15:37:43 +0400
Subject: Fix atom leak in AdminAPIController

---
 lib/pleroma/web/admin_api/controllers/admin_api_controller.ex | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index aa2af1ab5..f5e4d49f9 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -379,8 +379,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     filters
     |> String.split(",")
     |> Enum.filter(&Enum.member?(@filters, &1))
-    |> Enum.map(&String.to_atom/1)
-    |> Map.new(&{&1, true})
+    |> Map.new(&{String.to_existing_atom(&1), true})
   end
 
   def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
-- 
cgit v1.2.3


From 10ef532c63431811b3998ed7b14aea21755a2b57 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 7 Jul 2020 07:06:29 +0200
Subject: AP C2S: Restrict character limit on Note

---
 .../web/activity_pub/activity_pub_controller.ex    | 37 ++++++++++++++--------
 1 file changed, 23 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 220c4fe52..732c44271 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -399,21 +399,30 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
 
   defp handle_user_activity(
          %User{} = user,
-         %{"type" => "Create", "object" => %{"type" => "Note"}} = params
+         %{"type" => "Create", "object" => %{"type" => "Note"} = object} = params
        ) do
-    object =
-      params["object"]
-      |> Map.merge(Map.take(params, ["to", "cc"]))
-      |> Map.put("attributedTo", user.ap_id())
-      |> Transmogrifier.fix_object()
-
-    ActivityPub.create(%{
-      to: params["to"],
-      actor: user,
-      context: object["context"],
-      object: object,
-      additional: Map.take(params, ["cc"])
-    })
+    content = if is_binary(object["content"]), do: object["content"], else: ""
+    name = if is_binary(object["name"]), do: object["name"], else: ""
+    summary = if is_binary(object["summary"]), do: object["summary"], else: ""
+    length = String.length(content <> name <> summary)
+
+    if length > Pleroma.Config.get([:instance, :limit]) do
+      {:error, dgettext("errors", "Note is over the character limit")}
+    else
+      object =
+        object
+        |> Map.merge(Map.take(params, ["to", "cc"]))
+        |> Map.put("attributedTo", user.ap_id())
+        |> Transmogrifier.fix_object()
+
+      ActivityPub.create(%{
+        to: params["to"],
+        actor: user,
+        context: object["context"],
+        object: object,
+        additional: Map.take(params, ["cc"])
+      })
+    end
   end
 
   defp handle_user_activity(%User{} = user, %{"type" => "Delete"} = params) do
-- 
cgit v1.2.3


From 16c451f8f15b1b2907fb6fc40925b47821650f31 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Wed, 2 Sep 2020 20:11:24 +0200
Subject: search: Apply following filter only when user is usable

---
 lib/pleroma/user/search.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index adbef7fb8..7babd47ea 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -115,8 +115,8 @@ defmodule Pleroma.User.Search do
     )
   end
 
-  defp base_query(_user, false), do: User
-  defp base_query(user, true), do: User.get_friends_query(user)
+  defp base_query(%User{} = user, true), do: User.get_friends_query(user)
+  defp base_query(_user, _following), do: User
 
   defp filter_invisible_users(query) do
     from(q in query, where: q.invisible == false)
-- 
cgit v1.2.3


From 947ee55ae298a42c2667800c1aac96f637e31969 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Wed, 2 Sep 2020 20:24:03 +0200
Subject: user: harden get_friends_query(), get_followers_query() and their
 wrappers

---
 lib/pleroma/user.ex | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 94c96de8d..f323fc6ed 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1125,31 +1125,31 @@ defmodule Pleroma.User do
     User.Query.build(%{followers: user, deactivated: false})
   end
 
-  def get_followers_query(user, page) do
+  def get_followers_query(%User{} = user, page) do
     user
     |> get_followers_query(nil)
     |> User.Query.paginate(page, 20)
   end
 
   @spec get_followers_query(User.t()) :: Ecto.Query.t()
-  def get_followers_query(user), do: get_followers_query(user, nil)
+  def get_followers_query(%User{} = user), do: get_followers_query(user, nil)
 
   @spec get_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
-  def get_followers(user, page \\ nil) do
+  def get_followers(%User{} = user, page \\ nil) do
     user
     |> get_followers_query(page)
     |> Repo.all()
   end
 
   @spec get_external_followers(User.t(), pos_integer() | nil) :: {:ok, list(User.t())}
-  def get_external_followers(user, page \\ nil) do
+  def get_external_followers(%User{} = user, page \\ nil) do
     user
     |> get_followers_query(page)
     |> User.Query.build(%{external: true})
     |> Repo.all()
   end
 
-  def get_followers_ids(user, page \\ nil) do
+  def get_followers_ids(%User{} = user, page \\ nil) do
     user
     |> get_followers_query(page)
     |> select([u], u.id)
@@ -1161,29 +1161,29 @@ defmodule Pleroma.User do
     User.Query.build(%{friends: user, deactivated: false})
   end
 
-  def get_friends_query(user, page) do
+  def get_friends_query(%User{} = user, page) do
     user
     |> get_friends_query(nil)
     |> User.Query.paginate(page, 20)
   end
 
   @spec get_friends_query(User.t()) :: Ecto.Query.t()
-  def get_friends_query(user), do: get_friends_query(user, nil)
+  def get_friends_query(%User{} = user), do: get_friends_query(user, nil)
 
-  def get_friends(user, page \\ nil) do
+  def get_friends(%User{} = user, page \\ nil) do
     user
     |> get_friends_query(page)
     |> Repo.all()
   end
 
-  def get_friends_ap_ids(user) do
+  def get_friends_ap_ids(%User{} = user) do
     user
     |> get_friends_query(nil)
     |> select([u], u.ap_id)
     |> Repo.all()
   end
 
-  def get_friends_ids(user, page \\ nil) do
+  def get_friends_ids(%User{} = user, page \\ nil) do
     user
     |> get_friends_query(page)
     |> select([u], u.id)
-- 
cgit v1.2.3


From 630444ee0819ad5b58c5f9030758fe41e6fed530 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 31 Aug 2020 14:19:48 -0500
Subject: Do not make RelMe metadata provider optional.

There's really no sound reason to turn this off anyway.
---
 lib/pleroma/web/metadata.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex
index a9f70c43e..e45e74e7b 100644
--- a/lib/pleroma/web/metadata.ex
+++ b/lib/pleroma/web/metadata.ex
@@ -7,7 +7,8 @@ defmodule Pleroma.Web.Metadata do
 
   def build_tags(params) do
     providers = [
-      Pleroma.Web.Metadata.Providers.RestrictIndexing
+      Pleroma.Web.Metadata.Providers.RestrictIndexing,
+      Pleroma.Web.Metadata.Providers.RelMe,
       | Pleroma.Config.get([__MODULE__, :providers], [])
     ]
 
-- 
cgit v1.2.3


From ff07014b2657730101e826d7e82716989d43214c Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 31 Aug 2020 14:35:22 -0500
Subject: Disable providers of user and status metadata when instance is
 private

---
 lib/pleroma/web/metadata.ex | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex
index e45e74e7b..0f0b56321 100644
--- a/lib/pleroma/web/metadata.ex
+++ b/lib/pleroma/web/metadata.ex
@@ -8,8 +8,8 @@ defmodule Pleroma.Web.Metadata do
   def build_tags(params) do
     providers = [
       Pleroma.Web.Metadata.Providers.RestrictIndexing,
-      Pleroma.Web.Metadata.Providers.RelMe,
-      | Pleroma.Config.get([__MODULE__, :providers], [])
+      Pleroma.Web.Metadata.Providers.RelMe
+      | activated_providers()
     ]
 
     Enum.reduce(providers, "", fn parser, acc ->
@@ -43,4 +43,12 @@ defmodule Pleroma.Web.Metadata do
   def activity_nsfw?(_) do
     false
   end
+
+  defp activated_providers do
+    if Pleroma.Config.get!([:instance, :public]) do
+      Pleroma.Config.get([__MODULE__, :providers], [])
+    else
+      []
+    end
+  end
 end
-- 
cgit v1.2.3


From 14d07081fd82211071eafb3c31d8c756fe9af9f5 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 31 Aug 2020 14:48:22 -0500
Subject: Feed provider only generates a redirect, so always activate it.

Making this configurable is misleading.
---
 lib/pleroma/web/metadata.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex
index 0f0b56321..926b5b187 100644
--- a/lib/pleroma/web/metadata.ex
+++ b/lib/pleroma/web/metadata.ex
@@ -7,8 +7,9 @@ defmodule Pleroma.Web.Metadata do
 
   def build_tags(params) do
     providers = [
-      Pleroma.Web.Metadata.Providers.RestrictIndexing,
-      Pleroma.Web.Metadata.Providers.RelMe
+      Pleroma.Web.Metadata.Providers.Feed,
+      Pleroma.Web.Metadata.Providers.RelMe,
+      Pleroma.Web.Metadata.Providers.RestrictIndexing
       | activated_providers()
     ]
 
-- 
cgit v1.2.3


From a85ed6defbd2cec71d9a5594ef1de18d5333c7c7 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 31 Aug 2020 15:58:21 -0500
Subject: Do not serve RSS/Atom feeds when instance is private

---
 lib/pleroma/web/feed/tag_controller.ex  | 10 +++++++++-
 lib/pleroma/web/feed/user_controller.ex | 10 +++++++++-
 lib/pleroma/web/metadata/feed.ex        | 20 ++++++++++++--------
 3 files changed, 30 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex
index 39b2a766a..e090dd625 100644
--- a/lib/pleroma/web/feed/tag_controller.ex
+++ b/lib/pleroma/web/feed/tag_controller.ex
@@ -9,7 +9,15 @@ defmodule Pleroma.Web.Feed.TagController do
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.Feed.FeedView
 
-  def feed(conn, %{"tag" => raw_tag} = params) do
+  def feed(conn, params) do
+    if Pleroma.Config.get!([:instance, :public]) do
+      render_feed(conn, params)
+    else
+      render_error(conn, :not_found, "Not found")
+    end
+  end
+
+  def render_feed(conn, %{"tag" => raw_tag} = params) do
     {format, tag} = parse_tag(raw_tag)
 
     activities =
diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 9cd334a33..595889b9d 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -37,7 +37,15 @@ defmodule Pleroma.Web.Feed.UserController do
     end
   end
 
-  def feed(conn, %{"nickname" => nickname} = params) do
+  def feed(conn, params) do
+    if Pleroma.Config.get!([:instance, :public]) do
+      render_feed(conn, params)
+    else
+      errors(conn, {:error, :not_found})
+    end
+  end
+
+  def render_feed(conn, %{"nickname" => nickname} = params) do
     format = get_format(conn)
 
     format =
diff --git a/lib/pleroma/web/metadata/feed.ex b/lib/pleroma/web/metadata/feed.ex
index bd1459a17..dfe391b56 100644
--- a/lib/pleroma/web/metadata/feed.ex
+++ b/lib/pleroma/web/metadata/feed.ex
@@ -11,13 +11,17 @@ defmodule Pleroma.Web.Metadata.Providers.Feed do
 
   @impl Provider
   def build_tags(%{user: user}) do
-    [
-      {:link,
-       [
-         rel: "alternate",
-         type: "application/atom+xml",
-         href: Helpers.user_feed_path(Endpoint, :feed, user.nickname) <> ".atom"
-       ], []}
-    ]
+    if Pleroma.Config.get!([:instance, :public]) do
+      [
+        {:link,
+         [
+           rel: "alternate",
+           type: "application/atom+xml",
+           href: Helpers.user_feed_path(Endpoint, :feed, user.nickname) <> ".atom"
+         ], []}
+      ]
+    else
+      []
+    end
   end
 end
-- 
cgit v1.2.3


From 2011142ed9ae45f53496b3682da7114255c814a5 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 1 Sep 2020 10:43:44 -0500
Subject: Use :restrict_unauthenticated testing for more granular control

---
 lib/pleroma/web/feed/tag_controller.ex  | 2 +-
 lib/pleroma/web/feed/user_controller.ex | 2 +-
 lib/pleroma/web/metadata.ex             | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/feed/tag_controller.ex b/lib/pleroma/web/feed/tag_controller.ex
index e090dd625..93a8294b7 100644
--- a/lib/pleroma/web/feed/tag_controller.ex
+++ b/lib/pleroma/web/feed/tag_controller.ex
@@ -10,7 +10,7 @@ defmodule Pleroma.Web.Feed.TagController do
   alias Pleroma.Web.Feed.FeedView
 
   def feed(conn, params) do
-    if Pleroma.Config.get!([:instance, :public]) do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
       render_feed(conn, params)
     else
       render_error(conn, :not_found, "Not found")
diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex
index 595889b9d..71eb1ea7e 100644
--- a/lib/pleroma/web/feed/user_controller.ex
+++ b/lib/pleroma/web/feed/user_controller.ex
@@ -38,7 +38,7 @@ defmodule Pleroma.Web.Feed.UserController do
   end
 
   def feed(conn, params) do
-    if Pleroma.Config.get!([:instance, :public]) do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:profiles, :local) do
       render_feed(conn, params)
     else
       errors(conn, {:error, :not_found})
diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex
index 926b5b187..68835c826 100644
--- a/lib/pleroma/web/metadata.ex
+++ b/lib/pleroma/web/metadata.ex
@@ -46,7 +46,7 @@ defmodule Pleroma.Web.Metadata do
   end
 
   defp activated_providers do
-    if Pleroma.Config.get!([:instance, :public]) do
+    unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
       Pleroma.Config.get([__MODULE__, :providers], [])
     else
       []
-- 
cgit v1.2.3


From 0d2814ec8e41942cd5b056d9c1ed902be1e1773c Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 7 Sep 2020 15:06:06 +0300
Subject: Metadata: Move restriction check from Feed provider to
 activated_providers

---
 lib/pleroma/web/metadata.ex      |  3 +--
 lib/pleroma/web/metadata/feed.ex | 20 ++++++++------------
 2 files changed, 9 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/metadata.ex b/lib/pleroma/web/metadata.ex
index 68835c826..0f2d8d1e7 100644
--- a/lib/pleroma/web/metadata.ex
+++ b/lib/pleroma/web/metadata.ex
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.Metadata do
 
   def build_tags(params) do
     providers = [
-      Pleroma.Web.Metadata.Providers.Feed,
       Pleroma.Web.Metadata.Providers.RelMe,
       Pleroma.Web.Metadata.Providers.RestrictIndexing
       | activated_providers()
@@ -47,7 +46,7 @@ defmodule Pleroma.Web.Metadata do
 
   defp activated_providers do
     unless Pleroma.Config.restrict_unauthenticated_access?(:activities, :local) do
-      Pleroma.Config.get([__MODULE__, :providers], [])
+      [Pleroma.Web.Metadata.Providers.Feed | Pleroma.Config.get([__MODULE__, :providers], [])]
     else
       []
     end
diff --git a/lib/pleroma/web/metadata/feed.ex b/lib/pleroma/web/metadata/feed.ex
index dfe391b56..bd1459a17 100644
--- a/lib/pleroma/web/metadata/feed.ex
+++ b/lib/pleroma/web/metadata/feed.ex
@@ -11,17 +11,13 @@ defmodule Pleroma.Web.Metadata.Providers.Feed do
 
   @impl Provider
   def build_tags(%{user: user}) do
-    if Pleroma.Config.get!([:instance, :public]) do
-      [
-        {:link,
-         [
-           rel: "alternate",
-           type: "application/atom+xml",
-           href: Helpers.user_feed_path(Endpoint, :feed, user.nickname) <> ".atom"
-         ], []}
-      ]
-    else
-      []
-    end
+    [
+      {:link,
+       [
+         rel: "alternate",
+         type: "application/atom+xml",
+         href: Helpers.user_feed_path(Endpoint, :feed, user.nickname) <> ".atom"
+       ], []}
+    ]
   end
 end
-- 
cgit v1.2.3


From 68a74d66596f0e35f0e080de25e4679d2c8b1b76 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 9 Sep 2020 19:30:42 +0300
Subject: [#2497] Added missing alias, removed legacy `:adapter` option
 specification for HTTP.get/_.

---
 lib/pleroma/helpers/media_helper.ex                   | 4 ++--
 lib/pleroma/instances/instance.ex                     | 2 +-
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index a1205e10d..d834b4a07 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   def image_resize(url, options) do
     with executable when is_binary(executable) <- System.find_executable("convert"),
          {:ok, args} <- prepare_image_resize_args(options),
-         {:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]),
+         {:ok, env} <- HTTP.get(url, [], pool: :media),
          {:ok, fifo_path} <- mkfifo() do
       args = List.flatten([fifo_path, args])
       run_fifo(fifo_path, env, executable, args)
@@ -62,7 +62,7 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
-         {:ok, env} <- HTTP.get(url, [], adapter: [pool: :media]),
+         {:ok, env} <- HTTP.get(url, [], pool: :media),
          {:ok, fifo_path} <- mkfifo(),
          args = [
            "-y",
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 8bf53c090..4fe4b198d 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -157,7 +157,7 @@ defmodule Pleroma.Instances.Instance do
     try do
       with {:ok, %Tesla.Env{body: html}} <-
              Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
-               adapter: [pool: :media]
+               pool: :media
              ),
            favicon_rel <-
              html
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 89f4a23bd..acb581459 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -51,7 +51,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     media_proxy_url = MediaProxy.url(url)
 
     with {:ok, %{status: status} = head_response} when status in 200..299 <-
-           Pleroma.HTTP.request("head", media_proxy_url, [], [], adapter: [pool: :media]) do
+           Pleroma.HTTP.request("head", media_proxy_url, [], [], pool: :media) do
       content_type = Tesla.get_header(head_response, "content-type")
       handle_preview(content_type, conn, media_proxy_url)
     else
-- 
cgit v1.2.3


From b4860c57a63b48ded8eaa37b9f40cc0851c78882 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 9 Sep 2020 19:43:36 +0300
Subject: [#2497] Formatting fix.

---
 lib/pleroma/instances/instance.ex | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 4fe4b198d..ad7764f05 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -156,9 +156,7 @@ defmodule Pleroma.Instances.Instance do
   defp scrape_favicon(%URI{} = instance_uri) do
     try do
       with {:ok, %Tesla.Env{body: html}} <-
-             Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
-               pool: :media
-             ),
+             Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], pool: :media),
            favicon_rel <-
              html
              |> Floki.parse_document!()
-- 
cgit v1.2.3


From cad69669fc692da360929a5961e96550de1f1fe1 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 9 Sep 2020 22:44:38 +0300
Subject: [#2130] Fixed OAuth OOB authentication for users with enabled MFA.

---
 lib/pleroma/web/oauth/oauth_controller.ex                            | 5 ++++-
 .../web/templates/o_auth/o_auth/oob_authorization_created.html.eex   | 2 +-
 lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex    | 2 +-
 3 files changed, 6 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index dd00600ea..06b116368 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -145,7 +145,10 @@ defmodule Pleroma.Web.OAuth.OAuthController do
   def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{
         "authorization" => %{"redirect_uri" => @oob_token_redirect_uri}
       }) do
-    render(conn, "oob_authorization_created.html", %{auth: auth})
+    # Enforcing the view to reuse the template when calling from other controllers
+    conn
+    |> put_view(OAuthView)
+    |> render("oob_authorization_created.html", %{auth: auth})
   end
 
   def after_create_authorization(%Plug.Conn{} = conn, %Authorization{} = auth, %{
diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex
index 8443d906b..ffabe29a6 100644
--- a/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex
+++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_authorization_created.html.eex
@@ -1,2 +1,2 @@
 <h1>Successfully authorized</h1>
-<h2>Token code is <%= @auth.token %></h2>
+<h2>Token code is <br><%= @auth.token %></h2>
diff --git a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex
index 961aad976..82785c4b9 100644
--- a/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex
+++ b/lib/pleroma/web/templates/o_auth/o_auth/oob_token_exists.html.eex
@@ -1,2 +1,2 @@
 <h1>Authorization exists</h1>
-<h2>Access token is <%= @token.token %></h2>
+<h2>Access token is <br><%= @token.token %></h2>
-- 
cgit v1.2.3


From ab56dd54e787eae82cf00fddc90eab4c5cbac4a9 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Thu, 10 Sep 2020 11:23:39 +0300
Subject: use Pleroma.HTTP in emoji packs tasks

---
 lib/mix/tasks/pleroma/emoji.ex | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index 8f52ee98d..1750373f9 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -183,7 +183,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
     IO.puts("Downloading the pack and generating SHA256")
 
-    binary_archive = Tesla.get!(client(), src).body
+    {:ok, %{body: binary_archive}} = Pleroma.HTTP.get(src)
     archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16()
 
     IO.puts("SHA256 is #{archive_sha}")
@@ -252,7 +252,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
   end
 
   defp fetch("http" <> _ = from) do
-    with {:ok, %{body: body}} <- Tesla.get(client(), from) do
+    with {:ok, %{body: body}} <- Pleroma.HTTP.get(from) do
       {:ok, body}
     end
   end
@@ -271,13 +271,5 @@ defmodule Mix.Tasks.Pleroma.Emoji do
     )
   end
 
-  defp client do
-    middleware = [
-      {Tesla.Middleware.FollowRedirects, [max_redirects: 3]}
-    ]
-
-    Tesla.client(middleware)
-  end
-
   defp default_manifest, do: Pleroma.Config.get!([:emoji, :default_manifest])
 end
-- 
cgit v1.2.3


From 148bc244359e70c87ec2812c65da83fe87efbc68 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 10 Sep 2020 11:54:10 +0300
Subject: [#2497] Removed Hackney-specific code (no longer needed due to
 adapter options unification).

---
 .../web/activity_pub/mrf/media_proxy_warming_policy.ex      | 13 +++----------
 1 file changed, 3 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
index 6c63fe15c..0fb05d3c4 100644
--- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex
@@ -13,17 +13,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
   require Logger
 
   @adapter_options [
-    pool: :media
+    pool: :media,
+    recv_timeout: 10_000
   ]
 
-  defp adapter_options do
-    if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Hackney do
-      Keyword.put(@adapter_options, :recv_timeout, 10_000)
-    else
-      @adapter_options
-    end
-  end
-
   def perform(:prefetch, url) do
     # Fetching only proxiable resources
     if MediaProxy.enabled?() and MediaProxy.url_proxiable?(url) do
@@ -32,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do
 
       Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}")
 
-      HTTP.get(prefetch_url, [], adapter: adapter_options())
+      HTTP.get(prefetch_url, [], @adapter_options)
     end
   end
 
-- 
cgit v1.2.3


From 3ce658b93098551792a69f2455e6e9339a1722e2 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 25 Aug 2020 19:17:51 +0300
Subject: schedule expired oauth tokens deletion with Oban

---
 lib/pleroma/web/oauth/token.ex                     | 24 ++++++++++++-------
 lib/pleroma/web/oauth/token/clean_worker.ex        |  2 --
 lib/pleroma/web/oauth/token/query.ex               |  6 -----
 .../web/oauth/token/strategy/refresh_token.ex      |  2 +-
 .../workers/cron/clear_oauth_token_worker.ex       | 23 ------------------
 lib/pleroma/workers/purge_expired_token.ex         | 28 ++++++++++++++++++++++
 6 files changed, 45 insertions(+), 40 deletions(-)
 delete mode 100644 lib/pleroma/workers/cron/clear_oauth_token_worker.ex
 create mode 100644 lib/pleroma/workers/purge_expired_token.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex
index 08bb7326d..4d00fcb1c 100644
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@@ -50,7 +50,7 @@ defmodule Pleroma.Web.OAuth.Token do
          true <- auth.app_id == app.id do
       user = if auth.user_id, do: User.get_cached_by_id(auth.user_id), else: %User{}
 
-      create_token(
+      create(
         app,
         user,
         %{scopes: auth.scopes}
@@ -83,8 +83,21 @@ defmodule Pleroma.Web.OAuth.Token do
     |> validate_required([:valid_until])
   end
 
-  @spec create_token(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()}
-  def create_token(%App{} = app, %User{} = user, attrs \\ %{}) do
+  @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()}
+  def create(%App{} = app, %User{} = user, attrs \\ %{}) do
+    with {:ok, token} <- do_create(app, user, attrs) do
+      if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do
+        Pleroma.Workers.PurgeExpiredOAuthToken.enqueue(%{
+          token_id: token.id,
+          valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC")
+        })
+      end
+
+      {:ok, token}
+    end
+  end
+
+  defp do_create(app, user, attrs) do
     %__MODULE__{user_id: user.id, app_id: app.id}
     |> cast(%{scopes: attrs[:scopes] || app.scopes}, [:scopes])
     |> validate_required([:scopes, :app_id])
@@ -105,11 +118,6 @@ defmodule Pleroma.Web.OAuth.Token do
     |> Repo.delete_all()
   end
 
-  def delete_expired_tokens do
-    Query.get_expired_tokens()
-    |> Repo.delete_all()
-  end
-
   def get_user_tokens(%User{id: user_id}) do
     Query.get_by_user(user_id)
     |> Query.preload([:app])
diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex
index e3aa4eb7e..2f51bdb75 100644
--- a/lib/pleroma/web/oauth/token/clean_worker.ex
+++ b/lib/pleroma/web/oauth/token/clean_worker.ex
@@ -12,7 +12,6 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do
   @one_day 86_400_000
 
   alias Pleroma.MFA
-  alias Pleroma.Web.OAuth
   alias Pleroma.Workers.BackgroundWorker
 
   def start_link(_), do: GenServer.start_link(__MODULE__, %{})
@@ -32,7 +31,6 @@ defmodule Pleroma.Web.OAuth.Token.CleanWorker do
   end
 
   def perform(:clean) do
-    OAuth.Token.delete_expired_tokens()
     MFA.Token.delete_expired_tokens()
   end
 end
diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex
index 93d6e26ed..fd6d9b112 100644
--- a/lib/pleroma/web/oauth/token/query.ex
+++ b/lib/pleroma/web/oauth/token/query.ex
@@ -33,12 +33,6 @@ defmodule Pleroma.Web.OAuth.Token.Query do
     from(q in query, where: q.id == ^id)
   end
 
-  @spec get_expired_tokens(query, DateTime.t() | nil) :: query
-  def get_expired_tokens(query \\ Token, date \\ nil) do
-    expired_date = date || Timex.now()
-    from(q in query, where: fragment("?", q.valid_until) < ^expired_date)
-  end
-
   @spec get_by_user(query, String.t()) :: query
   def get_by_user(query \\ Token, user_id) do
     from(q in query, where: q.user_id == ^user_id)
diff --git a/lib/pleroma/web/oauth/token/strategy/refresh_token.ex b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex
index debc29b0b..625b0fde2 100644
--- a/lib/pleroma/web/oauth/token/strategy/refresh_token.ex
+++ b/lib/pleroma/web/oauth/token/strategy/refresh_token.ex
@@ -46,7 +46,7 @@ defmodule Pleroma.Web.OAuth.Token.Strategy.RefreshToken do
   defp create_access_token({:error, error}, _), do: {:error, error}
 
   defp create_access_token({:ok, token}, %{app: app, user: user} = token_params) do
-    Token.create_token(app, user, add_refresh_token(token_params, token.refresh_token))
+    Token.create(app, user, add_refresh_token(token_params, token.refresh_token))
   end
 
   defp add_refresh_token(params, token) do
diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex
deleted file mode 100644
index 276f47efc..000000000
--- a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex
+++ /dev/null
@@ -1,23 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do
-  @moduledoc """
-  The worker to cleanup expired oAuth tokens.
-  """
-
-  use Oban.Worker, queue: "background"
-
-  alias Pleroma.Config
-  alias Pleroma.Web.OAuth.Token
-
-  @impl Oban.Worker
-  def perform(_job) do
-    if Config.get([:oauth2, :clean_expired_tokens], false) do
-      Token.delete_expired_tokens()
-    end
-
-    :ok
-  end
-end
diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex
new file mode 100644
index 000000000..6068e43bf
--- /dev/null
+++ b/lib/pleroma/workers/purge_expired_token.ex
@@ -0,0 +1,28 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Workers.PurgeExpiredOAuthToken do
+  @moduledoc """
+  Worker which purges expired OAuth tokens
+  """
+
+  use Oban.Worker, queue: :oauth_token_expiration, max_attempts: 1
+
+  @spec enqueue(%{token_id: integer(), valid_until: DateTime.t()}) ::
+          {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
+  def enqueue(args) do
+    {scheduled_at, args} = Map.pop(args, :valid_until)
+
+    args
+    |> __MODULE__.new(scheduled_at: scheduled_at)
+    |> Oban.insert()
+  end
+
+  @impl true
+  def perform(%Oban.Job{args: %{"token_id" => id}}) do
+    Pleroma.Web.OAuth.Token
+    |> Pleroma.Repo.get(id)
+    |> Pleroma.Repo.delete()
+  end
+end
-- 
cgit v1.2.3


From 7dd986a563545cb63e8404d9b107f1d29c499940 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sat, 5 Sep 2020 18:35:01 +0300
Subject: expire mfa tokens  through Oban

---
 lib/pleroma/mfa/token.ex                           | 71 +++++++++++-----------
 lib/pleroma/web/oauth/oauth_controller.ex          |  4 +-
 lib/pleroma/web/oauth/token.ex                     |  5 +-
 lib/pleroma/web/oauth/token/clean_worker.ex        | 36 -----------
 .../controllers/remote_follow_controller.ex        |  2 +-
 lib/pleroma/workers/purge_expired_token.ex         | 11 ++--
 6 files changed, 47 insertions(+), 82 deletions(-)
 delete mode 100644 lib/pleroma/web/oauth/token/clean_worker.ex

(limited to 'lib')

diff --git a/lib/pleroma/mfa/token.ex b/lib/pleroma/mfa/token.ex
index 0b2449971..69b64c0e8 100644
--- a/lib/pleroma/mfa/token.ex
+++ b/lib/pleroma/mfa/token.ex
@@ -10,10 +10,11 @@ defmodule Pleroma.MFA.Token do
   alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.OAuth.Authorization
-  alias Pleroma.Web.OAuth.Token, as: OAuthToken
 
   @expires 300
 
+  @type t() :: %__MODULE__{}
+
   schema "mfa_tokens" do
     field(:token, :string)
     field(:valid_until, :naive_datetime_usec)
@@ -24,6 +25,7 @@ defmodule Pleroma.MFA.Token do
     timestamps()
   end
 
+  @spec get_by_token(String.t()) :: {:ok, t()} | {:error, :not_found}
   def get_by_token(token) do
     from(
       t in __MODULE__,
@@ -33,33 +35,40 @@ defmodule Pleroma.MFA.Token do
     |> Repo.find_resource()
   end
 
-  def validate(token) do
-    with {:fetch_token, {:ok, token}} <- {:fetch_token, get_by_token(token)},
-         {:expired, false} <- {:expired, is_expired?(token)} do
+  @spec validate(String.t()) :: {:ok, t()} | {:error, :not_found} | {:error, :expired_token}
+  def validate(token_str) do
+    with {:ok, token} <- get_by_token(token_str),
+         false <- expired?(token) do
       {:ok, token}
-    else
-      {:expired, _} -> {:error, :expired_token}
-      {:fetch_token, _} -> {:error, :not_found}
-      error -> {:error, error}
     end
   end
 
-  def create_token(%User{} = user) do
-    %__MODULE__{}
-    |> change
-    |> assign_user(user)
-    |> put_token
-    |> put_valid_until
-    |> Repo.insert()
+  defp expired?(%__MODULE__{valid_until: valid_until}) do
+    with true <- NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0 do
+      {:error, :expired_token}
+    end
+  end
+
+  @spec create(User.t(), Authorization.t() | nil) :: {:ok, t()} | {:error, Ecto.Changeset.t()}
+  def create(user, authorization \\ nil) do
+    with {:ok, token} <- do_create(user, authorization) do
+      Pleroma.Workers.PurgeExpiredToken.enqueue(%{
+        token_id: token.id,
+        valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"),
+        mod: __MODULE__
+      })
+
+      {:ok, token}
+    end
   end
 
-  def create_token(user, authorization) do
+  defp do_create(user, authorization) do
     %__MODULE__{}
-    |> change
+    |> change()
     |> assign_user(user)
-    |> assign_authorization(authorization)
-    |> put_token
-    |> put_valid_until
+    |> maybe_assign_authorization(authorization)
+    |> put_token()
+    |> put_valid_until()
     |> Repo.insert()
   end
 
@@ -69,15 +78,19 @@ defmodule Pleroma.MFA.Token do
     |> validate_required([:user])
   end
 
-  defp assign_authorization(changeset, authorization) do
+  defp maybe_assign_authorization(changeset, %Authorization{} = authorization) do
     changeset
     |> put_assoc(:authorization, authorization)
     |> validate_required([:authorization])
   end
 
+  defp maybe_assign_authorization(changeset, _), do: changeset
+
   defp put_token(changeset) do
+    token = Pleroma.Web.OAuth.Token.Utils.generate_token()
+
     changeset
-    |> change(%{token: OAuthToken.Utils.generate_token()})
+    |> change(%{token: token})
     |> validate_required([:token])
     |> unique_constraint(:token)
   end
@@ -89,18 +102,4 @@ defmodule Pleroma.MFA.Token do
     |> change(%{valid_until: expires_in})
     |> validate_required([:valid_until])
   end
-
-  def is_expired?(%__MODULE__{valid_until: valid_until}) do
-    NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0
-  end
-
-  def is_expired?(_), do: false
-
-  def delete_expired_tokens do
-    from(
-      q in __MODULE__,
-      where: fragment("?", q.valid_until) < ^Timex.now()
-    )
-    |> Repo.delete_all()
-  end
 end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index dd00600ea..bbe7aa8a0 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -197,7 +197,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
          {:mfa_required, user, auth, _},
          params
        ) do
-    {:ok, token} = MFA.Token.create_token(user, auth)
+    {:ok, token} = MFA.Token.create(user, auth)
 
     data = %{
       "mfa_token" => token.token,
@@ -579,7 +579,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
     do: put_session(conn, :registration_id, registration_id)
 
   defp build_and_response_mfa_token(user, auth) do
-    with {:ok, token} <- MFA.Token.create_token(user, auth) do
+    with {:ok, token} <- MFA.Token.create(user, auth) do
       MFAView.render("mfa_response.json", %{token: token, user: user})
     end
   end
diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex
index 4d00fcb1c..de37998f2 100644
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@@ -87,9 +87,10 @@ defmodule Pleroma.Web.OAuth.Token do
   def create(%App{} = app, %User{} = user, attrs \\ %{}) do
     with {:ok, token} <- do_create(app, user, attrs) do
       if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do
-        Pleroma.Workers.PurgeExpiredOAuthToken.enqueue(%{
+        Pleroma.Workers.PurgeExpiredToken.enqueue(%{
           token_id: token.id,
-          valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC")
+          valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"),
+          mod: __MODULE__
         })
       end
 
diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex
deleted file mode 100644
index 2f51bdb75..000000000
--- a/lib/pleroma/web/oauth/token/clean_worker.ex
+++ /dev/null
@@ -1,36 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OAuth.Token.CleanWorker do
-  @moduledoc """
-  The module represents functions to clean an expired OAuth and MFA tokens.
-  """
-  use GenServer
-
-  @ten_seconds 10_000
-  @one_day 86_400_000
-
-  alias Pleroma.MFA
-  alias Pleroma.Workers.BackgroundWorker
-
-  def start_link(_), do: GenServer.start_link(__MODULE__, %{})
-
-  def init(_) do
-    Process.send_after(self(), :perform, @ten_seconds)
-    {:ok, nil}
-  end
-
-  @doc false
-  def handle_info(:perform, state) do
-    BackgroundWorker.enqueue("clean_expired_tokens", %{})
-    interval = Pleroma.Config.get([:oauth2, :clean_expired_tokens_interval], @one_day)
-
-    Process.send_after(self(), :perform, interval)
-    {:noreply, state}
-  end
-
-  def perform(:clean) do
-    MFA.Token.delete_expired_tokens()
-  end
-end
diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex
index 521dc9322..072d889e2 100644
--- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex
+++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex
@@ -135,7 +135,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do
   end
 
   defp handle_follow_error(conn, {:mfa_required, followee, user, _} = _) do
-    {:ok, %{token: token}} = MFA.Token.create_token(user)
+    {:ok, %{token: token}} = MFA.Token.create(user)
     render(conn, "follow_mfa.html", %{followee: followee, mfa_token: token, error: false})
   end
 
diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex
index 6068e43bf..a81e0cd28 100644
--- a/lib/pleroma/workers/purge_expired_token.ex
+++ b/lib/pleroma/workers/purge_expired_token.ex
@@ -2,14 +2,14 @@
 # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
 # SPDX-License-Identifier: AGPL-3.0-only
 
-defmodule Pleroma.Workers.PurgeExpiredOAuthToken do
+defmodule Pleroma.Workers.PurgeExpiredToken do
   @moduledoc """
   Worker which purges expired OAuth tokens
   """
 
-  use Oban.Worker, queue: :oauth_token_expiration, max_attempts: 1
+  use Oban.Worker, queue: :token_expiration, max_attempts: 1
 
-  @spec enqueue(%{token_id: integer(), valid_until: DateTime.t()}) ::
+  @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) ::
           {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()}
   def enqueue(args) do
     {scheduled_at, args} = Map.pop(args, :valid_until)
@@ -20,8 +20,9 @@ defmodule Pleroma.Workers.PurgeExpiredOAuthToken do
   end
 
   @impl true
-  def perform(%Oban.Job{args: %{"token_id" => id}}) do
-    Pleroma.Web.OAuth.Token
+  def perform(%Oban.Job{args: %{"token_id" => id, "mod" => module}}) do
+    module
+    |> String.to_existing_atom()
     |> Pleroma.Repo.get(id)
     |> Pleroma.Repo.delete()
   end
-- 
cgit v1.2.3


From 8af1fd32234df7d0cdb74d78bcca9f68587b70f2 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Mon, 7 Sep 2020 20:06:28 +0300
Subject: oban warning

---
 lib/pleroma/config/oban.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex
index c2d56ebab..81758c93d 100644
--- a/lib/pleroma/config/oban.ex
+++ b/lib/pleroma/config/oban.ex
@@ -5,7 +5,7 @@ defmodule Pleroma.Config.Oban do
     oban_config = Pleroma.Config.get(Oban)
 
     crontab =
-      [Pleroma.Workers.Cron.StatsWorker]
+      [Pleroma.Workers.Cron.StatsWorker, Pleroma.Workers.Cron.ClearOauthTokenWorker]
       |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
         with acc when is_list(acc) <- acc,
              setting when is_tuple(setting) <-
-- 
cgit v1.2.3


From e8bfb50fa3c16f98845e326b153c8a89505e8a55 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Thu, 10 Sep 2020 20:09:44 +0300
Subject: pass options without adapter key

---
 lib/pleroma/reverse_proxy/client/tesla.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex
index d5a339681..4b118eec2 100644
--- a/lib/pleroma/reverse_proxy/client/tesla.ex
+++ b/lib/pleroma/reverse_proxy/client/tesla.ex
@@ -28,7 +28,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
              url,
              body,
              headers,
-             Keyword.put(opts, :adapter, opts)
+             opts
            ) do
       if is_map(response.body) and method != :head do
         {:ok, response.status, response.headers, response.body}
-- 
cgit v1.2.3


From cb06e98da27994ac8034f3ba387b6eeaf8a2c48f Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 10 Sep 2020 13:47:53 +0300
Subject: websocket handler: Do not log client ping frames as errors

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 94e4595d8..e6010bb4a 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -64,7 +64,9 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     {:ok, %{state | timer: timer()}}
   end
 
-  # We never receive messages.
+  # We only receive pings for now
+  def websocket_handle(:ping, state), do: {:ok, state}
+
   def websocket_handle(frame, state) do
     Logger.error("#{__MODULE__} received frame: #{inspect(frame)}")
     {:ok, state}
-- 
cgit v1.2.3


From e16e8f98169f822416c18778abfa8495a486c8f2 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 10 Sep 2020 13:48:24 +0300
Subject: Websocket handler: do not raise if handler is terminated before
 switching protocols

Closes #2131
---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index e6010bb4a..5090d9622 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -100,6 +100,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     {:reply, :ping, %{state | timer: nil, count: 0}, :hibernate}
   end
 
+  # State can be `[]` only in case we terminate before switching to websocket,
+  # we already log errors for these cases in `init/1`, so just do nothing here
+  def terminate(_reason, _req, []), do: :ok
+
   def terminate(reason, _req, state) do
     Logger.debug(
       "#{__MODULE__} terminating websocket connection for user #{
-- 
cgit v1.2.3


From 01fa68fe4542286519e3520793c6b59103b050ff Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 10 Sep 2020 21:26:52 +0300
Subject: Websocket handler: fix never matching code on failed auth

`:cowboy_req.reply` does not return tuples since 2.0, see
https://ninenines.eu/docs/en/cowboy/2.4/manual/cowboy_req.reply/
---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 5090d9622..cf923ded8 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -37,12 +37,12 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     else
       {:error, :bad_topic} ->
         Logger.debug("#{__MODULE__} bad topic #{inspect(req)}")
-        {:ok, req} = :cowboy_req.reply(404, req)
+        req = :cowboy_req.reply(404, req)
         {:ok, req, state}
 
       {:error, :unauthorized} ->
         Logger.debug("#{__MODULE__} authentication error: #{inspect(req)}")
-        {:ok, req} = :cowboy_req.reply(401, req)
+        req = :cowboy_req.reply(401, req)
         {:ok, req, state}
     end
   end
-- 
cgit v1.2.3


From dc4e06e1991379f9f1b64774c5bdaacec96639b7 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 10 Sep 2020 21:28:07 +0300
Subject: [#2497] Removed support for thumbnail_max_* params for media preview
 proxy (per
 https://git.pleroma.social/pleroma/pleroma/-/merge_requests/2497#note_70771)

---
 .../web/media_proxy/media_proxy_controller.ex      | 38 ++++++++--------------
 1 file changed, 14 insertions(+), 24 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index acb581459..5621f72dc 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -9,6 +9,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   alias Pleroma.Helpers.MediaHelper
   alias Pleroma.ReverseProxy
   alias Pleroma.Web.MediaProxy
+  alias Plug.Conn
 
   def remote(conn, %{"sig" => sig64, "url" => url64}) do
     with {_, true} <- {:enabled, MediaProxy.enabled?()},
@@ -18,29 +19,29 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
       ReverseProxy.call(conn, url, media_proxy_opts())
     else
       {:enabled, false} ->
-        send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
+        send_resp(conn, 404, Conn.Status.reason_phrase(404))
 
       {:in_banned_urls, true} ->
-        send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
+        send_resp(conn, 404, Conn.Status.reason_phrase(404))
 
       {:error, :invalid_signature} ->
-        send_resp(conn, 403, Plug.Conn.Status.reason_phrase(403))
+        send_resp(conn, 403, Conn.Status.reason_phrase(403))
 
       {:wrong_filename, filename} ->
         redirect(conn, external: MediaProxy.build_url(sig64, url64, filename))
     end
   end
 
-  def preview(conn, %{"sig" => sig64, "url" => url64}) do
+  def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do
     with {_, true} <- {:enabled, MediaProxy.preview_enabled?()},
          {:ok, url} <- MediaProxy.decode_url(sig64, url64) do
       handle_preview(conn, url)
     else
       {:enabled, false} ->
-        send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404))
+        send_resp(conn, 404, Conn.Status.reason_phrase(404))
 
       {:error, :invalid_signature} ->
-        send_resp(conn, 403, Plug.Conn.Status.reason_phrase(403))
+        send_resp(conn, 403, Conn.Status.reason_phrase(403))
 
       {:wrong_filename, filename} ->
         redirect(conn, external: MediaProxy.build_preview_url(sig64, url64, filename))
@@ -94,10 +95,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
   end
 
-  defp handle_png_preview(%{params: params} = conn, media_proxy_url) do
+  defp handle_png_preview(conn, media_proxy_url) do
     quality = Config.get!([:media_preview_proxy, :image_quality])
 
-    with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
+    with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(),
          {:ok, thumbnail_binary} <-
            MediaHelper.image_resize(
              media_proxy_url,
@@ -117,10 +118,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp handle_jpeg_preview(%{params: params} = conn, media_proxy_url) do
+  defp handle_jpeg_preview(conn, media_proxy_url) do
     quality = Config.get!([:media_preview_proxy, :image_quality])
 
-    with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(params),
+    with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(),
          {:ok, thumbnail_binary} <-
            MediaHelper.image_resize(
              media_proxy_url,
@@ -157,22 +158,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     |> put_resp_header("cache-control", ReverseProxy.default_cache_control_header())
   end
 
-  defp thumbnail_max_dimensions(params) do
+  defp thumbnail_max_dimensions() do
     config = Config.get([:media_preview_proxy], [])
 
-    thumbnail_max_width =
-      if w = params["thumbnail_max_width"] do
-        String.to_integer(w)
-      else
-        Keyword.fetch!(config, :thumbnail_max_width)
-      end
-
-    thumbnail_max_height =
-      if h = params["thumbnail_max_height"] do
-        String.to_integer(h)
-      else
-        Keyword.fetch!(config, :thumbnail_max_height)
-      end
+    thumbnail_max_width = Keyword.fetch!(config, :thumbnail_max_width)
+    thumbnail_max_height = Keyword.fetch!(config, :thumbnail_max_height)
 
     {thumbnail_max_width, thumbnail_max_height}
   end
-- 
cgit v1.2.3


From 9bf1065a06837b4c753549d89afe23a636a20972 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Sat, 22 Aug 2020 20:46:01 +0300
Subject: schedule activity expiration in Oban

---
 lib/mix/tasks/pleroma/database.ex                  | 18 ++++--
 lib/pleroma/activity.ex                            |  3 -
 lib/pleroma/activity_expiration.ex                 | 74 ----------------------
 lib/pleroma/web/activity_pub/activity_pub.ex       |  7 +-
 .../activity_pub/mrf/activity_expiration_policy.ex |  4 +-
 lib/pleroma/web/activity_pub/side_effects.ex       |  6 +-
 lib/pleroma/web/common_api/activity_draft.ex       |  2 +-
 lib/pleroma/web/common_api/common_api.ex           |  5 +-
 lib/pleroma/web/mastodon_api/views/status_view.ex  |  5 +-
 .../cron/purge_expired_activities_worker.ex        | 48 --------------
 lib/pleroma/workers/purge_expired_activity.ex      | 72 +++++++++++++++++++++
 11 files changed, 100 insertions(+), 144 deletions(-)
 delete mode 100644 lib/pleroma/activity_expiration.ex
 delete mode 100644 lib/pleroma/workers/cron/purge_expired_activities_worker.ex
 create mode 100644 lib/pleroma/workers/purge_expired_activity.ex

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 7d8f00b08..aab4b5e9a 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -133,8 +133,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
 
     Pleroma.Activity
-    |> join(:left, [a], u in assoc(a, :expiration))
-    |> join(:inner, [a, _u], o in Object,
+    |> join(:inner, [a], o in Object,
       on:
         fragment(
           "(?->>'id') = COALESCE((?)->'object'->> 'id', (?)->>'object')",
@@ -144,14 +143,21 @@ defmodule Mix.Tasks.Pleroma.Database do
         )
     )
     |> where(local: true)
-    |> where([a, u], is_nil(u))
     |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
-    |> where([_a, _u, o], fragment("?->>'type' = 'Note'", o.data))
+    |> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
     |> Pleroma.RepoStreamer.chunk_stream(100)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity ->
-        expires_at = Timex.shift(activity.inserted_at, days: days)
-        Pleroma.ActivityExpiration.create(activity, expires_at, false)
+        expires_at =
+          activity.inserted_at
+          |> DateTime.from_naive!("Etc/UTC")
+          |> Timex.shift(days: days)
+
+        Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
+          activity_id: activity.id,
+          expires_at: expires_at,
+          validate: false
+        })
       end)
     end)
     |> Stream.run()
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 97feebeaa..03cd3b8c0 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -7,7 +7,6 @@ defmodule Pleroma.Activity do
 
   alias Pleroma.Activity
   alias Pleroma.Activity.Queries
-  alias Pleroma.ActivityExpiration
   alias Pleroma.Bookmark
   alias Pleroma.Notification
   alias Pleroma.Object
@@ -60,8 +59,6 @@ defmodule Pleroma.Activity do
     # typical case.
     has_one(:object, Object, on_delete: :nothing, foreign_key: :id)
 
-    has_one(:expiration, ActivityExpiration, on_delete: :delete_all)
-
     timestamps()
   end
 
diff --git a/lib/pleroma/activity_expiration.ex b/lib/pleroma/activity_expiration.ex
deleted file mode 100644
index 955f0578e..000000000
--- a/lib/pleroma/activity_expiration.ex
+++ /dev/null
@@ -1,74 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.ActivityExpiration do
-  use Ecto.Schema
-
-  alias Pleroma.Activity
-  alias Pleroma.ActivityExpiration
-  alias Pleroma.Repo
-
-  import Ecto.Changeset
-  import Ecto.Query
-
-  @type t :: %__MODULE__{}
-  @min_activity_lifetime :timer.hours(1)
-
-  schema "activity_expirations" do
-    belongs_to(:activity, Activity, type: FlakeId.Ecto.CompatType)
-    field(:scheduled_at, :naive_datetime)
-  end
-
-  def changeset(%ActivityExpiration{} = expiration, attrs, validate_scheduled_at) do
-    expiration
-    |> cast(attrs, [:scheduled_at])
-    |> validate_required([:scheduled_at])
-    |> validate_scheduled_at(validate_scheduled_at)
-  end
-
-  def get_by_activity_id(activity_id) do
-    ActivityExpiration
-    |> where([exp], exp.activity_id == ^activity_id)
-    |> Repo.one()
-  end
-
-  def create(%Activity{} = activity, scheduled_at, validate_scheduled_at \\ true) do
-    %ActivityExpiration{activity_id: activity.id}
-    |> changeset(%{scheduled_at: scheduled_at}, validate_scheduled_at)
-    |> Repo.insert()
-  end
-
-  def due_expirations(offset \\ 0) do
-    naive_datetime =
-      NaiveDateTime.utc_now()
-      |> NaiveDateTime.add(offset, :millisecond)
-
-    ActivityExpiration
-    |> where([exp], exp.scheduled_at < ^naive_datetime)
-    |> limit(50)
-    |> preload(:activity)
-    |> Repo.all()
-    |> Enum.reject(fn %{activity: activity} ->
-      Activity.pinned_by_actor?(activity)
-    end)
-  end
-
-  def validate_scheduled_at(changeset, false), do: changeset
-
-  def validate_scheduled_at(changeset, true) do
-    validate_change(changeset, :scheduled_at, fn _, scheduled_at ->
-      if not expires_late_enough?(scheduled_at) do
-        [scheduled_at: "an ephemeral activity must live for at least one hour"]
-      else
-        []
-      end
-    end)
-  end
-
-  def expires_late_enough?(scheduled_at) do
-    now = NaiveDateTime.utc_now()
-    diff = NaiveDateTime.diff(scheduled_at, now, :millisecond)
-    diff > @min_activity_lifetime
-  end
-end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 333621413..c33848277 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity
   alias Pleroma.Activity.Ir.Topics
-  alias Pleroma.ActivityExpiration
   alias Pleroma.Config
   alias Pleroma.Constants
   alias Pleroma.Conversation
@@ -165,7 +164,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp maybe_create_activity_expiration({:ok, %{data: %{"expires_at" => expires_at}} = activity}) do
-    with {:ok, _} <- ActivityExpiration.create(activity, expires_at) do
+    with {:ok, _job} <-
+           Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
+             activity_id: activity.id,
+             expires_at: expires_at
+           }) do
       {:ok, activity}
     end
   end
diff --git a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
index 7b4c78e0f..bee47b4ed 100644
--- a/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/activity_expiration_policy.ex
@@ -31,10 +31,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.ActivityExpirationPolicy do
 
   defp maybe_add_expiration(activity) do
     days = Pleroma.Config.get([:mrf_activity_expiration, :days], 365)
-    expires_at = NaiveDateTime.utc_now() |> Timex.shift(days: days)
+    expires_at = DateTime.utc_now() |> Timex.shift(days: days)
 
     with %{"expires_at" => existing_expires_at} <- activity,
-         :lt <- NaiveDateTime.compare(existing_expires_at, expires_at) do
+         :lt <- DateTime.compare(existing_expires_at, expires_at) do
       activity
     else
       _ -> Map.put(activity, "expires_at", expires_at)
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index a5e2323bd..b30ca1bd7 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   """
   alias Pleroma.Activity
   alias Pleroma.Activity.Ir.Topics
-  alias Pleroma.ActivityExpiration
   alias Pleroma.Chat
   alias Pleroma.Chat.MessageReference
   alias Pleroma.FollowingRelationship
@@ -189,7 +188,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
       end
 
       if expires_at = activity.data["expires_at"] do
-        ActivityExpiration.create(activity, expires_at)
+        Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
+          activity_id: activity.id,
+          expires_at: expires_at
+        })
       end
 
       BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex
index f849b2e01..548f76609 100644
--- a/lib/pleroma/web/common_api/activity_draft.ex
+++ b/lib/pleroma/web/common_api/activity_draft.ex
@@ -202,7 +202,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do
 
     additional =
       case draft.expires_at do
-        %NaiveDateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at)
+        %DateTime{} = expires_at -> Map.put(additional, "expires_at", expires_at)
         _ -> additional
       end
 
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 4ab533658..500c3883e 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -4,7 +4,6 @@
 
 defmodule Pleroma.Web.CommonAPI do
   alias Pleroma.Activity
-  alias Pleroma.ActivityExpiration
   alias Pleroma.Conversation.Participation
   alias Pleroma.Formatter
   alias Pleroma.Object
@@ -381,9 +380,9 @@ defmodule Pleroma.Web.CommonAPI do
   def check_expiry_date({:ok, nil} = res), do: res
 
   def check_expiry_date({:ok, in_seconds}) do
-    expiry = NaiveDateTime.utc_now() |> NaiveDateTime.add(in_seconds)
+    expiry = DateTime.add(DateTime.utc_now(), in_seconds)
 
-    if ActivityExpiration.expires_late_enough?(expiry) do
+    if Pleroma.Workers.PurgeExpiredActivity.expires_late_enough?(expiry) do
       {:ok, expiry}
     else
       {:error, "Expiry date is too soon"}
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 3fe1967be..ca42917fc 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -8,7 +8,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   require Pleroma.Constants
 
   alias Pleroma.Activity
-  alias Pleroma.ActivityExpiration
   alias Pleroma.HTML
   alias Pleroma.Object
   alias Pleroma.Repo
@@ -245,8 +244,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     expires_at =
       with true <- client_posted_this_activity,
-           %ActivityExpiration{scheduled_at: scheduled_at} <-
-             ActivityExpiration.get_by_activity_id(activity.id) do
+           %Oban.Job{scheduled_at: scheduled_at} <-
+             Pleroma.Workers.PurgeExpiredActivity.get_expiration(activity.id) do
         scheduled_at
       else
         _ -> nil
diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex
deleted file mode 100644
index 6549207fc..000000000
--- a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex
+++ /dev/null
@@ -1,48 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do
-  @moduledoc """
-  The worker to purge expired activities.
-  """
-
-  use Oban.Worker, queue: "activity_expiration"
-
-  alias Pleroma.Activity
-  alias Pleroma.ActivityExpiration
-  alias Pleroma.Config
-  alias Pleroma.User
-  alias Pleroma.Web.CommonAPI
-
-  require Logger
-
-  @interval :timer.minutes(1)
-
-  @impl Oban.Worker
-  def perform(_job) do
-    if Config.get([ActivityExpiration, :enabled]) do
-      Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1)
-    end
-  after
-    :ok
-  end
-
-  def delete_activity(%ActivityExpiration{activity_id: activity_id}) do
-    with {:activity, %Activity{} = activity} <-
-           {:activity, Activity.get_by_id_with_object(activity_id)},
-         {:user, %User{} = user} <- {:user, User.get_by_ap_id(activity.object.data["actor"])} do
-      CommonAPI.delete(activity.id, user)
-    else
-      {:activity, _} ->
-        Logger.error(
-          "#{__MODULE__} Couldn't delete expired activity: not found activity ##{activity_id}"
-        )
-
-      {:user, _} ->
-        Logger.error(
-          "#{__MODULE__} Couldn't delete expired activity: not found actor of ##{activity_id}"
-        )
-    end
-  end
-end
diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
new file mode 100644
index 000000000..016b000c1
--- /dev/null
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -0,0 +1,72 @@
+defmodule Pleroma.Workers.PurgeExpiredActivity do
+  @moduledoc """
+  Worker which purges expired activity.
+  """
+
+  use Oban.Worker, queue: :activity_expiration, max_attempts: 1
+
+  import Ecto.Query
+
+  def enqueue(args) do
+    with true <- enabled?(),
+         args when is_map(args) <- validate_expires_at(args) do
+      {scheduled_at, args} = Map.pop(args, :expires_at)
+
+      args
+      |> __MODULE__.new(scheduled_at: scheduled_at)
+      |> Oban.insert()
+    end
+  end
+
+  @impl true
+  def perform(%Oban.Job{args: %{"activity_id" => id}}) do
+    with %Pleroma.Activity{} = activity <- find_activity(id),
+         %Pleroma.User{} = user <- find_user(activity.object.data["actor"]) do
+      Pleroma.Web.CommonAPI.delete(activity.id, user)
+    end
+  end
+
+  defp enabled? do
+    with false <- Pleroma.Config.get([__MODULE__, :enabled], false) do
+      {:error, :expired_activities_disabled}
+    end
+  end
+
+  defp validate_expires_at(%{validate: false} = args), do: Map.delete(args, :validate)
+
+  defp validate_expires_at(args) do
+    if expires_late_enough?(args[:expires_at]) do
+      args
+    else
+      {:error, :expiration_too_close}
+    end
+  end
+
+  defp find_activity(id) do
+    with nil <- Pleroma.Activity.get_by_id_with_object(id) do
+      {:error, :activity_not_found}
+    end
+  end
+
+  defp find_user(ap_id) do
+    with nil <- Pleroma.User.get_by_ap_id(ap_id) do
+      {:error, :user_not_found}
+    end
+  end
+
+  def get_expiration(id) do
+    from(j in Oban.Job,
+      where: j.state == "scheduled",
+      where: j.queue == "activity_expiration",
+      where: fragment("?->>'activity_id' = ?", j.args, ^id)
+    )
+    |> Pleroma.Repo.one()
+  end
+
+  @spec expires_late_enough?(DateTime.t()) :: boolean()
+  def expires_late_enough?(scheduled_at) do
+    now = DateTime.utc_now()
+    diff = DateTime.diff(scheduled_at, now, :millisecond)
+    diff > :timer.hours(1)
+  end
+end
-- 
cgit v1.2.3


From de4c935071a47c78d873484b202e09dce5399570 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Mon, 24 Aug 2020 13:43:02 +0300
Subject: don't expire pinned posts

---
 lib/pleroma/activity.ex                       |  9 +++++++--
 lib/pleroma/workers/purge_expired_activity.ex | 18 +++++++++++++++++-
 2 files changed, 24 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 03cd3b8c0..84aba9572 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -301,14 +301,14 @@ defmodule Pleroma.Activity do
     |> Repo.all()
   end
 
-  def follow_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do
+  def follow_requests_for_actor(%User{ap_id: ap_id}) do
     ap_id
     |> Queries.by_object_id()
     |> Queries.by_type("Follow")
     |> where([a], fragment("? ->> 'state' = 'pending'", a.data))
   end
 
-  def following_requests_for_actor(%Pleroma.User{ap_id: ap_id}) do
+  def following_requests_for_actor(%User{ap_id: ap_id}) do
     Queries.by_type("Follow")
     |> where([a], fragment("?->>'state' = 'pending'", a.data))
     |> where([a], a.actor == ^ap_id)
@@ -343,4 +343,9 @@ defmodule Pleroma.Activity do
     actor = user_actor(activity)
     activity.id in actor.pinned_activities
   end
+
+  @spec pinned_by_actor?(Activity.t(), User.t()) :: boolean()
+  def pinned_by_actor?(%Activity{id: id}, %User{} = user) do
+    id in user.pinned_activities
+  end
 end
diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index 016b000c1..ba0053008 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -21,8 +21,18 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   @impl true
   def perform(%Oban.Job{args: %{"activity_id" => id}}) do
     with %Pleroma.Activity{} = activity <- find_activity(id),
-         %Pleroma.User{} = user <- find_user(activity.object.data["actor"]) do
+         %Pleroma.User{} = user <- find_user(activity.object.data["actor"]),
+         false <- pinned_by_actor?(activity, user) do
       Pleroma.Web.CommonAPI.delete(activity.id, user)
+    else
+      :pinned_by_actor ->
+        # if activity is pinned, schedule deletion on next day
+        enqueue(%{activity_id: id, expires_at: DateTime.add(DateTime.utc_now(), 24 * 3600)})
+
+        :ok
+
+      error ->
+        error
     end
   end
 
@@ -54,6 +64,12 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
     end
   end
 
+  defp pinned_by_actor?(activity, user) do
+    with true <- Pleroma.Activity.pinned_by_actor?(activity, user) do
+      :pinned_by_actor
+    end
+  end
+
   def get_expiration(id) do
     from(j in Oban.Job,
       where: j.state == "scheduled",
-- 
cgit v1.2.3


From 629a8de9cb2ba2cc2d09679862a24031f34abc2f Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 25 Aug 2020 09:10:45 +0300
Subject: deprecation warning

changed namespace for activity expiration configuration
---
 lib/pleroma/config/deprecation_warnings.ex    | 19 ++++++++++++++++++-
 lib/pleroma/workers/purge_expired_activity.ex |  8 +++++---
 2 files changed, 23 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex
index 2bfe4ddba..412d55a77 100644
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@@ -8,7 +8,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
   require Logger
   alias Pleroma.Config
 
-  @type config_namespace() :: [atom()]
+  @type config_namespace() :: atom() | [atom()]
   @type config_map() :: {config_namespace(), config_namespace(), String.t()}
 
   @mrf_config_map [
@@ -57,6 +57,7 @@ defmodule Pleroma.Config.DeprecationWarnings do
     check_media_proxy_whitelist_config()
     check_welcome_message_config()
     check_gun_pool_options()
+    check_activity_expiration_config()
   end
 
   def check_welcome_message_config do
@@ -158,4 +159,20 @@ defmodule Pleroma.Config.DeprecationWarnings do
       Config.put(:pools, updated_config)
     end
   end
+
+  @spec check_activity_expiration_config() :: :ok | nil
+  def check_activity_expiration_config do
+    warning_preface = """
+    !!!DEPRECATION WARNING!!!
+      Your config is using old namespace for activity expiration configuration. Setting should work for now, but you are advised to change to new namespace to prevent possible issues later:
+    """
+
+    move_namespace_and_warn(
+      [
+        {Pleroma.ActivityExpiration, Pleroma.Workers.PurgeExpiredActivity,
+         "\n* `config :pleroma, Pleroma.ActivityExpiration` is now `config :pleroma, Pleroma.Workers.PurgeExpiredActivity`"}
+      ],
+      warning_preface
+    )
+  end
 end
diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index ba0053008..44a8ad0b9 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
 
   import Ecto.Query
 
+  alias Pleroma.Activity
+
   def enqueue(args) do
     with true <- enabled?(),
          args when is_map(args) <- validate_expires_at(args) do
@@ -20,7 +22,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
 
   @impl true
   def perform(%Oban.Job{args: %{"activity_id" => id}}) do
-    with %Pleroma.Activity{} = activity <- find_activity(id),
+    with %Activity{} = activity <- find_activity(id),
          %Pleroma.User{} = user <- find_user(activity.object.data["actor"]),
          false <- pinned_by_actor?(activity, user) do
       Pleroma.Web.CommonAPI.delete(activity.id, user)
@@ -53,7 +55,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   end
 
   defp find_activity(id) do
-    with nil <- Pleroma.Activity.get_by_id_with_object(id) do
+    with nil <- Activity.get_by_id_with_object(id) do
       {:error, :activity_not_found}
     end
   end
@@ -65,7 +67,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   end
 
   defp pinned_by_actor?(activity, user) do
-    with true <- Pleroma.Activity.pinned_by_actor?(activity, user) do
+    with true <- Activity.pinned_by_actor?(activity, user) do
       :pinned_by_actor
     end
   end
-- 
cgit v1.2.3


From 4981b5a1a3c097ca849552c3c6f650efd22c7451 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 25 Aug 2020 12:45:06 +0300
Subject: copyright header

---
 lib/pleroma/workers/purge_expired_activity.ex | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index 44a8ad0b9..42e2ae79c 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -1,3 +1,7 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
 defmodule Pleroma.Workers.PurgeExpiredActivity do
   @moduledoc """
   Worker which purges expired activity.
-- 
cgit v1.2.3


From 93e1c8df9dca697e7bdb822a8a5b3848b7870f53 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Thu, 3 Sep 2020 13:30:39 +0300
Subject: reject activity creation

if passed expires_at option and expiring activities are not configured
---
 lib/pleroma/web/activity_pub/activity_pub.ex  | 44 ++++++++++++++++++---------
 lib/pleroma/workers/purge_expired_activity.ex |  4 +++
 2 files changed, 33 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index c33848277..ee6dcf58a 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -110,23 +110,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def insert(map, local \\ true, fake \\ false, bypass_actor_check \\ false) when is_map(map) do
     with nil <- Activity.normalize(map),
          map <- lazy_put_activity_defaults(map, fake),
-         true <- bypass_actor_check || check_actor_is_active(map["actor"]),
-         {_, true} <- {:remote_limit_error, check_remote_limit(map)},
+         {_, true} <- {:actor_check, bypass_actor_check || check_actor_is_active(map["actor"])},
+         {_, true} <- {:remote_limit_pass, check_remote_limit(map)},
          {:ok, map} <- MRF.filter(map),
          {recipients, _, _} = get_recipients(map),
          {:fake, false, map, recipients} <- {:fake, fake, map, recipients},
          {:containment, :ok} <- {:containment, Containment.contain_child(map)},
-         {:ok, map, object} <- insert_full_object(map) do
-      {:ok, activity} =
-        %Activity{
-          data: map,
-          local: local,
-          actor: map["actor"],
-          recipients: recipients
-        }
-        |> Repo.insert()
-        |> maybe_create_activity_expiration()
-
+         {:ok, map, object} <- insert_full_object(map),
+         {:ok, activity} <- insert_activity_with_expiration(map, local, recipients) do
       # Splice in the child object if we have one.
       activity = Maps.put_if_present(activity, :object, object)
 
@@ -137,6 +128,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       %Activity{} = activity ->
         {:ok, activity}
 
+      {:actor_check, _} ->
+        {:error, false}
+
+      {:containment, _} = error ->
+        error
+
+      {:error, _} = error ->
+        error
+
       {:fake, true, map, recipients} ->
         activity = %Activity{
           data: map,
@@ -149,11 +149,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)
         {:ok, activity}
 
-      error ->
-        {:error, error}
+      {:remote_limit_pass, _} ->
+        {:error, :remote_limit}
+
+      {:reject, reason} ->
+        {:error, reason}
     end
   end
 
+  defp insert_activity_with_expiration(data, local, recipients) do
+    %Activity{
+      data: data,
+      local: local,
+      actor: data["actor"],
+      recipients: recipients
+    }
+    |> Repo.insert()
+    |> maybe_create_activity_expiration()
+  end
+
   def notify_and_stream(activity) do
     Notification.create_notifications(activity)
 
diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index 42e2ae79c..c70587b47 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -13,6 +13,10 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
 
   alias Pleroma.Activity
 
+  @spec enqueue(map()) ::
+          {:ok, Oban.Job.t()}
+          | {:error, :expired_activities_disabled}
+          | {:error, :expiration_too_close}
   def enqueue(args) do
     with true <- enabled?(),
          args when is_map(args) <- validate_expires_at(args) do
-- 
cgit v1.2.3


From 357d971a10c28780795af4d19b37b0ac80d6ad09 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Thu, 3 Sep 2020 17:56:20 +0300
Subject: expiration for new pipeline

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 18 ++++++++++++------
 lib/pleroma/web/activity_pub/side_effects.ex |  7 -------
 2 files changed, 12 insertions(+), 13 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index ee6dcf58a..66a9f78a3 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -101,7 +101,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              local: local,
              recipients: recipients,
              actor: object["actor"]
-           }) do
+           }),
+         # TODO: add tests for expired activities, when Note type will be supported in new pipeline
+         {:ok, _} <- maybe_create_activity_expiration(activity) do
       {:ok, activity, meta}
     end
   end
@@ -158,14 +160,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp insert_activity_with_expiration(data, local, recipients) do
-    %Activity{
+    struct = %Activity{
       data: data,
       local: local,
       actor: data["actor"],
       recipients: recipients
     }
-    |> Repo.insert()
-    |> maybe_create_activity_expiration()
+
+    with {:ok, activity} <- Repo.insert(struct) do
+      maybe_create_activity_expiration(activity)
+    end
   end
 
   def notify_and_stream(activity) do
@@ -177,7 +181,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     stream_out_participations(participations)
   end
 
-  defp maybe_create_activity_expiration({:ok, %{data: %{"expires_at" => expires_at}} = activity}) do
+  defp maybe_create_activity_expiration(
+         %{data: %{"expires_at" => %DateTime{} = expires_at}} = activity
+       ) do
     with {:ok, _job} <-
            Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
              activity_id: activity.id,
@@ -187,7 +193,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  defp maybe_create_activity_expiration(result), do: result
+  defp maybe_create_activity_expiration(activity), do: {:ok, activity}
 
   defp create_or_bump_conversation(activity, actor) do
     with {:ok, conversation} <- Conversation.create_or_bump_for(activity),
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index b30ca1bd7..46a8be767 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -187,13 +187,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
         Object.increase_replies_count(in_reply_to)
       end
 
-      if expires_at = activity.data["expires_at"] do
-        Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
-          activity_id: activity.id,
-          expires_at: expires_at
-        })
-      end
-
       BackgroundWorker.enqueue("fetch_data_for_activity", %{"activity_id" => activity.id})
 
       meta =
-- 
cgit v1.2.3


From b3485a6dbfb1a16dd5604294074ef5139fbf3ce9 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Thu, 3 Sep 2020 19:02:22 +0300
Subject: little clean up

---
 lib/pleroma/workers/purge_expired_activity.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index c70587b47..4be146194 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -23,7 +23,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
       {scheduled_at, args} = Map.pop(args, :expires_at)
 
       args
-      |> __MODULE__.new(scheduled_at: scheduled_at)
+      |> new(scheduled_at: scheduled_at)
       |> Oban.insert()
     end
   end
-- 
cgit v1.2.3


From eb5ff715f7917e174b9ae104a5d82779ff925301 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 4 Sep 2020 11:40:32 +0300
Subject: pin/unpin for activities with expires_at option

---
 lib/pleroma/activity.ex                       |  5 -----
 lib/pleroma/user.ex                           | 17 ++++++++++++++++-
 lib/pleroma/workers/purge_expired_activity.ex | 18 +-----------------
 3 files changed, 17 insertions(+), 23 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 84aba9572..17af04257 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -343,9 +343,4 @@ defmodule Pleroma.Activity do
     actor = user_actor(activity)
     activity.id in actor.pinned_activities
   end
-
-  @spec pinned_by_actor?(Activity.t(), User.t()) :: boolean()
-  def pinned_by_actor?(%Activity{id: id}, %User{} = user) do
-    id in user.pinned_activities
-  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index f323fc6ed..e73d19964 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -2315,6 +2315,11 @@ defmodule Pleroma.User do
       max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0)
       params = %{pinned_activities: user.pinned_activities ++ [id]}
 
+      # if pinned activity was scheduled for deletion, we remove job
+      if expiration = Pleroma.Workers.PurgeExpiredActivity.get_expiration(id) do
+        Oban.cancel_job(expiration.id)
+      end
+
       user
       |> cast(params, [:pinned_activities])
       |> validate_length(:pinned_activities,
@@ -2327,9 +2332,19 @@ defmodule Pleroma.User do
     |> update_and_set_cache()
   end
 
-  def remove_pinnned_activity(user, %Pleroma.Activity{id: id}) do
+  def remove_pinnned_activity(user, %Pleroma.Activity{id: id, data: data}) do
     params = %{pinned_activities: List.delete(user.pinned_activities, id)}
 
+    # if pinned activity was scheduled for deletion, we reschedule it for deletion
+    if data["expires_at"] do
+      {:ok, expires_at, _} = DateTime.from_iso8601(data["expires_at"])
+
+      Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
+        activity_id: id,
+        expires_at: expires_at
+      })
+    end
+
     user
     |> cast(params, [:pinned_activities])
     |> update_and_set_cache()
diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index 4be146194..f981eda8e 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -31,18 +31,8 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   @impl true
   def perform(%Oban.Job{args: %{"activity_id" => id}}) do
     with %Activity{} = activity <- find_activity(id),
-         %Pleroma.User{} = user <- find_user(activity.object.data["actor"]),
-         false <- pinned_by_actor?(activity, user) do
+         %Pleroma.User{} = user <- find_user(activity.object.data["actor"]) do
       Pleroma.Web.CommonAPI.delete(activity.id, user)
-    else
-      :pinned_by_actor ->
-        # if activity is pinned, schedule deletion on next day
-        enqueue(%{activity_id: id, expires_at: DateTime.add(DateTime.utc_now(), 24 * 3600)})
-
-        :ok
-
-      error ->
-        error
     end
   end
 
@@ -74,12 +64,6 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
     end
   end
 
-  defp pinned_by_actor?(activity, user) do
-    with true <- Activity.pinned_by_actor?(activity, user) do
-      :pinned_by_actor
-    end
-  end
-
   def get_expiration(id) do
     from(j in Oban.Job,
       where: j.state == "scheduled",
-- 
cgit v1.2.3


From f24828a3e848e6ce3bcdd254e8c6e451898cfdf7 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Mon, 7 Sep 2020 20:21:32 +0300
Subject: oban warning

---
 lib/pleroma/config/oban.ex | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex
index 81758c93d..9f601b1a3 100644
--- a/lib/pleroma/config/oban.ex
+++ b/lib/pleroma/config/oban.ex
@@ -5,7 +5,11 @@ defmodule Pleroma.Config.Oban do
     oban_config = Pleroma.Config.get(Oban)
 
     crontab =
-      [Pleroma.Workers.Cron.StatsWorker, Pleroma.Workers.Cron.ClearOauthTokenWorker]
+      [
+        Pleroma.Workers.Cron.StatsWorker,
+        Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker,
+        Pleroma.Workers.Cron.ClearOauthTokenWorker
+      ]
       |> Enum.reduce(oban_config[:crontab], fn removed_worker, acc ->
         with acc when is_list(acc) <- acc,
              setting when is_tuple(setting) <-
-- 
cgit v1.2.3


From 2c2094d4b2722cf511e3db8288c3754a48038f05 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Mon, 7 Sep 2020 20:57:38 +0300
Subject: configurable lifetime for ephemeral activities

---
 lib/pleroma/workers/purge_expired_activity.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index f981eda8e..ffcb89dc3 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -77,6 +77,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
   def expires_late_enough?(scheduled_at) do
     now = DateTime.utc_now()
     diff = DateTime.diff(scheduled_at, now, :millisecond)
-    diff > :timer.hours(1)
+    min_lifetime = Pleroma.Config.get([__MODULE__, :min_lifetime], 600)
+    diff > :timer.seconds(min_lifetime)
   end
 end
-- 
cgit v1.2.3


From 15aece72382fe1862a58728b9d02990147f91365 Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Tue, 8 Sep 2020 15:11:18 +0300
Subject: remove validate_expires_at from enqueue method

---
 lib/mix/tasks/pleroma/database.ex             |  3 +--
 lib/pleroma/workers/purge_expired_activity.ex | 13 +------------
 2 files changed, 2 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index aab4b5e9a..7f1108dcf 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -155,8 +155,7 @@ defmodule Mix.Tasks.Pleroma.Database do
 
         Pleroma.Workers.PurgeExpiredActivity.enqueue(%{
           activity_id: activity.id,
-          expires_at: expires_at,
-          validate: false
+          expires_at: expires_at
         })
       end)
     end)
diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex
index ffcb89dc3..c168890a2 100644
--- a/lib/pleroma/workers/purge_expired_activity.ex
+++ b/lib/pleroma/workers/purge_expired_activity.ex
@@ -18,8 +18,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
           | {:error, :expired_activities_disabled}
           | {:error, :expiration_too_close}
   def enqueue(args) do
-    with true <- enabled?(),
-         args when is_map(args) <- validate_expires_at(args) do
+    with true <- enabled?() do
       {scheduled_at, args} = Map.pop(args, :expires_at)
 
       args
@@ -42,16 +41,6 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do
     end
   end
 
-  defp validate_expires_at(%{validate: false} = args), do: Map.delete(args, :validate)
-
-  defp validate_expires_at(args) do
-    if expires_late_enough?(args[:expires_at]) do
-      args
-    else
-      {:error, :expiration_too_close}
-    end
-  end
-
   defp find_activity(id) do
     with nil <- Activity.get_by_id_with_object(id) do
       {:error, :activity_not_found}
-- 
cgit v1.2.3


From 4d18a50f3c4b6654339a6a8df71160e23b45cac0 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 10 Sep 2020 21:54:26 +0300
Subject: [#2497] Formatting fix.

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 5621f72dc..ff7fd2409 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -158,7 +158,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     |> put_resp_header("cache-control", ReverseProxy.default_cache_control_header())
   end
 
-  defp thumbnail_max_dimensions() do
+  defp thumbnail_max_dimensions do
     config = Config.get([:media_preview_proxy], [])
 
     thumbnail_max_width = Keyword.fetch!(config, :thumbnail_max_width)
-- 
cgit v1.2.3


From da876d09e89bcfec6f2d1eaddb396f68ce48e12a Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 10 Sep 2020 23:13:51 +0200
Subject: federator: normalize only actor, catch actor error

---
 lib/pleroma/web/federator/federator.ex | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index f5803578d..e4ab9ba32 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -66,14 +66,17 @@ defmodule Pleroma.Web.Federator do
   def perform(:incoming_ap_doc, params) do
     Logger.debug("Handling incoming AP activity")
 
-    params = Utils.normalize_params(params)
+    actor =
+      params
+      |> Map.get("actor")
+      |> Utils.get_ap_id()
 
     # NOTE: we use the actor ID to do the containment, this is fine because an
     # actor shouldn't be acting on objects outside their own AP server.
-    with {:ok, _user} <- ap_enabled_actor(params["actor"]),
+    with {_, {:ok, _user}} <- {:actor, ap_enabled_actor(actor)},
          nil <- Activity.normalize(params["id"]),
          {_, :ok} <-
-           {:correct_origin?, Containment.contain_origin_from_id(params["actor"], params)},
+           {:correct_origin?, Containment.contain_origin_from_id(actor, params)},
          {:ok, activity} <- Transmogrifier.handle_incoming(params) do
       {:ok, activity}
     else
@@ -85,10 +88,13 @@ defmodule Pleroma.Web.Federator do
         Logger.debug("Already had #{params["id"]}")
         {:error, :already_present}
 
+      {:actor, e} ->
+        Logger.debug("Unhandled actor #{actor}, #{inspect(e)}")
+        {:error, e}
+
       e ->
         # Just drop those for now
-        Logger.debug("Unhandled activity")
-        Logger.debug(Jason.encode!(params, pretty: true))
+        Logger.debug("Unhandled activity\n" <> Jason.encode!(params, pretty: true))
         {:error, e}
     end
   end
-- 
cgit v1.2.3


From b73e9ef68689a7094e80e2affa0af9b05e86effb Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Tue, 25 Aug 2020 09:19:53 +0200
Subject: transmogrifier: Call strip_internal_fields on pipeline ingestion

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index af4384213..ec3b24206 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -550,6 +550,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         _options
       )
       when objtype in ~w{Question Answer ChatMessage Audio Event} do
+    data = Map.put(data, "object", strip_internal_fields(data["object"]))
+
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
          {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do
       {:ok, activity}
-- 
cgit v1.2.3


From 846b59ccb09681bda0f54bed43f5b82883228e33 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 20 Aug 2020 02:00:04 +0200
Subject: Pipeline Ingestion: Video

---
 lib/pleroma/web/activity_pub/activity_pub.ex       |   2 +-
 lib/pleroma/web/activity_pub/object_validator.ex   |  16 +--
 .../object_validators/audio_validator.ex           | 107 ----------------
 .../object_validators/audio_video_validator.ex     | 134 +++++++++++++++++++++
 .../activity_pub/object_validators/common_fixes.ex |   9 ++
 .../object_validators/create_generic_validator.ex  |   8 +-
 lib/pleroma/web/activity_pub/side_effects.ex       |   2 +-
 lib/pleroma/web/activity_pub/transmogrifier.ex     |  37 +-----
 8 files changed, 162 insertions(+), 153 deletions(-)
 delete mode 100644 lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 66a9f78a3..bceec8bd1 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -84,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio Event]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index b77c06395..081f96389 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -12,11 +12,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Activity
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
+  alias Pleroma.Object.Containment
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator
@@ -149,10 +150,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
-  def validate(%{"type" => "Audio"} = object, meta) do
+  def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do
     with {:ok, object} <-
            object
-           |> AudioValidator.cast_and_validate()
+           |> AudioVideoValidator.cast_and_validate()
            |> Ecto.Changeset.apply_action(:insert) do
       object = stringify_keys(object)
       {:ok, object, meta}
@@ -198,7 +199,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
       )
-      when objtype in ~w[Question Answer Audio Event] do
+      when objtype in ~w[Question Answer Audio Video Event] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -232,8 +233,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     AnswerValidator.cast_and_apply(object)
   end
 
-  def cast_and_apply(%{"type" => "Audio"} = object) do
-    AudioValidator.cast_and_apply(object)
+  def cast_and_apply(%{"type" => type} = object) when type in ~w[Audio Video] do
+    AudioVideoValidator.cast_and_apply(object)
   end
 
   def cast_and_apply(%{"type" => "Event"} = object) do
@@ -262,7 +263,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   def stringify_keys(object), do: object
 
   def fetch_actor(object) do
-    with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do
+    with actor <- Containment.get_actor(object),
+         {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do
       User.get_or_fetch_by_ap_id(actor)
     end
   end
diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
deleted file mode 100644
index 1a97c504a..000000000
--- a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex
+++ /dev/null
@@ -1,107 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
-  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
-  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
-  alias Pleroma.Web.ActivityPub.Transmogrifier
-
-  import Ecto.Changeset
-
-  @primary_key false
-  @derive Jason.Encoder
-
-  embedded_schema do
-    field(:id, ObjectValidators.ObjectID, primary_key: true)
-    field(:to, ObjectValidators.Recipients, default: [])
-    field(:cc, ObjectValidators.Recipients, default: [])
-    field(:bto, ObjectValidators.Recipients, default: [])
-    field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
-    field(:tag, {:array, :map}, default: [])
-    field(:type, :string)
-    field(:content, :string)
-    field(:context, :string)
-
-    # TODO: Remove actor on objects
-    field(:actor, ObjectValidators.ObjectID)
-
-    field(:attributedTo, ObjectValidators.ObjectID)
-    field(:summary, :string)
-    field(:published, ObjectValidators.DateTime)
-    field(:emoji, ObjectValidators.Emoji, default: %{})
-    field(:sensitive, :boolean, default: false)
-    embeds_many(:attachment, AttachmentValidator)
-    field(:replies_count, :integer, default: 0)
-    field(:like_count, :integer, default: 0)
-    field(:announcement_count, :integer, default: 0)
-    field(:inReplyTo, :string)
-    field(:url, ObjectValidators.Uri)
-    # short identifier for PleromaFE to group statuses by context
-    field(:context_id, :integer)
-
-    field(:likes, {:array, :string}, default: [])
-    field(:announcements, {:array, :string}, default: [])
-  end
-
-  def cast_and_apply(data) do
-    data
-    |> cast_data
-    |> apply_action(:insert)
-  end
-
-  def cast_and_validate(data) do
-    data
-    |> cast_data()
-    |> validate_data()
-  end
-
-  def cast_data(data) do
-    %__MODULE__{}
-    |> changeset(data)
-  end
-
-  defp fix_url(%{"url" => url} = data) when is_list(url) do
-    attachment =
-      Enum.find(url, fn x -> is_map(x) and String.starts_with?(x["mimeType"], "audio/") end)
-
-    link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end)
-
-    data
-    |> Map.put("attachment", [attachment])
-    |> Map.put("url", link_element["href"])
-  end
-
-  defp fix_url(data), do: data
-
-  defp fix(data) do
-    data
-    |> CommonFixes.fix_defaults()
-    |> CommonFixes.fix_attribution()
-    |> Transmogrifier.fix_emoji()
-    |> fix_url()
-  end
-
-  def changeset(struct, data) do
-    data = fix(data)
-
-    struct
-    |> cast(data, __schema__(:fields) -- [:attachment])
-    |> cast_embed(:attachment)
-  end
-
-  def validate_data(data_cng) do
-    data_cng
-    |> validate_inclusion(:type, ["Audio"])
-    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
-    |> CommonValidations.validate_any_presence([:cc, :to])
-    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
-    |> CommonValidations.validate_actor_presence()
-    |> CommonValidations.validate_host_match()
-  end
-end
diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex
new file mode 100644
index 000000000..a6119e627
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex
@@ -0,0 +1,134 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EarmarkRenderer
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:bto, ObjectValidators.Recipients, default: [])
+    field(:bcc, ObjectValidators.Recipients, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
+    field(:content, :string)
+
+    field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:published, ObjectValidators.DateTime)
+    field(:emoji, ObjectValidators.Emoji, default: %{})
+    field(:sensitive, :boolean, default: false)
+    embeds_many(:attachment, AttachmentValidator)
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, ObjectValidators.ObjectID)
+    field(:url, ObjectValidators.Uri)
+
+    field(:likes, {:array, :string}, default: [])
+    field(:announcements, {:array, :string}, default: [])
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  defp fix_url(%{"url" => url} = data) when is_list(url) do
+    attachment =
+      Enum.find(url, fn x ->
+        mime_type = x["mimeType"] || x["mediaType"] || ""
+
+        is_map(x) and String.starts_with?(mime_type, ["video/", "audio/"])
+      end)
+
+    link_element =
+      Enum.find(url, fn x ->
+        mime_type = x["mimeType"] || x["mediaType"] || ""
+
+        is_map(x) and mime_type == "text/html"
+      end)
+
+    data
+    |> Map.put("attachment", [attachment])
+    |> Map.put("url", link_element["href"])
+  end
+
+  defp fix_url(data), do: data
+
+  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data)
+       when is_binary(content) do
+    content =
+      content
+      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
+      |> Pleroma.HTML.filter_tags()
+
+    Map.put(data, "content", content)
+  end
+
+  defp fix_content(data), do: data
+
+  defp fix(data) do
+    data
+    |> CommonFixes.fix_defaults()
+    |> CommonFixes.fix_attribution()
+    |> CommonFixes.fix_actor()
+    |> Transmogrifier.fix_emoji()
+    |> fix_url()
+    |> fix_content()
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast_embed(:attachment)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Audio", "Video"])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_host_match()
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
index 720213d73..b3638cfc7 100644
--- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex
@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
+  alias Pleroma.Object.Containment
   alias Pleroma.Web.ActivityPub.Utils
 
   # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults
@@ -19,4 +20,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do
     data
     |> Map.put_new("actor", data["attributedTo"])
   end
+
+  def fix_actor(data) do
+    actor = Containment.get_actor(data)
+
+    data
+    |> Map.put("actor", actor)
+    |> Map.put("attributedTo", actor)
+  end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
index b3dbeea57..422ee07be 100644
--- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex
@@ -10,9 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
 
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Object
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   import Ecto.Changeset
-  import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
 
   @primary_key false
 
@@ -75,14 +76,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do
     data
     |> fix_context(meta)
     |> fix_addressing(meta)
+    |> CommonFixes.fix_actor()
   end
 
   def validate_data(cng, meta \\ []) do
     cng
     |> validate_required([:actor, :type, :object])
     |> validate_inclusion(:type, ["Create"])
-    |> validate_actor_presence()
-    |> validate_any_presence([:to, :cc])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_any_presence([:to, :cc])
     |> validate_actors_match(meta)
     |> validate_context_match(meta)
     |> validate_object_nonexistence()
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index 46a8be767..b5c720c7a 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -336,7 +336,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   end
 
   def handle_object_creation(%{"type" => objtype} = object, meta)
-      when objtype in ~w[Audio Question Event] do
+      when objtype in ~w[Audio Video Question Event] do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
     end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index ec3b24206..e14936c10 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   A module to handle coding from internal to wire ActivityPub and back.
   """
   alias Pleroma.Activity
-  alias Pleroma.EarmarkRenderer
   alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Maps
   alias Pleroma.Object
@@ -45,7 +44,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> fix_addressing
     |> fix_summary
     |> fix_type(options)
-    |> fix_content
   end
 
   def fix_summary(%{"summary" => nil} = object) do
@@ -274,24 +272,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     Map.put(object, "url", url["href"])
   end
 
-  def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do
-    attachment =
-      Enum.find(url, fn x ->
-        media_type = x["mediaType"] || x["mimeType"] || ""
-
-        is_map(x) and String.starts_with?(media_type, "video/")
-      end)
-
-    link_element =
-      Enum.find(url, fn x -> is_map(x) and (x["mediaType"] || x["mimeType"]) == "text/html" end)
-
-    object
-    |> Map.put("attachment", [attachment])
-    |> Map.put("url", link_element["href"])
-  end
-
-  def fix_url(%{"type" => object_type, "url" => url} = object)
-      when object_type != "Video" and is_list(url) do
+  def fix_url(%{"url" => url} = object) when is_list(url) do
     first_element = Enum.at(url, 0)
 
     url_string =
@@ -371,18 +352,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_type(object, _), do: object
 
-  defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = object)
-       when is_binary(content) do
-    html_content =
-      content
-      |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer})
-      |> Pleroma.HTML.filter_tags()
-
-    Map.merge(object, %{"content" => html_content, "mediaType" => "text/html"})
-  end
-
-  defp fix_content(object), do: object
-
   # Reduce the object list to find the reported user.
   defp get_reported(objects) do
     Enum.reduce_while(objects, nil, fn ap_id, _ ->
@@ -455,7 +424,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ~w{Article Note Video Page} do
+      when objtype in ~w{Article Note Page} do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -549,7 +518,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype}} = data,
         _options
       )
-      when objtype in ~w{Question Answer ChatMessage Audio Event} do
+      when objtype in ~w{Question Answer ChatMessage Audio Video Event} do
     data = Map.put(data, "object", strip_internal_fields(data["object"]))
 
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
-- 
cgit v1.2.3


From 2132b24a9df8116e12abc8c458cff4c3850aeda0 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 20 Aug 2020 04:27:59 +0200
Subject: object_validators: likes & announcements as [ObjectID]

---
 .../web/activity_pub/object_validators/audio_video_validator.ex       | 4 ++--
 lib/pleroma/web/activity_pub/object_validators/note_validator.ex      | 4 ++--
 lib/pleroma/web/activity_pub/object_validators/question_validator.ex  | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex
index a6119e627..16973e5db 100644
--- a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex
@@ -49,8 +49,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do
     field(:inReplyTo, ObjectValidators.ObjectID)
     field(:url, ObjectValidators.Uri)
 
-    field(:likes, {:array, :string}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
   end
 
   def cast_and_apply(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
index ab4469a59..e47cbaaea 100644
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
@@ -43,8 +43,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
     field(:inReplyTo, ObjectValidators.ObjectID)
     field(:url, ObjectValidators.Uri)
 
-    field(:likes, {:array, :string}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
   end
 
   def cast_and_validate(data) do
diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
index 934d3c1ea..9310485dc 100644
--- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex
@@ -47,8 +47,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do
     # short identifier for PleromaFE to group statuses by context
     field(:context_id, :integer)
 
-    field(:likes, {:array, :string}, default: [])
-    field(:announcements, {:array, :string}, default: [])
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
 
     field(:closed, ObjectValidators.DateTime)
     field(:voters, {:array, ObjectValidators.ObjectID}, default: [])
-- 
cgit v1.2.3


From 1b3d5956b1be7faac4e1230d788307650acce991 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Thu, 20 Aug 2020 20:03:07 +0200
Subject: Pipeline Ingestion: Article

---
 lib/pleroma/web/activity_pub/activity_pub.ex       |   2 +-
 lib/pleroma/web/activity_pub/object_validator.ex   |  17 +++-
 .../object_validators/article_note_validator.ex    | 106 +++++++++++++++++++++
 .../object_validators/note_validator.ex            |  73 --------------
 lib/pleroma/web/activity_pub/side_effects.ex       |   2 +-
 lib/pleroma/web/activity_pub/transmogrifier.ex     |   4 +-
 6 files changed, 126 insertions(+), 78 deletions(-)
 create mode 100644 lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex
 delete mode 100644 lib/pleroma/web/activity_pub/object_validators/note_validator.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index bceec8bd1..3ab045737 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -84,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp increase_replies_count_if_reply(_create_data), do: :noop
 
-  @object_types ~w[ChatMessage Question Answer Audio Video Event]
+  @object_types ~w[ChatMessage Question Answer Audio Video Event Article]
   @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()}
   def persist(%{"type" => type} = object, meta) when type in @object_types do
     with {:ok, object} <- Object.create(object) do
diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex
index 081f96389..bd0a2a8dc 100644
--- a/lib/pleroma/web/activity_pub/object_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validator.ex
@@ -17,6 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
   alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator
   alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator
@@ -160,6 +161,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     end
   end
 
+  def validate(%{"type" => "Article"} = object, meta) do
+    with {:ok, object} <-
+           object
+           |> ArticleNoteValidator.cast_and_validate()
+           |> Ecto.Changeset.apply_action(:insert) do
+      object = stringify_keys(object)
+      {:ok, object, meta}
+    end
+  end
+
   def validate(%{"type" => "Answer"} = object, meta) do
     with {:ok, object} <-
            object
@@ -199,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity,
         meta
       )
-      when objtype in ~w[Question Answer Audio Video Event] do
+      when objtype in ~w[Question Answer Audio Video Event Article] do
     with {:ok, object_data} <- cast_and_apply(object),
          meta = Keyword.put(meta, :object_data, object_data |> stringify_keys),
          {:ok, create_activity} <-
@@ -241,6 +252,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do
     EventValidator.cast_and_apply(object)
   end
 
+  def cast_and_apply(%{"type" => "Article"} = object) do
+    ArticleNoteValidator.cast_and_apply(object)
+  end
+
   def cast_and_apply(o), do: {:error, {:validator_not_set, o}}
 
   # is_struct/1 isn't present in Elixir 1.8.x
diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex
new file mode 100644
index 000000000..5b7dad517
--- /dev/null
+++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex
@@ -0,0 +1,106 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do
+  use Ecto.Schema
+
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
+  alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes
+  alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+
+  import Ecto.Changeset
+
+  @primary_key false
+  @derive Jason.Encoder
+
+  embedded_schema do
+    field(:id, ObjectValidators.ObjectID, primary_key: true)
+    field(:to, ObjectValidators.Recipients, default: [])
+    field(:cc, ObjectValidators.Recipients, default: [])
+    field(:bto, ObjectValidators.Recipients, default: [])
+    field(:bcc, ObjectValidators.Recipients, default: [])
+    # TODO: Write type
+    field(:tag, {:array, :map}, default: [])
+    field(:type, :string)
+
+    field(:name, :string)
+    field(:summary, :string)
+    field(:content, :string)
+
+    field(:context, :string)
+    # short identifier for PleromaFE to group statuses by context
+    field(:context_id, :integer)
+
+    # TODO: Remove actor on objects
+    field(:actor, ObjectValidators.ObjectID)
+
+    field(:attributedTo, ObjectValidators.ObjectID)
+    field(:published, ObjectValidators.DateTime)
+    field(:emoji, ObjectValidators.Emoji, default: %{})
+    field(:sensitive, :boolean, default: false)
+    embeds_many(:attachment, AttachmentValidator)
+    field(:replies_count, :integer, default: 0)
+    field(:like_count, :integer, default: 0)
+    field(:announcement_count, :integer, default: 0)
+    field(:inReplyTo, ObjectValidators.ObjectID)
+    field(:url, ObjectValidators.Uri)
+
+    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
+    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
+  end
+
+  def cast_and_apply(data) do
+    data
+    |> cast_data
+    |> apply_action(:insert)
+  end
+
+  def cast_and_validate(data) do
+    data
+    |> cast_data()
+    |> validate_data()
+  end
+
+  def cast_data(data) do
+    data = fix(data)
+
+    %__MODULE__{}
+    |> changeset(data)
+  end
+
+  defp fix_url(%{"url" => url} = data) when is_map(url) do
+    Map.put(data, "url", url["href"])
+  end
+
+  defp fix_url(data), do: data
+
+  defp fix(data) do
+    data
+    |> CommonFixes.fix_defaults()
+    |> CommonFixes.fix_attribution()
+    |> CommonFixes.fix_actor()
+    |> fix_url()
+    |> Transmogrifier.fix_emoji()
+  end
+
+  def changeset(struct, data) do
+    data = fix(data)
+
+    struct
+    |> cast(data, __schema__(:fields) -- [:attachment])
+    |> cast_embed(:attachment)
+  end
+
+  def validate_data(data_cng) do
+    data_cng
+    |> validate_inclusion(:type, ["Article", "Note"])
+    |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id])
+    |> CommonValidations.validate_any_presence([:cc, :to])
+    |> CommonValidations.validate_fields_match([:actor, :attributedTo])
+    |> CommonValidations.validate_actor_presence()
+    |> CommonValidations.validate_host_match()
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
deleted file mode 100644
index e47cbaaea..000000000
--- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex
+++ /dev/null
@@ -1,73 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-  alias Pleroma.Web.ActivityPub.Transmogrifier
-
-  import Ecto.Changeset
-
-  @primary_key false
-
-  embedded_schema do
-    field(:id, ObjectValidators.ObjectID, primary_key: true)
-    field(:to, ObjectValidators.Recipients, default: [])
-    field(:cc, ObjectValidators.Recipients, default: [])
-    field(:bto, ObjectValidators.Recipients, default: [])
-    field(:bcc, ObjectValidators.Recipients, default: [])
-    # TODO: Write type
-    field(:tag, {:array, :map}, default: [])
-    field(:type, :string)
-
-    field(:name, :string)
-    field(:summary, :string)
-    field(:content, :string)
-
-    field(:context, :string)
-    # short identifier for PleromaFE to group statuses by context
-    field(:context_id, :integer)
-
-    field(:actor, ObjectValidators.ObjectID)
-    field(:attributedTo, ObjectValidators.ObjectID)
-    field(:published, ObjectValidators.DateTime)
-    field(:emoji, ObjectValidators.Emoji, default: %{})
-    field(:sensitive, :boolean, default: false)
-    # TODO: Write type
-    field(:attachment, {:array, :map}, default: [])
-    field(:replies_count, :integer, default: 0)
-    field(:like_count, :integer, default: 0)
-    field(:announcement_count, :integer, default: 0)
-    field(:inReplyTo, ObjectValidators.ObjectID)
-    field(:url, ObjectValidators.Uri)
-
-    field(:likes, {:array, ObjectValidators.ObjectID}, default: [])
-    field(:announcements, {:array, ObjectValidators.ObjectID}, default: [])
-  end
-
-  def cast_and_validate(data) do
-    data
-    |> cast_data()
-    |> validate_data()
-  end
-
-  defp fix(data) do
-    data
-    |> Transmogrifier.fix_emoji()
-  end
-
-  def cast_data(data) do
-    data = fix(data)
-
-    %__MODULE__{}
-    |> cast(data, __schema__(:fields))
-  end
-
-  def validate_data(data_cng) do
-    data_cng
-    |> validate_inclusion(:type, ["Note"])
-    |> validate_required([:id, :actor, :to, :cc, :type, :content, :context])
-  end
-end
diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex
index b5c720c7a..b9a83a544 100644
--- a/lib/pleroma/web/activity_pub/side_effects.ex
+++ b/lib/pleroma/web/activity_pub/side_effects.ex
@@ -336,7 +336,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do
   end
 
   def handle_object_creation(%{"type" => objtype} = object, meta)
-      when objtype in ~w[Audio Video Question Event] do
+      when objtype in ~w[Audio Video Question Event Article] do
     with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do
       {:ok, object, meta}
     end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index e14936c10..80f529704 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -424,7 +424,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype} = object} = data,
         options
       )
-      when objtype in ~w{Article Note Page} do
+      when objtype in ~w{Note Page} do
     actor = Containment.get_actor(data)
 
     with nil <- Activity.get_create_by_object_ap_id(object["id"]),
@@ -518,7 +518,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         %{"type" => "Create", "object" => %{"type" => objtype}} = data,
         _options
       )
-      when objtype in ~w{Question Answer ChatMessage Audio Video Event} do
+      when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do
     data = Map.put(data, "object", strip_internal_fields(data["object"]))
 
     with {:ok, %User{}} <- ObjectValidator.fetch_actor(data),
-- 
cgit v1.2.3


From f18178cb096b9a00ed12ff0fe36893f118ec6649 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 22 Aug 2020 02:01:33 +0200
Subject: AttachmentValidator: directly embed url schema and pass it
 fix_media_type

---
 .../object_validators/attachment_validator.ex      | 21 +++++++++++++++++--
 .../object_validators/url_object_validator.ex      | 24 ----------------------
 2 files changed, 19 insertions(+), 26 deletions(-)
 delete mode 100644 lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
index c8b148280..df102a134 100644
--- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
+++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
   use Ecto.Schema
 
+  alias Pleroma.EctoType.ActivityPub.ObjectValidators
   alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator
 
   import Ecto.Changeset
@@ -15,7 +16,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
     field(:mediaType, :string, default: "application/octet-stream")
     field(:name, :string)
 
-    embeds_many(:url, UrlObjectValidator)
+    embeds_many :url, UrlObjectValidator, primary_key: false do
+      field(:type, :string)
+      field(:href, ObjectValidators.Uri)
+      field(:mediaType, :string, default: "application/octet-stream")
+    end
   end
 
   def cast_and_validate(data) do
@@ -37,7 +42,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
 
     struct
     |> cast(data, [:type, :mediaType, :name])
-    |> cast_embed(:url, required: true)
+    |> cast_embed(:url, with: &url_changeset/2)
+    |> validate_inclusion(:type, ~w[Link Document Audio Image Video])
+    |> validate_required([:type, :mediaType, :url])
+  end
+
+  def url_changeset(struct, data) do
+    data = fix_media_type(data)
+
+    struct
+    |> cast(data, [:type, :href, :mediaType])
+    |> validate_inclusion(:type, ["Link"])
+    |> validate_required([:type, :href, :mediaType])
   end
 
   def fix_media_type(data) do
@@ -75,6 +91,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do
 
   def validate_data(cng) do
     cng
+    |> validate_inclusion(:type, ~w[Document Audio Image Video])
     |> validate_required([:mediaType, :url, :type])
   end
 end
diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
deleted file mode 100644
index 881030f38..000000000
--- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex
+++ /dev/null
@@ -1,24 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do
-  use Ecto.Schema
-
-  alias Pleroma.EctoType.ActivityPub.ObjectValidators
-
-  import Ecto.Changeset
-  @primary_key false
-
-  embedded_schema do
-    field(:type, :string)
-    field(:href, ObjectValidators.Uri)
-    field(:mediaType, :string, default: "application/octet-stream")
-  end
-
-  def changeset(struct, data) do
-    struct
-    |> cast(data, __schema__(:fields))
-    |> validate_required([:type, :href, :mediaType])
-  end
-end
-- 
cgit v1.2.3


From dbc013f24c3885960714425f201e372335d22345 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 11 Sep 2020 11:22:50 +0200
Subject: instance: Handle not getting a favicon

---
 lib/pleroma/instances/instance.ex | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 8bf53c090..6948651c7 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -159,13 +159,11 @@ defmodule Pleroma.Instances.Instance do
              Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
                adapter: [pool: :media]
              ),
-           favicon_rel <-
-             html
-             |> Floki.parse_document!()
-             |> Floki.attribute("link[rel=icon]", "href")
-             |> List.first(),
-           favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(),
-           true <- is_binary(favicon) do
+           {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
+             {:parse,
+              html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
+           {_, favicon} when is_binary(favicon) <-
+             {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
         favicon
       else
         _ -> nil
-- 
cgit v1.2.3


From 36c9197ac36707cdfe3d679bbd64972b4b03ea84 Mon Sep 17 00:00:00 2001
From: Haelwenn <contact+git.pleroma.social@hacktivis.me>
Date: Fri, 11 Sep 2020 10:46:16 +0000
Subject: Apply 1 suggestion(s) to 1 file(s)

---
 lib/pleroma/web/federator/federator.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index e4ab9ba32..130654145 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -94,7 +94,7 @@ defmodule Pleroma.Web.Federator do
 
       e ->
         # Just drop those for now
-        Logger.debug("Unhandled activity\n" <> Jason.encode!(params, pretty: true))
+        Logger.debug(fn -> "Unhandled activity\n" <> Jason.encode!(params, pretty: true) end)
         {:error, e}
     end
   end
-- 
cgit v1.2.3


From 89a7efab69d905cc3521388b1e1cf43851848627 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 11 Sep 2020 14:22:54 +0300
Subject: ConnectionPool: Log possible HTTP1 blocks

---
 lib/pleroma/gun/conn.ex                   | 12 ++++++------
 lib/pleroma/gun/connection_pool/worker.ex | 22 ++++++++++++++++------
 lib/pleroma/telemetry/logger.ex           | 18 ++++++++++++++++--
 3 files changed, 38 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index 75b1ffc0a..477e19c6e 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -50,10 +50,10 @@ defmodule Pleroma.Gun.Conn do
 
     with open_opts <- Map.delete(opts, :tls_opts),
          {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]),
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
          stream <- Gun.connect(conn, connect_opts),
          {:response, :fin, 200, _} <- Gun.await(conn, stream) do
-      {:ok, conn}
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(
@@ -88,8 +88,8 @@ defmodule Pleroma.Gun.Conn do
       |> Map.put(:socks_opts, socks_opts)
 
     with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
-      {:ok, conn}
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(
@@ -106,8 +106,8 @@ defmodule Pleroma.Gun.Conn do
     host = Pleroma.HTTP.AdapterHelper.parse_host(host)
 
     with {:ok, conn} <- Gun.open(host, port, opts),
-         {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
-      {:ok, conn}
+         {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
+      {:ok, conn, protocol}
     else
       error ->
         Logger.warn(
diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index c36332817..49d41e4c7 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -15,7 +15,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
 
   @impl true
   def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
-    with {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
+    with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
          Process.link(conn_pid) do
       time = :erlang.monotonic_time(:millisecond)
 
@@ -27,8 +27,12 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
       send(client_pid, {:conn_pid, conn_pid})
 
       {:noreply,
-       %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}},
-       :hibernate}
+       %{
+         key: key,
+         timer: nil,
+         client_monitors: %{client_pid => Process.monitor(client_pid)},
+         protocol: protocol
+       }, :hibernate}
     else
       err ->
         {:stop, {:shutdown, err}, nil}
@@ -53,14 +57,20 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
   end
 
   @impl true
-  def handle_call(:add_client, {client_pid, _}, %{key: key} = state) do
+  def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
     time = :erlang.monotonic_time(:millisecond)
 
-    {{conn_pid, _, _, _}, _} =
+    {{conn_pid, used_by, _, _}, _} =
       Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
         {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
       end)
 
+    :telemetry.execute(
+      [:pleroma, :connection_pool, :client, :add],
+      %{client_pid: client_pid, clients: used_by},
+      %{key: state.key, protocol: protocol}
+    )
+
     state =
       if state.timer != nil do
         Process.cancel_timer(state[:timer])
@@ -131,7 +141,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
   @impl true
   def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
     :telemetry.execute(
-      [:pleroma, :connection_pool, :client_death],
+      [:pleroma, :connection_pool, :client, :dead],
       %{client_pid: pid, reason: reason},
       %{key: state.key}
     )
diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex
index 4cacae02f..197b1d091 100644
--- a/lib/pleroma/telemetry/logger.ex
+++ b/lib/pleroma/telemetry/logger.ex
@@ -7,7 +7,8 @@ defmodule Pleroma.Telemetry.Logger do
     [:pleroma, :connection_pool, :reclaim, :start],
     [:pleroma, :connection_pool, :reclaim, :stop],
     [:pleroma, :connection_pool, :provision_failure],
-    [:pleroma, :connection_pool, :client_death]
+    [:pleroma, :connection_pool, :client, :dead],
+    [:pleroma, :connection_pool, :client, :add]
   ]
   def attach do
     :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, [])
@@ -62,7 +63,7 @@ defmodule Pleroma.Telemetry.Logger do
   end
 
   def handle_event(
-        [:pleroma, :connection_pool, :client_death],
+        [:pleroma, :connection_pool, :client, :dead],
         %{client_pid: client_pid, reason: reason},
         %{key: key},
         _
@@ -73,4 +74,17 @@ defmodule Pleroma.Telemetry.Logger do
       }"
     end)
   end
+
+  def handle_event(
+        [:pleroma, :connection_pool, :client, :add],
+        %{clients: [_, _ | _] = clients},
+        %{key: key, protocol: :http},
+        _
+      ) do
+    Logger.info(fn ->
+      "Pool worker for #{key}: #{length(clients)} clients are using an HTTP1 connection at the same time, head-of-line blocking might occur."
+    end)
+  end
+
+  def handle_event([:pleroma, :connection_pool, :client, :add], _, _, _), do: :ok
 end
-- 
cgit v1.2.3


From f1f44069ae525fd21127e5ceccc61016c12f4427 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 11 Sep 2020 19:58:58 +0200
Subject: Fetcher: Correctly return MRF reject reason

---
 lib/pleroma/object/fetcher.ex                | 4 ++--
 lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 1de2ce6c3..24dc7cb95 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -98,8 +98,8 @@ defmodule Pleroma.Object.Fetcher do
       {:containment, _} ->
         {:error, "Object containment failed."}
 
-      {:transmogrifier, {:error, {:reject, nil}}} ->
-        {:reject, nil}
+      {:transmogrifier, {:error, {:reject, e}}} ->
+        {:reject, e}
 
       {:transmogrifier, _} = e ->
         {:error, e}
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 66a9f78a3..b2205bff7 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -154,8 +154,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
       {:remote_limit_pass, _} ->
         {:error, :remote_limit}
 
-      {:reject, reason} ->
-        {:error, reason}
+      {:reject, _} = e ->
+        {:error, e}
     end
   end
 
-- 
cgit v1.2.3


From f88dc1937e5aa4208143fa68400a5c38a1b9eddf Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Mon, 31 Aug 2020 16:48:24 -0500
Subject: MastodonAPI.StatusView.get_user/1 --> CommonAPI.get_user/1

---
 lib/pleroma/web/admin_api/views/status_view.ex     |  3 ++-
 lib/pleroma/web/common_api/common_api.ex           | 17 +++++++++++++++
 lib/pleroma/web/mastodon_api/views/status_view.ex  | 25 ++++------------------
 lib/pleroma/web/pleroma_api/views/scrobble_view.ex |  4 ++--
 4 files changed, 25 insertions(+), 24 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/views/status_view.ex b/lib/pleroma/web/admin_api/views/status_view.ex
index 500800be2..6042a22b6 100644
--- a/lib/pleroma/web/admin_api/views/status_view.ex
+++ b/lib/pleroma/web/admin_api/views/status_view.ex
@@ -8,6 +8,7 @@ defmodule Pleroma.Web.AdminAPI.StatusView do
   require Pleroma.Constants
 
   alias Pleroma.Web.AdminAPI
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.MastodonAPI
 
   defdelegate merge_account_views(user), to: AdminAPI.AccountView
@@ -17,7 +18,7 @@ defmodule Pleroma.Web.AdminAPI.StatusView do
   end
 
   def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do
-    user = MastodonAPI.StatusView.get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
 
     MastodonAPI.StatusView.render("show.json", opts)
     |> Map.merge(%{account: merge_account_views(user)})
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 5ad2b91c2..d6e9d3d67 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -550,4 +550,21 @@ defmodule Pleroma.Web.CommonAPI do
   def show_reblogs(%User{} = user, %User{} = target) do
     UserRelationship.delete_reblog_mute(user, target)
   end
+
+  def get_user(ap_id, fake_record_fallback \\ true) do
+    cond do
+      user = User.get_cached_by_ap_id(ap_id) ->
+        user
+
+      user = User.get_by_guessed_nickname(ap_id) ->
+        user
+
+      fake_record_fallback ->
+        # TODO: refactor (fake records is never a good idea)
+        User.error_user(ap_id)
+
+      true ->
+        nil
+    end
+  end
 end
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 3fe1967be..66732d09e 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -56,23 +56,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end)
   end
 
-  def get_user(ap_id, fake_record_fallback \\ true) do
-    cond do
-      user = User.get_cached_by_ap_id(ap_id) ->
-        user
-
-      user = User.get_by_guessed_nickname(ap_id) ->
-        user
-
-      fake_record_fallback ->
-        # TODO: refactor (fake records is never a good idea)
-        User.error_user(ap_id)
-
-      true ->
-        nil
-    end
-  end
-
   defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id),
     do: context_id
 
@@ -120,7 +103,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
           # Note: unresolved users are filtered out
           actors =
             (activities ++ parent_activities)
-            |> Enum.map(&get_user(&1.data["actor"], false))
+            |> Enum.map(&CommonAPI.get_user(&1.data["actor"], false))
             |> Enum.filter(& &1)
 
           UserRelationship.view_relationships_option(reading_user, actors, subset: :source_mutes)
@@ -139,7 +122,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
         "show.json",
         %{activity: %{data: %{"type" => "Announce", "object" => _object}} = activity} = opts
       ) do
-    user = get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
     activity_object = Object.normalize(activity)
 
@@ -212,7 +195,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   def render("show.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do
     object = Object.normalize(activity)
 
-    user = get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
     user_follower_address = user.follower_address
 
     like_count = object.data["like_count"] || 0
@@ -266,7 +249,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     reply_to = get_reply_to(activity, opts)
 
-    reply_to_user = reply_to && get_user(reply_to.data["actor"])
+    reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"])
 
     content =
       object
diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex
index bbff93abe..95bd4c368 100644
--- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex
@@ -10,14 +10,14 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do
   alias Pleroma.Activity
   alias Pleroma.HTML
   alias Pleroma.Object
+  alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MastodonAPI.AccountView
-  alias Pleroma.Web.MastodonAPI.StatusView
 
   def render("show.json", %{activity: %Activity{data: %{"type" => "Listen"}} = activity} = opts) do
     object = Object.normalize(activity)
 
-    user = StatusView.get_user(activity.data["actor"])
+    user = CommonAPI.get_user(activity.data["actor"])
     created_at = Utils.to_masto_date(activity.data["published"])
 
     %{
-- 
cgit v1.2.3


From b40a627ab02f9f63eac42ce6fc65282fc6cb6b92 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Mon, 31 Aug 2020 19:56:05 -0500
Subject: AdminAPI: delete a chat message

---
 lib/pleroma/moderation_log.ex                      | 24 ++++++++++++
 .../web/admin_api/controllers/chat_controller.ex   | 37 ++++++++++++++++++
 .../api_spec/operations/admin/chat_operation.ex    | 44 ++++++++++++++++++++++
 lib/pleroma/web/router.ex                          |  2 +
 4 files changed, 107 insertions(+)
 create mode 100644 lib/pleroma/web/admin_api/controllers/chat_controller.ex
 create mode 100644 lib/pleroma/web/api_spec/operations/admin/chat_operation.ex

(limited to 'lib')

diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex
index 31c9afe2a..47036a6f6 100644
--- a/lib/pleroma/moderation_log.ex
+++ b/lib/pleroma/moderation_log.ex
@@ -320,6 +320,19 @@ defmodule Pleroma.ModerationLog do
     |> insert_log_entry_with_message()
   end
 
+  @spec insert_log(%{actor: User, action: String.t(), subject_id: String.t()}) ::
+          {:ok, ModerationLog} | {:error, any}
+  def insert_log(%{actor: %User{} = actor, action: "chat_message_delete", subject_id: subject_id}) do
+    %ModerationLog{
+      data: %{
+        "actor" => %{"nickname" => actor.nickname},
+        "action" => "chat_message_delete",
+        "subject_id" => subject_id
+      }
+    }
+    |> insert_log_entry_with_message()
+  end
+
   @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any}
   defp insert_log_entry_with_message(entry) do
     entry.data["message"]
@@ -627,6 +640,17 @@ defmodule Pleroma.ModerationLog do
     "@#{actor_nickname} updated users: #{users_to_nicknames_string(subjects)}"
   end
 
+  @spec get_log_entry_message(ModerationLog) :: String.t()
+  def get_log_entry_message(%ModerationLog{
+        data: %{
+          "actor" => %{"nickname" => actor_nickname},
+          "action" => "chat_message_delete",
+          "subject_id" => subject_id
+        }
+      }) do
+    "@#{actor_nickname} deleted chat message ##{subject_id}"
+  end
+
   defp nicknames_to_string(nicknames) do
     nicknames
     |> Enum.map(&"@#{&1}")
diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
new file mode 100644
index 000000000..bcce824d2
--- /dev/null
+++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
@@ -0,0 +1,37 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ChatController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Activity
+  alias Pleroma.ModerationLog
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.CommonAPI
+
+  require Logger
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["write:chats"], admin: true} when action in [:delete_message]
+  )
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ChatOperation
+
+  def delete_message(%{assigns: %{user: user}} = conn, %{message_id: id}) do
+    with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do
+      ModerationLog.insert_log(%{
+        action: "chat_message_delete",
+        actor: user,
+        subject_id: id
+      })
+
+      json(conn, %{})
+    end
+  end
+end
diff --git a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
new file mode 100644
index 000000000..7045fd7ce
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
@@ -0,0 +1,44 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
+  alias OpenApiSpex.Operation
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
+
+  import Pleroma.Web.ApiSpec.Helpers
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def delete_message_operation do
+    %Operation{
+      tags: ["Admin", "Chats"],
+      summary: "Delete an individual chat message",
+      operationId: "AdminAPI.ChatController.delete",
+      parameters: [id_param(), message_id_param()] ++ admin_api_params(),
+      security: [%{"oAuth" => ["write:chats"]}],
+      responses: %{
+        200 => empty_object_response(),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def id_param do
+    Operation.parameter(:id, :path, FlakeID, "Chat ID",
+      example: "9umDrYheeY451cQnEe",
+      required: true
+    )
+  end
+
+  def message_id_param do
+    Operation.parameter(:message_id, :path, FlakeID, "Chat message ID",
+      example: "9umDrYheeY451cQnEe",
+      required: true
+    )
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index c6433cc53..e438768ed 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -214,6 +214,8 @@ defmodule Pleroma.Web.Router do
     get("/media_proxy_caches", MediaProxyCacheController, :index)
     post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
     post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
+
+    delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
   end
 
   scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do
-- 
cgit v1.2.3


From fb0de073439b5e3be823e736b44608e80f1027f1 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Mon, 31 Aug 2020 20:23:33 -0500
Subject: AdminAPI: list chats for a user

---
 .../admin_api/controllers/admin_api_controller.ex  | 27 ++++++++++++++++++++++
 lib/pleroma/web/router.ex                          |  3 +++
 2 files changed, 30 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index f5e4d49f9..9b66c2f10 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
 
+  import Ecto.Query
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
   alias Pleroma.Config
@@ -21,6 +22,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.AdminAPI.ModerationLogView
   alias Pleroma.Web.AdminAPI.Search
   alias Pleroma.Web.Endpoint
+  alias Pleroma.Web.PleromaAPI
   alias Pleroma.Web.Router
 
   require Logger
@@ -68,6 +70,12 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     when action in [:list_user_statuses, :list_instance_statuses]
   )
 
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:chats"], admin: true}
+    when action in [:list_user_chats]
+  )
+
   plug(
     OAuthScopesPlug,
     %{scopes: ["read"], admin: true}
@@ -256,6 +264,25 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = _params) do
+    with %User{id: user_id} <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
+      chats =
+        from(c in Pleroma.Chat,
+          where: c.user_id == ^user_id,
+          order_by: [desc: c.updated_at],
+          inner_join: u in User,
+          on: u.ap_id == c.recipient
+        )
+        |> Pleroma.Repo.all()
+
+      conn
+      |> put_view(PleromaAPI.ChatView)
+      |> render("index.json", chats: chats)
+    else
+      _ -> {:error, :not_found}
+    end
+  end
+
   def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
     user = User.get_cached_by_nickname(nickname)
 
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e438768ed..ad3282df4 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -178,6 +178,7 @@ defmodule Pleroma.Web.Router do
     get("/users", AdminAPIController, :list_users)
     get("/users/:nickname", AdminAPIController, :user_show)
     get("/users/:nickname/statuses", AdminAPIController, :list_user_statuses)
+    get("/users/:nickname/chats", AdminAPIController, :list_user_chats)
 
     get("/instances/:instance/statuses", AdminAPIController, :list_instance_statuses)
 
@@ -215,6 +216,8 @@ defmodule Pleroma.Web.Router do
     post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
     post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
 
+    # get("/chats/:id", ChatController, :show)
+    # get("/chats/:id/messages", ChatController, :messages)
     delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
   end
 
-- 
cgit v1.2.3


From c41430b23eaf3fd15b227e66215aa2a4ff31dfdb Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 1 Sep 2020 19:05:24 -0500
Subject: Refactor with Chat.for_user_query/1

---
 lib/pleroma/chat.ex                                          | 12 ++++++++++++
 .../web/admin_api/controllers/admin_api_controller.ex        |  8 +-------
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex   |  9 ++-------
 3 files changed, 15 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex
index 24a86371e..b38c5c3dd 100644
--- a/lib/pleroma/chat.ex
+++ b/lib/pleroma/chat.ex
@@ -6,7 +6,9 @@ defmodule Pleroma.Chat do
   use Ecto.Schema
 
   import Ecto.Changeset
+  import Ecto.Query
 
+  alias Pleroma.Chat
   alias Pleroma.Repo
   alias Pleroma.User
 
@@ -69,4 +71,14 @@ defmodule Pleroma.Chat do
       conflict_target: [:user_id, :recipient]
     )
   end
+
+  @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t()
+  def for_user_query(user_id) do
+    from(c in Chat,
+      where: c.user_id == ^user_id,
+      order_by: [desc: c.updated_at],
+      inner_join: u in User,
+      on: u.ap_id == c.recipient
+    )
+  end
 end
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index 9b66c2f10..fccdbabb4 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -5,7 +5,6 @@
 defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   use Pleroma.Web, :controller
 
-  import Ecto.Query
   import Pleroma.Web.ControllerHelper, only: [json_response: 3]
 
   alias Pleroma.Config
@@ -267,12 +266,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   def list_user_chats(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = _params) do
     with %User{id: user_id} <- User.get_cached_by_nickname_or_id(nickname, for: admin) do
       chats =
-        from(c in Pleroma.Chat,
-          where: c.user_id == ^user_id,
-          order_by: [desc: c.updated_at],
-          inner_join: u in User,
-          on: u.ap_id == c.recipient
-        )
+        Pleroma.Chat.for_user_query(user_id)
         |> Pleroma.Repo.all()
 
       conn
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 1f2e953f7..27c9a2e0f 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -146,13 +146,8 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
     blocked_ap_ids = User.blocked_users_ap_ids(user)
 
     chats =
-      from(c in Chat,
-        where: c.user_id == ^user_id,
-        where: c.recipient not in ^blocked_ap_ids,
-        order_by: [desc: c.updated_at],
-        inner_join: u in User,
-        on: u.ap_id == c.recipient
-      )
+      Chat.for_user_query(user_id)
+      |> where([c], c.recipient not in ^blocked_ap_ids)
       |> Repo.all()
 
     conn
-- 
cgit v1.2.3


From f13b52a703d5c60cf12b2fff69f458e5c467c783 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 1 Sep 2020 19:39:34 -0500
Subject: AdminAPI: list messages in a chat

---
 .../web/admin_api/controllers/chat_controller.ex   | 27 ++++++++++++++++++++++
 .../api_spec/operations/admin/chat_operation.ex    | 26 ++++++++++++++++++++-
 lib/pleroma/web/router.ex                          |  2 +-
 3 files changed, 53 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
index bcce824d2..b423188d7 100644
--- a/lib/pleroma/web/admin_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
@@ -6,14 +6,23 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
   use Pleroma.Web, :controller
 
   alias Pleroma.Activity
+  alias Pleroma.Chat
+  alias Pleroma.Chat.MessageReference
   alias Pleroma.ModerationLog
+  alias Pleroma.Pagination
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
 
   require Logger
 
   plug(Pleroma.Web.ApiSpec.CastAndValidate)
 
+  plug(
+    OAuthScopesPlug,
+    %{scopes: ["read:chats"], admin: true} when action in [:messages]
+  )
+
   plug(
     OAuthScopesPlug,
     %{scopes: ["write:chats"], admin: true} when action in [:delete_message]
@@ -34,4 +43,22 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
       json(conn, %{})
     end
   end
+
+  def messages(conn, %{id: id} = params) do
+    with %Chat{} = chat <- Chat.get_by_id(id) do
+      cm_refs =
+        chat
+        |> MessageReference.for_chat_query()
+        |> Pagination.fetch_paginated(params)
+
+      conn
+      |> put_view(MessageReferenceView)
+      |> render("index.json", chat_message_references: cm_refs)
+    else
+      _ ->
+        conn
+        |> put_status(:not_found)
+        |> json(%{error: "not found"})
+    end
+  end
 end
diff --git a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
index 7045fd7ce..a382bd35a 100644
--- a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
 
   def delete_message_operation do
     %Operation{
-      tags: ["Admin", "Chats"],
+      tags: ["admin", "chat"],
       summary: "Delete an individual chat message",
       operationId: "AdminAPI.ChatController.delete",
       parameters: [id_param(), message_id_param()] ++ admin_api_params(),
@@ -28,6 +28,30 @@ defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
     }
   end
 
+  def messages_operation do
+    %Operation{
+      tags: ["admin", "chat"],
+      summary: "Get the most recent messages of the chat",
+      operationId: "AdminAPI.ChatController.messages",
+      parameters:
+        [Operation.parameter(:id, :path, :string, "The ID of the Chat")] ++
+          pagination_params(),
+      responses: %{
+        200 =>
+          Operation.response(
+            "The messages in the chat",
+            "application/json",
+            Pleroma.Web.ApiSpec.ChatOperation.chat_messages_response()
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["read:chats"]
+        }
+      ]
+    }
+  end
+
   def id_param do
     Operation.parameter(:id, :path, FlakeID, "Chat ID",
       example: "9umDrYheeY451cQnEe",
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index ad3282df4..02836114a 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -217,7 +217,7 @@ defmodule Pleroma.Web.Router do
     post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
 
     # get("/chats/:id", ChatController, :show)
-    # get("/chats/:id/messages", ChatController, :messages)
+    get("/chats/:id/messages", ChatController, :messages)
     delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
   end
 
-- 
cgit v1.2.3


From 9dd0b23da424c380a37897d8bf69ab241efa6f91 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 1 Sep 2020 19:49:46 -0500
Subject: AdminAPI: show chat

---
 .../web/admin_api/controllers/chat_controller.ex   | 11 +++++++-
 .../api_spec/operations/admin/chat_operation.ex    | 32 ++++++++++++++++++++++
 lib/pleroma/web/router.ex                          |  2 +-
 3 files changed, 43 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
index b423188d7..ac362c430 100644
--- a/lib/pleroma/web/admin_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
@@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
+  alias Pleroma.Web.PleromaAPI.ChatView
 
   require Logger
 
@@ -20,7 +21,7 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
 
   plug(
     OAuthScopesPlug,
-    %{scopes: ["read:chats"], admin: true} when action in [:messages]
+    %{scopes: ["read:chats"], admin: true} when action in [:show, :messages]
   )
 
   plug(
@@ -61,4 +62,12 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
         |> json(%{error: "not found"})
     end
   end
+
+  def show(conn, %{id: id}) do
+    with %Chat{} = chat <- Chat.get_by_id(id) do
+      conn
+      |> put_view(ChatView)
+      |> render("show.json", chat: chat)
+    end
+  end
 end
diff --git a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
index a382bd35a..3550d531e 100644
--- a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
   alias OpenApiSpex.Operation
   alias Pleroma.Web.ApiSpec.Schemas.ApiError
+  alias Pleroma.Web.ApiSpec.Schemas.Chat
   alias Pleroma.Web.ApiSpec.Schemas.FlakeID
 
   import Pleroma.Web.ApiSpec.Helpers
@@ -52,6 +53,37 @@ defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
     }
   end
 
+  def show_operation do
+    %Operation{
+      tags: ["chat"],
+      summary: "Create a chat",
+      operationId: "AdminAPI.ChatController.show",
+      parameters: [
+        Operation.parameter(
+          :id,
+          :path,
+          :string,
+          "The id of the chat",
+          required: true,
+          example: "1234"
+        )
+      ],
+      responses: %{
+        200 =>
+          Operation.response(
+            "The existing chat",
+            "application/json",
+            Chat
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["read"]
+        }
+      ]
+    }
+  end
+
   def id_param do
     Operation.parameter(:id, :path, FlakeID, "Chat ID",
       example: "9umDrYheeY451cQnEe",
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 02836114a..e4440d442 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -216,7 +216,7 @@ defmodule Pleroma.Web.Router do
     post("/media_proxy_caches/delete", MediaProxyCacheController, :delete)
     post("/media_proxy_caches/purge", MediaProxyCacheController, :purge)
 
-    # get("/chats/:id", ChatController, :show)
+    get("/chats/:id", ChatController, :show)
     get("/chats/:id/messages", ChatController, :messages)
     delete("/chats/:id/messages/:message_id", ChatController, :delete_message)
   end
-- 
cgit v1.2.3


From 02d70228b566d5de2cbdd6d1f9958caf2db173f1 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 1 Sep 2020 20:40:36 -0500
Subject: AdminAPI: fix delete chat message

---
 .../web/admin_api/controllers/chat_controller.ex   | 20 ++++++++---
 .../api_spec/operations/admin/chat_operation.ex    | 40 ++++++++++------------
 2 files changed, 34 insertions(+), 26 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
index ac362c430..61d45b970 100644
--- a/lib/pleroma/web/admin_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
@@ -33,15 +33,27 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ChatOperation
 
-  def delete_message(%{assigns: %{user: user}} = conn, %{message_id: id}) do
-    with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do
+  def delete_message(%{assigns: %{user: user}} = conn, %{
+        message_id: message_id,
+        id: chat_id
+      }) do
+    with %MessageReference{object: %{data: %{"id" => object_ap_id}}} = cm_ref <-
+           MessageReference.get_by_id(message_id),
+         ^chat_id <- to_string(cm_ref.chat_id),
+         %Activity{id: activity_id} <- Activity.get_create_by_object_ap_id(object_ap_id),
+         {:ok, _} <- CommonAPI.delete(activity_id, user) do
       ModerationLog.insert_log(%{
         action: "chat_message_delete",
         actor: user,
-        subject_id: id
+        subject_id: message_id
       })
 
-      json(conn, %{})
+      conn
+      |> put_view(MessageReferenceView)
+      |> render("show.json", chat_message_reference: cm_ref)
+    else
+      _e ->
+        {:error, :could_not_delete}
     end
   end
 
diff --git a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
index 3550d531e..d3e5dfc1c 100644
--- a/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/chat_operation.ex
@@ -4,9 +4,8 @@
 
 defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
   alias OpenApiSpex.Operation
-  alias Pleroma.Web.ApiSpec.Schemas.ApiError
   alias Pleroma.Web.ApiSpec.Schemas.Chat
-  alias Pleroma.Web.ApiSpec.Schemas.FlakeID
+  alias Pleroma.Web.ApiSpec.Schemas.ChatMessage
 
   import Pleroma.Web.ApiSpec.Helpers
 
@@ -19,13 +18,24 @@ defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
     %Operation{
       tags: ["admin", "chat"],
       summary: "Delete an individual chat message",
-      operationId: "AdminAPI.ChatController.delete",
-      parameters: [id_param(), message_id_param()] ++ admin_api_params(),
-      security: [%{"oAuth" => ["write:chats"]}],
+      operationId: "AdminAPI.ChatController.delete_message",
+      parameters: [
+        Operation.parameter(:id, :path, :string, "The ID of the Chat"),
+        Operation.parameter(:message_id, :path, :string, "The ID of the message")
+      ],
       responses: %{
-        200 => empty_object_response(),
-        404 => Operation.response("Not Found", "application/json", ApiError)
-      }
+        200 =>
+          Operation.response(
+            "The deleted ChatMessage",
+            "application/json",
+            ChatMessage
+          )
+      },
+      security: [
+        %{
+          "oAuth" => ["write:chats"]
+        }
+      ]
     }
   end
 
@@ -83,18 +93,4 @@ defmodule Pleroma.Web.ApiSpec.Admin.ChatOperation do
       ]
     }
   end
-
-  def id_param do
-    Operation.parameter(:id, :path, FlakeID, "Chat ID",
-      example: "9umDrYheeY451cQnEe",
-      required: true
-    )
-  end
-
-  def message_id_param do
-    Operation.parameter(:message_id, :path, FlakeID, "Chat message ID",
-      example: "9umDrYheeY451cQnEe",
-      required: true
-    )
-  end
 end
-- 
cgit v1.2.3


From e229536e5cca65d811f85d25c86bf3c92b3d8c45 Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 10 Sep 2020 01:44:32 -0500
Subject: Chat Moderation: use explicit `sender` and `recipient` fields

---
 .../admin_api/controllers/admin_api_controller.ex  |  5 +---
 .../web/admin_api/controllers/chat_controller.ex   |  4 +--
 lib/pleroma/web/admin_api/views/chat_view.ex       | 30 ++++++++++++++++++++++
 3 files changed, 33 insertions(+), 6 deletions(-)
 create mode 100644 lib/pleroma/web/admin_api/views/chat_view.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index fccdbabb4..d5713c3dd 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -21,11 +21,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   alias Pleroma.Web.AdminAPI.ModerationLogView
   alias Pleroma.Web.AdminAPI.Search
   alias Pleroma.Web.Endpoint
-  alias Pleroma.Web.PleromaAPI
   alias Pleroma.Web.Router
 
-  require Logger
-
   @users_page_size 50
 
   plug(
@@ -270,7 +267,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
         |> Pleroma.Repo.all()
 
       conn
-      |> put_view(PleromaAPI.ChatView)
+      |> put_view(AdminAPI.ChatView)
       |> render("index.json", chats: chats)
     else
       _ -> {:error, :not_found}
diff --git a/lib/pleroma/web/admin_api/controllers/chat_controller.ex b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
index 61d45b970..967600d69 100644
--- a/lib/pleroma/web/admin_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/chat_controller.ex
@@ -11,9 +11,9 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
   alias Pleroma.ModerationLog
   alias Pleroma.Pagination
   alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.AdminAPI
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.PleromaAPI.Chat.MessageReferenceView
-  alias Pleroma.Web.PleromaAPI.ChatView
 
   require Logger
 
@@ -78,7 +78,7 @@ defmodule Pleroma.Web.AdminAPI.ChatController do
   def show(conn, %{id: id}) do
     with %Chat{} = chat <- Chat.get_by_id(id) do
       conn
-      |> put_view(ChatView)
+      |> put_view(AdminAPI.ChatView)
       |> render("show.json", chat: chat)
     end
   end
diff --git a/lib/pleroma/web/admin_api/views/chat_view.ex b/lib/pleroma/web/admin_api/views/chat_view.ex
new file mode 100644
index 000000000..847df1423
--- /dev/null
+++ b/lib/pleroma/web/admin_api/views/chat_view.ex
@@ -0,0 +1,30 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.ChatView do
+  use Pleroma.Web, :view
+
+  alias Pleroma.Chat
+  alias Pleroma.User
+  alias Pleroma.Web.MastodonAPI
+  alias Pleroma.Web.PleromaAPI
+
+  def render("index.json", %{chats: chats} = opts) do
+    render_many(chats, __MODULE__, "show.json", Map.delete(opts, :chats))
+  end
+
+  def render("show.json", %{chat: %Chat{user_id: user_id}} = opts) do
+    user = User.get_by_id(user_id)
+    sender = MastodonAPI.AccountView.render("show.json", user: user, skip_visibility_check: true)
+
+    serialized_chat = PleromaAPI.ChatView.render("show.json", opts)
+
+    serialized_chat
+    |> Map.put(:sender, sender)
+    |> Map.put(:receiver, serialized_chat[:account])
+    |> Map.delete(:account)
+  end
+
+  def render(view, opts), do: PleromaAPI.ChatView.render(view, opts)
+end
-- 
cgit v1.2.3


From c0b36621f1149734e97f268e267202cc53700abb Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 11 Sep 2020 16:59:45 -0500
Subject: Ensure we only apply NSFW Simple policy on parsable objects

---
 lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index bb193475a..161177727 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -66,7 +66,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
            "type" => "Create",
            "object" => child_object
          } = object
-       ) do
+       )
+       when is_map(child_object) do
     media_nsfw =
       Config.get([:mrf_simple, :media_nsfw])
       |> MRF.subdomains_regex()
-- 
cgit v1.2.3


From 32831f371ff426ac0c6f5d6c1381313f5f92af42 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 12 Sep 2020 10:33:42 +0300
Subject: [#2497] Media preview proxy: redirecting to media proxy url in case
 of preview error or unsupported content type.

---
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index ff7fd2409..08d62a51a 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -91,8 +91,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     handle_video_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview(content_type, conn, _media_proxy_url) do
-    send_resp(conn, :unprocessable_entity, "Unsupported content type: #{content_type}.")
+  defp handle_preview(_unsupported_content_type, conn, media_proxy_url) do
+    fallback_on_preview_error(conn, media_proxy_url)
   end
 
   defp handle_png_preview(conn, media_proxy_url) do
@@ -114,7 +114,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
-        send_resp(conn, :failed_dependency, "Can't handle preview.")
+        fallback_on_preview_error(conn, media_proxy_url)
     end
   end
 
@@ -132,7 +132,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
-        send_resp(conn, :failed_dependency, "Can't handle preview.")
+        fallback_on_preview_error(conn, media_proxy_url)
     end
   end
 
@@ -144,10 +144,14 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
       |> send_resp(200, thumbnail_binary)
     else
       _ ->
-        send_resp(conn, :failed_dependency, "Can't handle preview.")
+        fallback_on_preview_error(conn, media_proxy_url)
     end
   end
 
+  defp fallback_on_preview_error(conn, media_proxy_url) do
+    redirect(conn, external: media_proxy_url)
+  end
+
   defp put_preview_response_headers(
          conn,
          [content_type, filename] = _content_info \\ ["image/jpeg", "preview.jpg"]
-- 
cgit v1.2.3


From cd234a5321b9d33146b90be95d84fa67aa4f7707 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 12 Sep 2020 11:20:41 +0300
Subject: [#2497] Media preview proxy: preview bypass for small images (basing
 on Content-Length and Content-Type).

---
 .../web/media_proxy/media_proxy_controller.ex      | 25 ++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 08d62a51a..78df7763e 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -11,6 +11,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   alias Pleroma.Web.MediaProxy
   alias Plug.Conn
 
+  @min_content_length_for_preview 100 * 1024
+
   def remote(conn, %{"sig" => sig64, "url" => url64}) do
     with {_, true} <- {:enabled, MediaProxy.enabled?()},
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
@@ -54,8 +56,12 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     with {:ok, %{status: status} = head_response} when status in 200..299 <-
            Pleroma.HTTP.request("head", media_proxy_url, [], [], pool: :media) do
       content_type = Tesla.get_header(head_response, "content-type")
-      handle_preview(content_type, conn, media_proxy_url)
+      content_length = Tesla.get_header(head_response, "content-length")
+      content_length = content_length && String.to_integer(content_length)
+
+      handle_preview(content_type, content_length, conn, media_proxy_url)
     else
+      # If HEAD failed, redirecting to media proxy URI doesn't make much sense; returning an error
       {_, %{status: status}} ->
         send_resp(conn, :failed_dependency, "Can't fetch HTTP headers (HTTP #{status}).")
 
@@ -69,29 +75,36 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   defp handle_preview(
          "image/" <> _ = _content_type,
+         _content_length,
          %{params: %{"output_format" => "jpeg"}} = conn,
          media_proxy_url
        ) do
     handle_jpeg_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview("image/gif" = _content_type, conn, media_proxy_url) do
+  defp handle_preview("image/gif" = _content_type, _content_length, conn, media_proxy_url) do
+    redirect(conn, external: media_proxy_url)
+  end
+
+  defp handle_preview("image/" <> _ = _content_type, content_length, conn, media_proxy_url)
+       when is_integer(content_length) and content_length > 0 and
+              content_length < @min_content_length_for_preview do
     redirect(conn, external: media_proxy_url)
   end
 
-  defp handle_preview("image/png" <> _ = _content_type, conn, media_proxy_url) do
+  defp handle_preview("image/png" <> _ = _content_type, _content_length, conn, media_proxy_url) do
     handle_png_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview("image/" <> _ = _content_type, conn, media_proxy_url) do
+  defp handle_preview("image/" <> _ = _content_type, _content_length, conn, media_proxy_url) do
     handle_jpeg_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview("video/" <> _ = _content_type, conn, media_proxy_url) do
+  defp handle_preview("video/" <> _ = _content_type, _content_length, conn, media_proxy_url) do
     handle_video_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview(_unsupported_content_type, conn, media_proxy_url) do
+  defp handle_preview(_unsupported_content_type, _content_length, conn, media_proxy_url) do
     fallback_on_preview_error(conn, media_proxy_url)
   end
 
-- 
cgit v1.2.3


From 3e53ab4e98e6294f593f2185998f555ccd6fee73 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Mon, 14 Sep 2020 14:08:12 +0300
Subject: added notification constraints

---
 lib/pleroma/migration_helper/notification_backfill.ex | 15 +++++++--------
 lib/pleroma/repo.ex                                   |  9 +++++++--
 2 files changed, 14 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex
index d260e62ca..24f4733fe 100644
--- a/lib/pleroma/migration_helper/notification_backfill.ex
+++ b/lib/pleroma/migration_helper/notification_backfill.ex
@@ -19,13 +19,13 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
     query
     |> Repo.chunk_stream(100)
     |> Enum.each(fn notification ->
-      type =
-        notification.activity
-        |> type_from_activity()
+      if notification.activity do
+        type = type_from_activity(notification.activity)
 
-      notification
-      |> Ecto.Changeset.change(%{type: type})
-      |> Repo.update()
+        notification
+        |> Ecto.Changeset.change(%{type: type})
+        |> Repo.update()
+      end
     end)
   end
 
@@ -72,8 +72,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
         "pleroma:emoji_reaction"
 
       "Create" ->
-        activity
-        |> type_from_activity_object()
+        type_from_activity_object(activity)
 
       t ->
         raise "No notification type for activity type #{t}"
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index f317e4d58..a75610879 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -49,7 +49,7 @@ defmodule Pleroma.Repo do
     end
   end
 
-  def chunk_stream(query, chunk_size) do
+  def chunk_stream(query, chunk_size, returns_as \\ :one) do
     # We don't actually need start and end funcitons of resource streaming,
     # but it seems to be the only way to not fetch records one-by-one and
     # have individual records be the elements of the stream, instead of
@@ -69,7 +69,12 @@ defmodule Pleroma.Repo do
 
             records ->
               last_id = List.last(records).id
-              {records, last_id}
+
+              if returns_as == :one do
+                {records, last_id}
+              else
+                {[records], last_id}
+              end
           end
       end,
       fn _ -> :ok end
-- 
cgit v1.2.3


From f66a15c4a51e1c8f614b4c1609b2385a29762931 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 14 Sep 2020 14:44:25 +0300
Subject: RichMedia parser: do not set a cache TTL for unchanging errors

---
 lib/pleroma/web/rich_media/parser.ex | 8 ++++++++
 1 file changed, 8 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 5727fda18..ab8f35922 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -36,6 +36,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
         {:ok, _data} = res ->
           res
 
+        {:error, :body_too_large} = e ->
+          e
+
+        {:error, {:content_type, _}} ->
+          e
+
+        # The TTL is not set for the errors above, since they are unlikely to change
+        # with time
         {:error, _} = e ->
           ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
           Cachex.expire(:rich_media_cache, url, ttl)
-- 
cgit v1.2.3


From f70335002df9b2b3f47f0ccaed6aaeebfb14435f Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 14 Sep 2020 14:45:58 +0300
Subject: RichMedia: Do a HEAD request to check content type/length

This shouldn't be too expensive, since the connections are pooled,
but it should save us some bandwidth since we won't fetch non-html
files and files that are too large for us to process (especially
since you can't cancel a request without closing the connection
with HTTP1).
---
 lib/pleroma/web/rich_media/helpers.ex | 46 ++++++++++++++++++++++++++++++++++-
 1 file changed, 45 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index bd7f03cbe..d7a19df4a 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -87,6 +87,50 @@ defmodule Pleroma.Web.RichMedia.Helpers do
   def rich_media_get(url) do
     headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
 
-    Pleroma.HTTP.get(url, headers, @options)
+    head_check =
+      case Pleroma.HTTP.head(url, headers, @options) do
+        # If the HEAD request didn't reach the server for whatever reason,
+        # we assume the GET that comes right after won't either
+        {:error, _} = e ->
+          e
+
+        {:ok, %Tesla.Env{status: 200, headers: headers}} ->
+          with :ok <- check_content_type(headers),
+               :ok <- check_content_length(headers),
+               do: :ok
+
+        _ ->
+          :ok
+      end
+
+    with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options)
+  end
+
+  defp check_content_type(headers) do
+    case List.keyfind(headers, "content-type", 0) do
+      {_, content_type} ->
+        case Plug.Conn.Utils.media_type(content_type) do
+          {:ok, "text", "html", _} -> :ok
+          _ -> {:error, {:content_type, content_type}}
+        end
+
+      _ ->
+        :ok
+    end
+  end
+
+  @max_body @options[:max_body]
+  defp check_content_length(headers) do
+    case List.keyfind(headers, "content-length", 0) do
+      {_, maybe_content_length} ->
+        case Integer.parse(maybe_content_length) do
+          {content_length, ""} when content_length <= @max_body -> :ok
+          {_, ""} -> {:error, :body_too_large}
+          _ -> :ok
+        end
+
+      _ ->
+        :ok
+    end
   end
 end
-- 
cgit v1.2.3


From bb407edce4b512aae74c12ea0c1abcc92bc18ddb Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Mon, 14 Sep 2020 15:46:00 +0300
Subject: RichMedia: fix a compilation error due to nonexistent variable

No idea why this passed Gitlab CI
---
 lib/pleroma/web/rich_media/parser.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index ab8f35922..33f6f1fa1 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -39,7 +39,7 @@ defmodule Pleroma.Web.RichMedia.Parser do
         {:error, :body_too_large} = e ->
           e
 
-        {:error, {:content_type, _}} ->
+        {:error, {:content_type, _}} = e ->
           e
 
         # The TTL is not set for the errors above, since they are unlikely to change
-- 
cgit v1.2.3


From 38b2db297b3207607072347b408dc7eacbac600e Mon Sep 17 00:00:00 2001
From: stwf <steven.fuchs@dockyard.com>
Date: Mon, 14 Sep 2020 13:18:11 -0400
Subject: search indexing metadata respects discoverable flag

---
 lib/pleroma/web/metadata/restrict_indexing.ex | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/metadata/restrict_indexing.ex b/lib/pleroma/web/metadata/restrict_indexing.ex
index f15607896..a1dcb6e15 100644
--- a/lib/pleroma/web/metadata/restrict_indexing.ex
+++ b/lib/pleroma/web/metadata/restrict_indexing.ex
@@ -10,7 +10,9 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
   """
 
   @impl true
-  def build_tags(%{user: %{local: false}}) do
+  def build_tags(%{user: %{local: true, discoverable: true}}), do: []
+
+  def build_tags(_) do
     [
       {:meta,
        [
@@ -19,7 +21,4 @@ defmodule Pleroma.Web.Metadata.Providers.RestrictIndexing do
        ], []}
     ]
   end
-
-  @impl true
-  def build_tags(%{user: %{local: true}}), do: []
 end
-- 
cgit v1.2.3


From 709723182d69e1bb41a23c8abeb5d7c2c67b8c49 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 14 Sep 2020 17:06:42 -0500
Subject: Ensure SimplePolicy's tags in string representation don't trip up the
 object validator

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index af4384213..8fe430644 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -309,7 +309,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do
     emoji =
       tags
-      |> Enum.filter(fn data -> data["type"] == "Emoji" and data["icon"] end)
+      |> Enum.filter(fn data -> is_map(data) and data["type"] == "Emoji" and data["icon"] end)
       |> Enum.reduce(%{}, fn data, mapping ->
         name = String.trim(data["name"], ":")
 
-- 
cgit v1.2.3


From 3ab59a6f3c7b7bae2e69d1a8d1bf484d039a5420 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Tue, 15 Sep 2020 13:00:07 +0300
Subject: Mastodon API: fix the public timeline returning an error when the
 `reply_visibility` parameter is set to `self` for an unauthenticated user

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 66a9f78a3..5aac3f53b 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -767,7 +767,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp restrict_replies(query, %{
-         reply_filtering_user: user,
+         reply_filtering_user: %User{} = user,
          reply_visibility: "self"
        }) do
     from(
@@ -783,7 +783,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   defp restrict_replies(query, %{
-         reply_filtering_user: user,
+         reply_filtering_user: %User{} = user,
          reply_visibility: "following"
        }) do
     from(
-- 
cgit v1.2.3


From 599f8bb152ca0669d17baa5f313f00f0791209b6 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Wed, 16 Sep 2020 09:47:18 +0300
Subject: RepoStreamer.chunk_stream ->  Repo.chunk_stream

---
 lib/mix/tasks/pleroma/database.ex |  4 ++--
 lib/mix/tasks/pleroma/user.ex     |  4 ++--
 lib/pleroma/repo.ex               | 14 ++++++++++++++
 lib/pleroma/repo_streamer.ex      | 34 ----------------------------------
 lib/pleroma/user.ex               |  3 +--
 5 files changed, 19 insertions(+), 40 deletions(-)
 delete mode 100644 lib/pleroma/repo_streamer.ex

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 7f1108dcf..a01c36ece 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -99,7 +99,7 @@ defmodule Mix.Tasks.Pleroma.Database do
       where: fragment("(?)->>'likes' is not null", object.data),
       select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
     )
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn objects ->
       ids =
         objects
@@ -145,7 +145,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     |> where(local: true)
     |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
     |> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
-    |> Pleroma.RepoStreamer.chunk_stream(100)
+    |> Pleroma.Repo.chunk_stream(100, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity ->
         expires_at =
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 01824aa18..b20c49d89 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -179,7 +179,7 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     Pleroma.User.Query.build(%{nickname: "@#{instance}"})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
@@ -370,7 +370,7 @@ defmodule Mix.Tasks.Pleroma.User do
     start_pleroma()
 
     Pleroma.User.Query.build(%{local: true})
-    |> Pleroma.RepoStreamer.chunk_stream(500)
+    |> Pleroma.Repo.chunk_stream(500, :batches)
     |> Stream.each(fn users ->
       users
       |> Enum.each(fn user ->
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index a75610879..4524bd5e2 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -49,6 +49,20 @@ defmodule Pleroma.Repo do
     end
   end
 
+  @doc """
+  Returns a lazy enumerable that emits all entries from the data store matching the given query.
+
+  `returns_as` use to group records. use the `batches` option to fetch records in bulk.
+
+  ## Examples
+
+  # fetch records one-by-one
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500)
+
+  # fetch records in bulk
+  iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
+  """
+  @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
   def chunk_stream(query, chunk_size, returns_as \\ :one) do
     # We don't actually need start and end funcitons of resource streaming,
     # but it seems to be the only way to not fetch records one-by-one and
diff --git a/lib/pleroma/repo_streamer.ex b/lib/pleroma/repo_streamer.ex
deleted file mode 100644
index cb4d7bb7a..000000000
--- a/lib/pleroma/repo_streamer.ex
+++ /dev/null
@@ -1,34 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.RepoStreamer do
-  alias Pleroma.Repo
-  import Ecto.Query
-
-  def chunk_stream(query, chunk_size) do
-    Stream.unfold(0, fn
-      :halt ->
-        {[], :halt}
-
-      last_id ->
-        query
-        |> order_by(asc: :id)
-        |> where([r], r.id > ^last_id)
-        |> limit(^chunk_size)
-        |> Repo.all()
-        |> case do
-          [] ->
-            {[], :halt}
-
-          records ->
-            last_id = List.last(records).id
-            {records, last_id}
-        end
-    end)
-    |> Stream.take_while(fn
-      [] -> false
-      _ -> true
-    end)
-  end
-end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index e73d19964..57497eb83 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -25,7 +25,6 @@ defmodule Pleroma.User do
   alias Pleroma.Object
   alias Pleroma.Registration
   alias Pleroma.Repo
-  alias Pleroma.RepoStreamer
   alias Pleroma.User
   alias Pleroma.UserRelationship
   alias Pleroma.Web
@@ -1775,7 +1774,7 @@ defmodule Pleroma.User do
   def delete_user_activities(%User{ap_id: ap_id} = user) do
     ap_id
     |> Activity.Queries.by_actor()
-    |> RepoStreamer.chunk_stream(50)
+    |> Repo.chunk_stream(50, :batches)
     |> Stream.each(fn activities ->
       Enum.each(activities, fn activity -> delete_activity(activity, user) end)
     end)
-- 
cgit v1.2.3


From adb1b0282dfbced2b2986c90cff765be37dd5151 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 16 Sep 2020 17:23:05 +0300
Subject: ConnectionPool Worker: use monitor flush instead of checking ref

`:flush` removes the DOWN message if one had arrived, so this check
should no longer be necessary.
---
 lib/pleroma/gun/connection_pool/worker.ex | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index 49d41e4c7..bf57e9e5f 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -93,25 +93,18 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
       end)
 
     {ref, state} = pop_in(state.client_monitors[client_pid])
-    # DOWN message can receive right after `remove_client` call and cause worker to terminate
-    state =
-      if is_nil(ref) do
-        state
-      else
-        Process.demonitor(ref)
 
-        timer =
-          if used_by == [] do
-            max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
-            Process.send_after(self(), :idle_close, max_idle)
-          else
-            nil
-          end
+    Process.demonitor(ref, [:flush])
 
-        %{state | timer: timer}
+    timer =
+      if used_by == [] do
+        max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
+        Process.send_after(self(), :idle_close, max_idle)
+      else
+        nil
       end
 
-    {:reply, :ok, state, :hibernate}
+    {:reply, :ok, %{state | timer: timer}, :hibernate}
   end
 
   @impl true
-- 
cgit v1.2.3


From 7a88b726bf81e1610ade2b07ffd6af672b701600 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Wed, 16 Sep 2020 17:29:16 +0200
Subject: User: Remote users don't need to be confirmed or approved

---
 lib/pleroma/user.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 57497eb83..1ffe60dfc 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -275,9 +275,9 @@ defmodule Pleroma.User do
   @spec account_status(User.t()) :: account_status()
   def account_status(%User{deactivated: true}), do: :deactivated
   def account_status(%User{password_reset_pending: true}), do: :password_reset_pending
-  def account_status(%User{approval_pending: true}), do: :approval_pending
+  def account_status(%User{local: true, approval_pending: true}), do: :approval_pending
 
-  def account_status(%User{confirmation_pending: true}) do
+  def account_status(%User{local: true, confirmation_pending: true}) do
     if Config.get([:instance, :account_activation_required]) do
       :confirmation_pending
     else
-- 
cgit v1.2.3


From a781f41f969bd1a929005b2b5006a40d42855ae8 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 16 Sep 2020 22:30:42 +0300
Subject: [#2497] Media preview proxy: misc. improvements (`static` param
 support, dynamic fifo pipe path), refactoring.

---
 lib/pleroma/helpers/media_helper.ex                   |  4 +---
 lib/pleroma/helpers/uri_helper.ex                     | 13 ++++++++-----
 lib/pleroma/web/mastodon_api/views/account_view.ex    |  4 ++--
 lib/pleroma/web/media_proxy/media_proxy.ex            |  2 +-
 lib/pleroma/web/media_proxy/media_proxy_controller.ex | 19 ++++++++++++++++---
 lib/pleroma/web/oauth/oauth_controller.ex             |  4 ++--
 6 files changed, 30 insertions(+), 16 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index d834b4a07..9b7348ee2 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -9,8 +9,6 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   alias Pleroma.HTTP
 
-  @tmp_base "/tmp/pleroma-media_preview-pipe"
-
   def image_resize(url, options) do
     with executable when is_binary(executable) <- System.find_executable("convert"),
          {:ok, args} <- prepare_image_resize_args(options),
@@ -103,7 +101,7 @@ defmodule Pleroma.Helpers.MediaHelper do
   end
 
   defp mkfifo do
-    path = "#{@tmp_base}#{to_charlist(:erlang.phash2(self()))}"
+    path = Path.join(System.tmp_dir!(), "pleroma-media-preview-pipe-#{Ecto.UUID.generate()}")
 
     case System.cmd("mkfifo", [path]) do
       {_, 0} ->
diff --git a/lib/pleroma/helpers/uri_helper.ex b/lib/pleroma/helpers/uri_helper.ex
index 9c9e53447..f1301f055 100644
--- a/lib/pleroma/helpers/uri_helper.ex
+++ b/lib/pleroma/helpers/uri_helper.ex
@@ -3,14 +3,17 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Helpers.UriHelper do
-  def append_uri_params(uri, appended_params) do
+  def modify_uri_params(uri, overridden_params, deleted_params \\ []) do
     uri = URI.parse(uri)
-    appended_params = for {k, v} <- appended_params, into: %{}, do: {to_string(k), v}
-    existing_params = URI.query_decoder(uri.query || "") |> Enum.into(%{})
-    updated_params_keys = Enum.uniq(Map.keys(existing_params) ++ Map.keys(appended_params))
+
+    existing_params = URI.query_decoder(uri.query || "") |> Map.new()
+    overridden_params = Map.new(overridden_params, fn {k, v} -> {to_string(k), v} end)
+    deleted_params = Enum.map(deleted_params, &to_string/1)
 
     updated_params =
-      for k <- updated_params_keys, do: {k, appended_params[k] || existing_params[k]}
+      existing_params
+      |> Map.merge(overridden_params)
+      |> Map.drop(deleted_params)
 
     uri
     |> Map.put(:query, URI.encode_query(updated_params))
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index a811f81c2..121ba1693 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -182,9 +182,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
     display_name = user.name || user.nickname
 
     avatar = User.avatar_url(user) |> MediaProxy.url()
-    avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(output_format: "jpeg")
+    avatar_static = User.avatar_url(user) |> MediaProxy.preview_url(static: true)
     header = User.banner_url(user) |> MediaProxy.url()
-    header_static = User.banner_url(user) |> MediaProxy.preview_url(output_format: "jpeg")
+    header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true)
 
     following_count =
       if !user.hide_follows_count or !user.hide_follows or opts[:for] == user do
diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex
index ba553998b..8656b8cad 100644
--- a/lib/pleroma/web/media_proxy/media_proxy.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy.ex
@@ -157,7 +157,7 @@ defmodule Pleroma.Web.MediaProxy do
   def build_preview_url(sig_base64, url_base64, filename \\ nil, preview_params \\ []) do
     uri = proxy_url("proxy/preview", sig_base64, url_base64, filename)
 
-    UriHelper.append_uri_params(uri, preview_params)
+    UriHelper.modify_uri_params(uri, preview_params)
   end
 
   def verify_request_path_and_url(
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index 78df7763e..fe279e964 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   alias Pleroma.Config
   alias Pleroma.Helpers.MediaHelper
+  alias Pleroma.Helpers.UriHelper
   alias Pleroma.ReverseProxy
   alias Pleroma.Web.MediaProxy
   alias Plug.Conn
@@ -74,14 +75,26 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   end
 
   defp handle_preview(
-         "image/" <> _ = _content_type,
+         "image/gif" = _content_type,
          _content_length,
-         %{params: %{"output_format" => "jpeg"}} = conn,
+         %{params: %{"static" => static}} = conn,
          media_proxy_url
-       ) do
+       )
+       when static in ["true", true] do
     handle_jpeg_preview(conn, media_proxy_url)
   end
 
+  defp handle_preview(
+         _content_type,
+         _content_length,
+         %{params: %{"static" => static}} = conn,
+         _media_proxy_url
+       )
+       when static in ["true", true] do
+    uri_without_static_param = UriHelper.modify_uri_params(current_url(conn), %{}, ["static"])
+    redirect(conn, external: uri_without_static_param)
+  end
+
   defp handle_preview("image/gif" = _content_type, _content_length, conn, media_proxy_url) do
     redirect(conn, external: media_proxy_url)
   end
diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex
index 26e68be42..a4152e840 100644
--- a/lib/pleroma/web/oauth/oauth_controller.ex
+++ b/lib/pleroma/web/oauth/oauth_controller.ex
@@ -119,7 +119,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
       redirect_uri = redirect_uri(conn, redirect_uri)
       url_params = %{access_token: token.token}
       url_params = Maps.put_if_present(url_params, :state, params["state"])
-      url = UriHelper.append_uri_params(redirect_uri, url_params)
+      url = UriHelper.modify_uri_params(redirect_uri, url_params)
       redirect(conn, external: url)
     else
       conn
@@ -161,7 +161,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do
       redirect_uri = redirect_uri(conn, redirect_uri)
       url_params = %{code: auth.token}
       url_params = Maps.put_if_present(url_params, :state, auth_attrs["state"])
-      url = UriHelper.append_uri_params(redirect_uri, url_params)
+      url = UriHelper.modify_uri_params(redirect_uri, url_params)
       redirect(conn, external: url)
     else
       conn
-- 
cgit v1.2.3


From 5a8ea0a5b07c22d567a60af36345483fe880b638 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Thu, 17 Sep 2020 09:13:43 +0300
Subject: small refactoring

---
 lib/pleroma/user/import.ex               | 10 ++--------
 lib/pleroma/workers/background_worker.ex | 29 +++--------------------------
 2 files changed, 5 insertions(+), 34 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user/import.ex b/lib/pleroma/user/import.ex
index de27bdc4c..e458021c8 100644
--- a/lib/pleroma/user/import.ex
+++ b/lib/pleroma/user/import.ex
@@ -65,20 +65,14 @@ defmodule Pleroma.User.Import do
   def blocks_import(%User{} = blocker, [_ | _] = identifiers) do
     BackgroundWorker.enqueue(
       "blocks_import",
-      %{
-        "blocker_id" => blocker.id,
-        "blocked_identifiers" => identifiers
-      }
+      %{"user_id" => blocker.id, "identifiers" => identifiers}
     )
   end
 
   def follow_import(%User{} = follower, [_ | _] = identifiers) do
     BackgroundWorker.enqueue(
       "follow_import",
-      %{
-        "follower_id" => follower.id,
-        "followed_identifiers" => identifiers
-      }
+      %{"user_id" => follower.id, "identifiers" => identifiers}
     )
   end
 
diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex
index f9c767ee0..55b5a13d9 100644
--- a/lib/pleroma/workers/background_worker.ex
+++ b/lib/pleroma/workers/background_worker.ex
@@ -26,33 +26,10 @@ defmodule Pleroma.Workers.BackgroundWorker do
     User.perform(:force_password_reset, user)
   end
 
-  def perform(%Job{
-        args: %{
-          "op" => "blocks_import",
-          "blocker_id" => blocker_id,
-          "blocked_identifiers" => blocked_identifiers
-        }
-      }) do
-    blocker = User.get_cached_by_id(blocker_id)
-    {:ok, User.Import.perform(:blocks_import, blocker, blocked_identifiers)}
-  end
-
-  def perform(%Job{
-        args: %{
-          "op" => "follow_import",
-          "follower_id" => follower_id,
-          "followed_identifiers" => followed_identifiers
-        }
-      }) do
-    follower = User.get_cached_by_id(follower_id)
-    {:ok, User.Import.perform(:follow_import, follower, followed_identifiers)}
-  end
-
-  def perform(%Job{
-        args: %{"op" => "mutes_import", "user_id" => user_id, "identifiers" => identifiers}
-      }) do
+  def perform(%Job{args: %{"op" => op, "user_id" => user_id, "identifiers" => identifiers}})
+      when op in ["blocks_import", "follow_import", "mutes_import"] do
     user = User.get_cached_by_id(user_id)
-    {:ok, User.Import.perform(:mutes_import, user, identifiers)}
+    {:ok, User.Import.perform(String.to_atom(op), user, identifiers)}
   end
 
   def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do
-- 
cgit v1.2.3


From 582ad5d4e1587b3dba9d879bd68dd9a315c8446e Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Sun, 30 Aug 2020 15:15:14 +0300
Subject: AdminAPI: Allow to modify Terms of Service and Instance Panel via
 Admin API

---
 .../controllers/instance_document_controller.ex    |  37 +++++++
 .../admin/instance_document_operation.ex           | 108 +++++++++++++++++++++
 lib/pleroma/web/instance_document.ex               |  62 ++++++++++++
 lib/pleroma/web/router.ex                          |   4 +
 4 files changed, 211 insertions(+)
 create mode 100644 lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
 create mode 100644 lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
 create mode 100644 lib/pleroma/web/instance_document.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
new file mode 100644
index 000000000..2144e44ac
--- /dev/null
+++ b/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
@@ -0,0 +1,37 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
+  use Pleroma.Web, :controller
+
+  alias Pleroma.Plugs.OAuthScopesPlug
+  alias Pleroma.Web.InstanceDocument
+
+  plug(Pleroma.Web.ApiSpec.CastAndValidate)
+
+  action_fallback(Pleroma.Web.AdminAPI.FallbackController)
+
+  defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation
+
+  plug(OAuthScopesPlug, %{scopes: ["read"], admin: true} when action == :show)
+  plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action in [:update, :delete])
+
+  def show(conn, %{name: document_name}) do
+    with {:ok, url} <- InstanceDocument.get(document_name) do
+      json(conn, %{"url" => url})
+    end
+  end
+
+  def update(%{body_params: %{file: file}} = conn, %{name: document_name}) do
+    with {:ok, url} <- InstanceDocument.put(document_name, file.path) do
+      json(conn, %{"url" => url})
+    end
+  end
+
+  def delete(conn, %{name: document_name}) do
+    with :ok <- InstanceDocument.delete(document_name) do
+      json(conn, %{})
+    end
+  end
+end
diff --git a/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex b/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
new file mode 100644
index 000000000..e0eb993fb
--- /dev/null
+++ b/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
@@ -0,0 +1,108 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation do
+  alias OpenApiSpex.Operation
+  alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Helpers
+  alias Pleroma.Web.ApiSpec.Schemas.ApiError
+
+  def open_api_operation(action) do
+    operation = String.to_existing_atom("#{action}_operation")
+    apply(__MODULE__, operation, [])
+  end
+
+  def show_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Get the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.show",
+      security: [%{"oAuth" => ["read"]}],
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  def update_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Update the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.update",
+      security: [%{"oAuth" => ["write"]}],
+      requestBody: Helpers.request_body("Parameters", update_request()),
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp update_request do
+    %Schema{
+      title: "UpdateRequest",
+      description: "POST body for uploading the file",
+      type: :object,
+      required: [:file],
+      properties: %{
+        file: %Schema{
+          type: :string,
+          format: :binary,
+          description: "The file to be uploaded, using multipart form data."
+        }
+      }
+    }
+  end
+
+  def delete_operation do
+    %Operation{
+      tags: ["Admin", "InstanceDocument"],
+      summary: "Get the instance document",
+      operationId: "AdminAPI.InstanceDocumentController.delete",
+      security: [%{"oAuth" => ["write"]}],
+      parameters: [
+        Operation.parameter(:name, :path, %Schema{type: :string}, "The document name",
+          required: true
+        )
+        | Helpers.admin_api_params()
+      ],
+      responses: %{
+        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        403 => Operation.response("Forbidden", "application/json", ApiError),
+        404 => Operation.response("Not Found", "application/json", ApiError)
+      }
+    }
+  end
+
+  defp instance_document do
+    %Schema{
+      title: "InstanceDocument",
+      type: :object,
+      properties: %{
+        url: %Schema{type: :string}
+      },
+      example: %{
+        "url" => "https://example.com/static/terms-of-service.html"
+      }
+    }
+  end
+end
diff --git a/lib/pleroma/web/instance_document.ex b/lib/pleroma/web/instance_document.ex
new file mode 100644
index 000000000..969a44e41
--- /dev/null
+++ b/lib/pleroma/web/instance_document.ex
@@ -0,0 +1,62 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.InstanceDocument do
+  alias Pleroma.Config
+  alias Pleroma.Web.Endpoint
+
+  @instance_documents %{
+    "terms-of-service" => "/static/terms-of-service.html",
+    "instance-panel" => "/instance/panel.html"
+  }
+
+  @spec get(String.t()) :: {:ok, String.t()} | {:error, atom()}
+  def get(document_name) do
+    case Map.fetch(@instance_documents, document_name) do
+      {:ok, path} -> {:ok, Path.join(Endpoint.url(), path)}
+      _ -> {:error, :not_found}
+    end
+  end
+
+  @spec put(String.t(), String.t()) :: {:ok, String.t()} | {:error, atom()}
+  def put(document_name, origin_path) do
+    with {_, {:ok, destination_path}} <-
+           {:instance_document, Map.fetch(@instance_documents, document_name)},
+         :ok <- put_file(origin_path, destination_path) do
+      {:ok, Path.join(Endpoint.url(), destination_path)}
+    else
+      {:instance_document, :error} -> {:error, :not_found}
+      error -> error
+    end
+  end
+
+  @spec delete(String.t()) :: :ok | {:error, atom()}
+  def delete(document_name) do
+    with {_, {:ok, path}} <- {:instance_document, Map.fetch(@instance_documents, document_name)},
+         instance_static_dir_path <- instance_static_dir(path),
+         :ok <- File.rm(instance_static_dir_path) do
+      :ok
+    else
+      {:instance_document, :error} -> {:error, :not_found}
+      {:error, :enoent} -> {:error, :not_found}
+      error -> error
+    end
+  end
+
+  defp put_file(origin_path, destination_path) do
+    with destination <- instance_static_dir(destination_path),
+         {_, :ok} <- {:mkdir_p, File.mkdir_p(Path.dirname(destination))},
+         {_, {:ok, _}} <- {:copy, File.copy(origin_path, destination)} do
+      :ok
+    else
+      {error, _} -> {:error, error}
+    end
+  end
+
+  defp instance_static_dir(filename) do
+    [:instance, :static_dir]
+    |> Config.get!()
+    |> Path.join(filename)
+  end
+end
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index e4440d442..a4a58c2c4 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -182,6 +182,10 @@ defmodule Pleroma.Web.Router do
 
     get("/instances/:instance/statuses", AdminAPIController, :list_instance_statuses)
 
+    get("/instance_document/:name", InstanceDocumentController, :show)
+    patch("/instance_document/:name", InstanceDocumentController, :update)
+    delete("/instance_document/:name", InstanceDocumentController, :delete)
+
     patch("/users/confirm_email", AdminAPIController, :confirm_email)
     patch("/users/resend_confirmation_email", AdminAPIController, :resend_confirmation_email)
 
-- 
cgit v1.2.3


From c711a2b15761db9d2d30035e9fee0783f0bf77b0 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Thu, 17 Sep 2020 16:54:38 +0300
Subject: Return the file content for `GET
 /api/pleroma/admin/instance_document/:document_name`

---
 .../web/admin_api/controllers/instance_document_controller.ex    | 8 ++++++--
 .../web/api_spec/operations/admin/instance_document_operation.ex | 9 ++++++++-
 lib/pleroma/web/instance_document.ex                             | 2 +-
 3 files changed, 15 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex b/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
index 2144e44ac..504d9b517 100644
--- a/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/instance_document_controller.ex
@@ -5,6 +5,7 @@
 defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
   use Pleroma.Web, :controller
 
+  alias Pleroma.Plugs.InstanceStatic
   alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.Web.InstanceDocument
 
@@ -18,8 +19,11 @@ defmodule Pleroma.Web.AdminAPI.InstanceDocumentController do
   plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action in [:update, :delete])
 
   def show(conn, %{name: document_name}) do
-    with {:ok, url} <- InstanceDocument.get(document_name) do
-      json(conn, %{"url" => url})
+    with {:ok, url} <- InstanceDocument.get(document_name),
+         {:ok, content} <- File.read(InstanceStatic.file_path(url)) do
+      conn
+      |> put_resp_content_type("text/html")
+      |> send_resp(200, content)
     end
   end
 
diff --git a/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex b/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
index e0eb993fb..a120ff4e8 100644
--- a/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/admin/instance_document_operation.ex
@@ -26,7 +26,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation do
         | Helpers.admin_api_params()
       ],
       responses: %{
-        200 => Operation.response("InstanceDocument", "application/json", instance_document()),
+        200 => document_content(),
         400 => Operation.response("Bad Request", "application/json", ApiError),
         403 => Operation.response("Forbidden", "application/json", ApiError),
         404 => Operation.response("Not Found", "application/json", ApiError)
@@ -105,4 +105,11 @@ defmodule Pleroma.Web.ApiSpec.Admin.InstanceDocumentOperation do
       }
     }
   end
+
+  defp document_content do
+    Operation.response("InstanceDocumentContent", "text/html", %Schema{
+      type: :string,
+      example: "<h1>Instance panel</h1>"
+    })
+  end
 end
diff --git a/lib/pleroma/web/instance_document.ex b/lib/pleroma/web/instance_document.ex
index 969a44e41..df5caebf0 100644
--- a/lib/pleroma/web/instance_document.ex
+++ b/lib/pleroma/web/instance_document.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.InstanceDocument do
   @spec get(String.t()) :: {:ok, String.t()} | {:error, atom()}
   def get(document_name) do
     case Map.fetch(@instance_documents, document_name) do
-      {:ok, path} -> {:ok, Path.join(Endpoint.url(), path)}
+      {:ok, path} -> {:ok, path}
       _ -> {:error, :not_found}
     end
   end
-- 
cgit v1.2.3


From db80b9d630f9fc72ebc269cb24142501116c269a Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 17 Sep 2020 16:13:21 +0300
Subject: RichMedia: Fix log spam on failures and resetting TTL on cached
 errors

---
 lib/pleroma/web/rich_media/parser.ex | 73 ++++++++++++++++++++++++------------
 1 file changed, 49 insertions(+), 24 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 33f6f1fa1..c70d2fdba 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -20,36 +20,61 @@ defmodule Pleroma.Web.RichMedia.Parser do
       with {:ok, data} <- get_cached_or_parse(url),
            {:ok, _} <- set_ttl_based_on_image(data, url) do
         {:ok, data}
-      else
-        {:error, {:invalid_metadata, data}} = e ->
-          Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
-          e
-
-        error ->
-          Logger.error(fn -> "Rich media error for #{url}: #{inspect(error)}" end)
-          error
       end
     end
 
     defp get_cached_or_parse(url) do
-      case Cachex.fetch!(:rich_media_cache, url, fn _ -> {:commit, parse_url(url)} end) do
-        {:ok, _data} = res ->
-          res
-
-        {:error, :body_too_large} = e ->
-          e
-
-        {:error, {:content_type, _}} = e ->
-          e
-
-        # The TTL is not set for the errors above, since they are unlikely to change
-        # with time
-        {:error, _} = e ->
-          ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
-          Cachex.expire(:rich_media_cache, url, ttl)
-          e
+      case Cachex.fetch(:rich_media_cache, url, fn ->
+             case parse_url(url) do
+               {:ok, _} = res ->
+                 {:commit, res}
+
+               {:error, reason} = e ->
+                 # Unfortunately we have to log errors here, instead of doing that
+                 # along with ttl setting at the bottom. Otherwise we can get log spam
+                 # if more than one process was waiting for the rich media card
+                 # while it was generated. Ideally we would set ttl here as well,
+                 # so we don't override it number_of_waiters_on_generation
+                 # times, but one, obviously, can't set ttl for not-yet-created entry
+                 # and Cachex doesn't support returning ttl from the fetch callback.
+                 log_error(url, reason)
+                 {:commit, e}
+             end
+           end) do
+        {action, res} when action in [:commit, :ok] ->
+          case res do
+            {:ok, _data} = res ->
+              res
+
+            {:error, reason} = e ->
+              if action == :commit, do: set_error_ttl(url, reason)
+              e
+          end
+
+        {:error, e} ->
+          {:error, {:cachex_error, e}}
       end
     end
+
+    defp set_error_ttl(_url, :body_too_large), do: :ok
+    defp set_error_ttl(_url, {:content_type, _}), do: :ok
+
+    # The TTL is not set for the errors above, since they are unlikely to change
+    # with time
+
+    defp set_error_ttl(url, _reason) do
+      ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
+      Cachex.expire(:rich_media_cache, url, ttl)
+      :ok
+    end
+
+    defp log_error(url, {:invalid_metadata, data}) do
+      Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end)
+    end
+
+    defp log_error(url, reason) do
+      Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end)
+    end
   end
 
   @doc """
-- 
cgit v1.2.3


From 7cdbd91d83c02a79c22783ca489ef82e82b31a51 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 17 Sep 2020 17:13:40 +0300
Subject: [#2497] Configurability of :min_content_length (preview proxy).
 Refactoring, documentation, tests.

---
 lib/pleroma/helpers/media_helper.ex                |  1 +
 .../web/media_proxy/media_proxy_controller.ex      | 90 +++++++++++-----------
 2 files changed, 47 insertions(+), 44 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex
index 9b7348ee2..b6f35a24b 100644
--- a/lib/pleroma/helpers/media_helper.ex
+++ b/lib/pleroma/helpers/media_helper.ex
@@ -58,6 +58,7 @@ defmodule Pleroma.Helpers.MediaHelper do
 
   defp prepare_image_resize_args(_), do: {:error, :missing_options}
 
+  # Note: video thumbnail is intentionally not resized (always has original dimensions)
   def video_framegrab(url) do
     with executable when is_binary(executable) <- System.find_executable("ffmpeg"),
          {:ok, env} <- HTTP.get(url, [], pool: :media),
diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
index fe279e964..90651ed9b 100644
--- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex
+++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex
@@ -12,8 +12,6 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   alias Pleroma.Web.MediaProxy
   alias Plug.Conn
 
-  @min_content_length_for_preview 100 * 1024
-
   def remote(conn, %{"sig" => sig64, "url" => url64}) do
     with {_, true} <- {:enabled, MediaProxy.enabled?()},
          {:ok, url} <- MediaProxy.decode_url(sig64, url64),
@@ -37,7 +35,8 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   def preview(%Conn{} = conn, %{"sig" => sig64, "url" => url64}) do
     with {_, true} <- {:enabled, MediaProxy.preview_enabled?()},
-         {:ok, url} <- MediaProxy.decode_url(sig64, url64) do
+         {:ok, url} <- MediaProxy.decode_url(sig64, url64),
+         :ok <- MediaProxy.verify_request_path_and_url(conn, url) do
       handle_preview(conn, url)
     else
       {:enabled, false} ->
@@ -59,8 +58,25 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
       content_type = Tesla.get_header(head_response, "content-type")
       content_length = Tesla.get_header(head_response, "content-length")
       content_length = content_length && String.to_integer(content_length)
+      static = conn.params["static"] in ["true", true]
+
+      cond do
+        static and content_type == "image/gif" ->
+          handle_jpeg_preview(conn, media_proxy_url)
 
-      handle_preview(content_type, content_length, conn, media_proxy_url)
+        static ->
+          drop_static_param_and_redirect(conn)
+
+        content_type == "image/gif" ->
+          redirect(conn, external: media_proxy_url)
+
+        min_content_length_for_preview() > 0 and content_length > 0 and
+            content_length < min_content_length_for_preview() ->
+          redirect(conn, external: media_proxy_url)
+
+        true ->
+          handle_preview(content_type, conn, media_proxy_url)
+      end
     else
       # If HEAD failed, redirecting to media proxy URI doesn't make much sense; returning an error
       {_, %{status: status}} ->
@@ -74,58 +90,27 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
-  defp handle_preview(
-         "image/gif" = _content_type,
-         _content_length,
-         %{params: %{"static" => static}} = conn,
-         media_proxy_url
-       )
-       when static in ["true", true] do
-    handle_jpeg_preview(conn, media_proxy_url)
-  end
-
-  defp handle_preview(
-         _content_type,
-         _content_length,
-         %{params: %{"static" => static}} = conn,
-         _media_proxy_url
-       )
-       when static in ["true", true] do
-    uri_without_static_param = UriHelper.modify_uri_params(current_url(conn), %{}, ["static"])
-    redirect(conn, external: uri_without_static_param)
-  end
-
-  defp handle_preview("image/gif" = _content_type, _content_length, conn, media_proxy_url) do
-    redirect(conn, external: media_proxy_url)
-  end
-
-  defp handle_preview("image/" <> _ = _content_type, content_length, conn, media_proxy_url)
-       when is_integer(content_length) and content_length > 0 and
-              content_length < @min_content_length_for_preview do
-    redirect(conn, external: media_proxy_url)
-  end
-
-  defp handle_preview("image/png" <> _ = _content_type, _content_length, conn, media_proxy_url) do
+  defp handle_preview("image/png" <> _ = _content_type, conn, media_proxy_url) do
     handle_png_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview("image/" <> _ = _content_type, _content_length, conn, media_proxy_url) do
+  defp handle_preview("image/" <> _ = _content_type, conn, media_proxy_url) do
     handle_jpeg_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview("video/" <> _ = _content_type, _content_length, conn, media_proxy_url) do
+  defp handle_preview("video/" <> _ = _content_type, conn, media_proxy_url) do
     handle_video_preview(conn, media_proxy_url)
   end
 
-  defp handle_preview(_unsupported_content_type, _content_length, conn, media_proxy_url) do
+  defp handle_preview(_unsupported_content_type, conn, media_proxy_url) do
     fallback_on_preview_error(conn, media_proxy_url)
   end
 
   defp handle_png_preview(conn, media_proxy_url) do
     quality = Config.get!([:media_preview_proxy, :image_quality])
+    {thumbnail_max_width, thumbnail_max_height} = thumbnail_max_dimensions()
 
-    with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(),
-         {:ok, thumbnail_binary} <-
+    with {:ok, thumbnail_binary} <-
            MediaHelper.image_resize(
              media_proxy_url,
              %{
@@ -146,9 +131,9 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
 
   defp handle_jpeg_preview(conn, media_proxy_url) do
     quality = Config.get!([:media_preview_proxy, :image_quality])
+    {thumbnail_max_width, thumbnail_max_height} = thumbnail_max_dimensions()
 
-    with {thumbnail_max_width, thumbnail_max_height} <- thumbnail_max_dimensions(),
-         {:ok, thumbnail_binary} <-
+    with {:ok, thumbnail_binary} <-
            MediaHelper.image_resize(
              media_proxy_url,
              %{max_width: thumbnail_max_width, max_height: thumbnail_max_height, quality: quality}
@@ -174,6 +159,15 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     end
   end
 
+  defp drop_static_param_and_redirect(conn) do
+    uri_without_static_param =
+      conn
+      |> current_url()
+      |> UriHelper.modify_uri_params(%{}, ["static"])
+
+    redirect(conn, external: uri_without_static_param)
+  end
+
   defp fallback_on_preview_error(conn, media_proxy_url) do
     redirect(conn, external: media_proxy_url)
   end
@@ -189,7 +183,7 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
   end
 
   defp thumbnail_max_dimensions do
-    config = Config.get([:media_preview_proxy], [])
+    config = media_preview_proxy_config()
 
     thumbnail_max_width = Keyword.fetch!(config, :thumbnail_max_width)
     thumbnail_max_height = Keyword.fetch!(config, :thumbnail_max_height)
@@ -197,6 +191,14 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do
     {thumbnail_max_width, thumbnail_max_height}
   end
 
+  defp min_content_length_for_preview do
+    Keyword.get(media_preview_proxy_config(), :min_content_length, 0)
+  end
+
+  defp media_preview_proxy_config do
+    Config.get!([:media_preview_proxy])
+  end
+
   defp media_proxy_opts do
     Config.get([:media_proxy, :proxy_opts], [])
   end
-- 
cgit v1.2.3


From f7e40f7ef134a3030aa61114daa39810efb5889d Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 17 Sep 2020 09:32:50 -0500
Subject: Deny ConfigDB migration when deprecated settings found

---
 lib/mix/tasks/pleroma/config.ex            | 10 +++++--
 lib/pleroma/config/deprecation_warnings.ex | 43 ++++++++++++++++++++++++------
 2 files changed, 43 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index 904c5a74b..18f99318d 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -32,7 +32,8 @@ defmodule Mix.Tasks.Pleroma.Config do
 
   @spec migrate_to_db(Path.t() | nil) :: any()
   def migrate_to_db(file_path \\ nil) do
-    if Pleroma.Config.get([:configurable_from_database]) do
+    with true <- Pleroma.Config.get([:configurable_from_database]),
+         :ok <- Pleroma.Config.DeprecationWarnings.warn() do
       config_file =
         if file_path do
           file_path
@@ -46,7 +47,8 @@ defmodule Mix.Tasks.Pleroma.Config do
 
       do_migrate_to_db(config_file)
     else
-      migration_error()
+      :error -> deprecation_error()
+      _ -> migration_error()
     end
   end
 
@@ -120,6 +122,10 @@ defmodule Mix.Tasks.Pleroma.Config do
     )
   end
 
+  defp deprecation_error do
+    shell_error("Migration is not allowed until all deprecation warnings have been resolved.")
+  end
+
   if Code.ensure_loaded?(Config.Reader) do
     defp config_header, do: "import Config\r\n\r\n"
     defp read_file(config_file), do: Config.Reader.read_imports!(config_file)
diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex
index 412d55a77..98c4dc9c8 100644
--- a/lib/pleroma/config/deprecation_warnings.ex
+++ b/lib/pleroma/config/deprecation_warnings.ex
@@ -26,6 +26,10 @@ defmodule Pleroma.Config.DeprecationWarnings do
       !!!DEPRECATION WARNING!!!
       You are using the old configuration mechanism for the hellthread filter. Please check config.md.
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -47,17 +51,26 @@ defmodule Pleroma.Config.DeprecationWarnings do
 
       config :pleroma, :mrf_user_allowlist, #{inspect(rewritten, pretty: true)}
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
   def warn do
-    check_hellthread_threshold()
-    mrf_user_allowlist()
-    check_old_mrf_config()
-    check_media_proxy_whitelist_config()
-    check_welcome_message_config()
-    check_gun_pool_options()
-    check_activity_expiration_config()
+    with :ok <- check_hellthread_threshold(),
+         :ok <- mrf_user_allowlist(),
+         :ok <- check_old_mrf_config(),
+         :ok <- check_media_proxy_whitelist_config(),
+         :ok <- check_welcome_message_config(),
+         :ok <- check_gun_pool_options(),
+         :ok <- check_activity_expiration_config() do
+      :ok
+    else
+      _ ->
+        :error
+    end
   end
 
   def check_welcome_message_config do
@@ -74,6 +87,10 @@ defmodule Pleroma.Config.DeprecationWarnings do
       \n* `config :pleroma, :instance, welcome_user_nickname` is now `config :pleroma, :welcome, :direct_message, :sender_nickname`
       \n* `config :pleroma, :instance, welcome_message` is now `config :pleroma, :welcome, :direct_message, :message`
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -101,8 +118,11 @@ defmodule Pleroma.Config.DeprecationWarnings do
           end
       end)
 
-    if warning != "" do
+    if warning == "" do
+      :ok
+    else
       Logger.warn(warning_preface <> warning)
+      :error
     end
   end
 
@@ -115,6 +135,10 @@ defmodule Pleroma.Config.DeprecationWarnings do
       !!!DEPRECATION WARNING!!!
       Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later.
       """)
+
+      :error
+    else
+      :ok
     end
   end
 
@@ -157,6 +181,9 @@ defmodule Pleroma.Config.DeprecationWarnings do
       Logger.warn(Enum.join([warning_preface | pool_warnings]))
 
       Config.put(:pools, updated_config)
+      :error
+    else
+      :ok
     end
   end
 
-- 
cgit v1.2.3


From 41939e3175cf31884cb84acd136c303a84c77f8c Mon Sep 17 00:00:00 2001
From: stwf <steven.fuchs@dockyard.com>
Date: Mon, 14 Sep 2020 11:40:52 -0400
Subject: User search respect discoverable flag

---
 lib/pleroma/user/search.ex | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index 7babd47ea..b8c648672 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -52,6 +52,7 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_invisible_users()
+    |> filter_discoverable_users()
     |> filter_internal_users()
     |> filter_blocked_domains(for_user)
     |> fts_search(query_string)
@@ -122,6 +123,10 @@ defmodule Pleroma.User.Search do
     from(q in query, where: q.invisible == false)
   end
 
+  defp filter_discoverable_users(query) do
+    from(q in query, where: q.discoverable == true)
+  end
+
   defp filter_internal_users(query) do
     from(q in query, where: q.actor_type != "Application")
   end
-- 
cgit v1.2.3


From 3a0f99ed35a84145e713d4c640c50dc82c1b0dbb Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 14 Sep 2020 13:52:13 +0200
Subject: KeywordPolicy: Still match when fields are absent

---
 lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 67 +++++++++++-----------
 1 file changed, 32 insertions(+), 35 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
index 15e09dcf0..db66cfa3e 100644
--- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
@@ -20,9 +20,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     String.match?(string, pattern)
   end
 
-  defp check_reject(%{"object" => %{"content" => content, "summary" => summary}} = message) do
+  defp object_payload(%{} = object) do
+    [object["content"], object["summary"], object["name"]]
+    |> Enum.filter(& &1)
+    |> Enum.join("\n")
+  end
+
+  defp check_reject(%{"object" => %{} = object} = message) do
+    payload = object_payload(object)
+
     if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
-         string_matches?(content, pattern) or string_matches?(summary, pattern)
+         string_matches?(payload, pattern)
        end) do
       {:reject, "[KeywordPolicy] Matches with rejected keyword"}
     else
@@ -30,12 +38,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     end
   end
 
-  defp check_ftl_removal(
-         %{"to" => to, "object" => %{"content" => content, "summary" => summary}} = message
-       ) do
+  defp check_ftl_removal(%{"to" => to, "object" => %{} = object} = message) do
+    payload = object_payload(object)
+
     if Pleroma.Constants.as_public() in to and
          Enum.any?(Pleroma.Config.get([:mrf_keyword, :federated_timeline_removal]), fn pattern ->
-           string_matches?(content, pattern) or string_matches?(summary, pattern)
+           string_matches?(payload, pattern)
          end) do
       to = List.delete(to, Pleroma.Constants.as_public())
       cc = [Pleroma.Constants.as_public() | message["cc"] || []]
@@ -51,35 +59,24 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     end
   end
 
-  defp check_replace(%{"object" => %{"content" => content, "summary" => summary}} = message) do
-    content =
-      if is_binary(content) do
-        content
-      else
-        ""
-      end
-
-    summary =
-      if is_binary(summary) do
-        summary
-      else
-        ""
-      end
-
-    {content, summary} =
-      Enum.reduce(
-        Pleroma.Config.get([:mrf_keyword, :replace]),
-        {content, summary},
-        fn {pattern, replacement}, {content_acc, summary_acc} ->
-          {String.replace(content_acc, pattern, replacement),
-           String.replace(summary_acc, pattern, replacement)}
-        end
-      )
-
-    {:ok,
-     message
-     |> put_in(["object", "content"], content)
-     |> put_in(["object", "summary"], summary)}
+  defp check_replace(%{"object" => %{} = object} = message) do
+    object =
+      ["content", "name", "summary"]
+      |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end)
+      |> Enum.reduce(object, fn field, object ->
+        data =
+          Enum.reduce(
+            Pleroma.Config.get([:mrf_keyword, :replace]),
+            object[field],
+            fn {pat, repl}, acc -> String.replace(acc, pat, repl) end
+          )
+
+        Map.put(object, field, data)
+      end)
+
+    message = Map.put(message, "object", object)
+
+    {:ok, message}
   end
 
   @impl true
-- 
cgit v1.2.3


From abf25e5d5254edc88a65610bf5a0fd7e52f545c3 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Sat, 12 Sep 2020 12:05:36 +0200
Subject: Create MRF.filter_pipeline to inject :object_data when present

---
 lib/pleroma/web/activity_pub/mrf.ex                | 24 +++++++++++++++++++---
 .../web/activity_pub/mrf/subchain_policy.ex        |  3 +--
 lib/pleroma/web/activity_pub/pipeline.ex           |  8 ++++++--
 3 files changed, 28 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex
index 206d6af52..5e5361082 100644
--- a/lib/pleroma/web/activity_pub/mrf.ex
+++ b/lib/pleroma/web/activity_pub/mrf.ex
@@ -5,16 +5,34 @@
 defmodule Pleroma.Web.ActivityPub.MRF do
   @callback filter(Map.t()) :: {:ok | :reject, Map.t()}
 
-  def filter(policies, %{} = object) do
+  def filter(policies, %{} = message) do
     policies
-    |> Enum.reduce({:ok, object}, fn
-      policy, {:ok, object} -> policy.filter(object)
+    |> Enum.reduce({:ok, message}, fn
+      policy, {:ok, message} -> policy.filter(message)
       _, error -> error
     end)
   end
 
   def filter(%{} = object), do: get_policies() |> filter(object)
 
+  def pipeline_filter(%{} = message, meta) do
+    object = meta[:object_data]
+    ap_id = message["object"]
+
+    if object && ap_id do
+      with {:ok, message} <- filter(Map.put(message, "object", object)) do
+        meta = Keyword.put(meta, :object_data, message["object"])
+        {:ok, Map.put(message, "object", ap_id), meta}
+      else
+        {err, message} -> {err, message, meta}
+      end
+    else
+      {err, message} = filter(message)
+
+      {err, message, meta}
+    end
+  end
+
   def get_policies do
     Pleroma.Config.get([:mrf, :policies], []) |> get_policies()
   end
diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
index c9f20571f..048052da6 100644
--- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex
@@ -28,8 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do
         }"
       )
 
-      subchain
-      |> MRF.filter(message)
+      MRF.filter(subchain, message)
     else
       _e -> {:ok, message}
     end
diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex
index 36e325c37..2db86f116 100644
--- a/lib/pleroma/web/activity_pub/pipeline.ex
+++ b/lib/pleroma/web/activity_pub/pipeline.ex
@@ -26,13 +26,17 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
 
       {:error, e} ->
         {:error, e}
+
+      {:reject, e} ->
+        {:reject, e}
     end
   end
 
   def do_common_pipeline(object, meta) do
     with {_, {:ok, validated_object, meta}} <-
            {:validate_object, ObjectValidator.validate(object, meta)},
-         {_, {:ok, mrfd_object}} <- {:mrf_object, MRF.filter(validated_object)},
+         {_, {:ok, mrfd_object, meta}} <-
+           {:mrf_object, MRF.pipeline_filter(validated_object, meta)},
          {_, {:ok, activity, meta}} <-
            {:persist_object, ActivityPub.persist(mrfd_object, meta)},
          {_, {:ok, activity, meta}} <-
@@ -40,7 +44,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do
          {_, {:ok, _}} <- {:federation, maybe_federate(activity, meta)} do
       {:ok, activity, meta}
     else
-      {:mrf_object, {:reject, _}} -> {:ok, nil, meta}
+      {:mrf_object, {:reject, message, _}} -> {:reject, message}
       e -> {:error, e}
     end
   end
-- 
cgit v1.2.3


From 7bf269fe836ded974d2187c6b36eba4ab185ff25 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 14 Sep 2020 14:07:22 +0200
Subject: Fix MRF reject for ChatMessage

---
 lib/pleroma/web/api_spec/operations/chat_operation.ex      |  3 ++-
 lib/pleroma/web/api_spec/operations/status_operation.ex    |  2 +-
 lib/pleroma/web/common_api/common_api.ex                   |  3 +++
 lib/pleroma/web/pleroma_api/controllers/chat_controller.ex | 10 ++++++++++
 4 files changed, 16 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index b1a0d26ab..56554d5b4 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -184,7 +184,8 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
             "application/json",
             ChatMessage
           ),
-        400 => Operation.response("Bad Request", "application/json", ApiError)
+        400 => Operation.response("Bad Request", "application/json", ApiError),
+        422 => Operation.response("MRF Rejection", "application/json", ApiError)
       },
       security: [
         %{
diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex
index 5bd4619d5..d7ebde6f6 100644
--- a/lib/pleroma/web/api_spec/operations/status_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/status_operation.ex
@@ -55,7 +55,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do
             "application/json",
             %Schema{oneOf: [Status, ScheduledStatus]}
           ),
-        422 => Operation.response("Bad Request", "application/json", ApiError)
+        422 => Operation.response("Bad Request / MRF Rejection", "application/json", ApiError)
       }
     }
   end
diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index a8c83bc8f..60a50b027 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -48,6 +48,9 @@ defmodule Pleroma.Web.CommonAPI do
               local: true
             )} do
       {:ok, activity}
+    else
+      {:common_pipeline, {:reject, _} = e} -> e
+      e -> e
     end
   end
 
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index 27c9a2e0f..867cff829 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -90,6 +90,16 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
       conn
       |> put_view(MessageReferenceView)
       |> render("show.json", chat_message_reference: cm_ref)
+    else
+      {:reject, message} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: message})
+
+      {:error, message} ->
+        conn
+        |> put_status(:bad_request)
+        |> json(%{error: message})
     end
   end
 
-- 
cgit v1.2.3


From f2ef9735c52c648a03de4af41f19bb4ec857de03 Mon Sep 17 00:00:00 2001
From: Steven Fuchs <steven.fuchs@dockyard.com>
Date: Fri, 18 Sep 2020 11:58:22 +0000
Subject: Federate data through persistent websocket connections

---
 lib/pleroma/application.ex                      |   9 +-
 lib/pleroma/object/fetcher.ex                   |  60 ++++++--
 lib/pleroma/signature.ex                        |   6 +-
 lib/pleroma/user.ex                             |  10 +-
 lib/pleroma/web/activity_pub/activity_pub.ex    |  14 +-
 lib/pleroma/web/activity_pub/publisher.ex       |  34 +++--
 lib/pleroma/web/activity_pub/transmogrifier.ex  |   2 +-
 lib/pleroma/web/fed_sockets/fed_registry.ex     | 185 ++++++++++++++++++++++++
 lib/pleroma/web/fed_sockets/fed_socket.ex       | 137 ++++++++++++++++++
 lib/pleroma/web/fed_sockets/fed_sockets.ex      | 182 +++++++++++++++++++++++
 lib/pleroma/web/fed_sockets/fetch_registry.ex   | 151 +++++++++++++++++++
 lib/pleroma/web/fed_sockets/incoming_handler.ex |  88 +++++++++++
 lib/pleroma/web/fed_sockets/ingester_worker.ex  |  33 +++++
 lib/pleroma/web/fed_sockets/outgoing_handler.ex | 146 +++++++++++++++++++
 lib/pleroma/web/fed_sockets/socket_info.ex      |  52 +++++++
 lib/pleroma/web/fed_sockets/supervisor.ex       |  59 ++++++++
 16 files changed, 1123 insertions(+), 45 deletions(-)
 create mode 100644 lib/pleroma/web/fed_sockets/fed_registry.ex
 create mode 100644 lib/pleroma/web/fed_sockets/fed_socket.ex
 create mode 100644 lib/pleroma/web/fed_sockets/fed_sockets.ex
 create mode 100644 lib/pleroma/web/fed_sockets/fetch_registry.ex
 create mode 100644 lib/pleroma/web/fed_sockets/incoming_handler.ex
 create mode 100644 lib/pleroma/web/fed_sockets/ingester_worker.ex
 create mode 100644 lib/pleroma/web/fed_sockets/outgoing_handler.ex
 create mode 100644 lib/pleroma/web/fed_sockets/socket_info.ex
 create mode 100644 lib/pleroma/web/fed_sockets/supervisor.ex

(limited to 'lib')

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index c39e24919..00ec79a2a 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -99,7 +99,7 @@ defmodule Pleroma.Application do
           {Oban, Config.get(Oban)}
         ] ++
         task_children(@env) ++
-        streamer_child(@env) ++
+        dont_run_in_test(@env) ++
         chat_child(@env, chat_enabled?()) ++
         [
           Pleroma.Web.Endpoint,
@@ -188,16 +188,17 @@ defmodule Pleroma.Application do
 
   defp chat_enabled?, do: Config.get([:chat, :enabled])
 
-  defp streamer_child(env) when env in [:test, :benchmark], do: []
+  defp dont_run_in_test(env) when env in [:test, :benchmark], do: []
 
-  defp streamer_child(_) do
+  defp dont_run_in_test(_) do
     [
       {Registry,
        [
          name: Pleroma.Web.Streamer.registry(),
          keys: :duplicate,
          partitions: System.schedulers_online()
-       ]}
+       ]},
+      Pleroma.Web.FedSockets.Supervisor
     ]
   end
 
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 24dc7cb95..169298b34 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -12,6 +12,7 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Web.ActivityPub.ObjectValidator
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.Federator
+  alias Pleroma.Web.FedSockets
 
   require Logger
   require Pleroma.Constants
@@ -182,9 +183,47 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(id) when is_binary(id) do
+  def fetch_and_contain_remote_object_from_id(prm, opts \\ [])
+
+  def fetch_and_contain_remote_object_from_id(%{"id" => id}, opts),
+    do: fetch_and_contain_remote_object_from_id(id, opts)
+
+  def fetch_and_contain_remote_object_from_id(id, opts) when is_binary(id) do
     Logger.debug("Fetching object #{id} via AP")
 
+    with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
+         {:ok, body} <- get_object(id, opts),
+         {:ok, data} <- safe_json_decode(body),
+         :ok <- Containment.contain_origin_from_id(id, data) do
+      {:ok, data}
+    else
+      {:scheme, _} ->
+        {:error, "Unsupported URI scheme"}
+
+      {:error, e} ->
+        {:error, e}
+
+      e ->
+        {:error, e}
+    end
+  end
+
+  def fetch_and_contain_remote_object_from_id(_id, _opts),
+    do: {:error, "id must be a string"}
+
+  defp get_object(id, opts) do
+    with false <- Keyword.get(opts, :force_http, false),
+         {:ok, fedsocket} <- FedSockets.get_or_create_fed_socket(id) do
+      Logger.debug("fetching via fedsocket - #{inspect(id)}")
+      FedSockets.fetch(fedsocket, id)
+    else
+      _other ->
+        Logger.debug("fetching via http - #{inspect(id)}")
+        get_object_http(id)
+    end
+  end
+
+  defp get_object_http(id) do
     date = Pleroma.Signature.signed_date()
 
     headers =
@@ -192,20 +231,13 @@ defmodule Pleroma.Object.Fetcher do
       |> maybe_date_fetch(date)
       |> sign_fetch(id, date)
 
-    Logger.debug("Fetch headers: #{inspect(headers)}")
+    case HTTP.get(id, headers) do
+      {:ok, %{body: body, status: code}} when code in 200..299 ->
+        {:ok, body}
 
-    with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
-         {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get(id, headers),
-         {:ok, data} <- Jason.decode(body),
-         :ok <- Containment.contain_origin_from_id(id, data) do
-      {:ok, data}
-    else
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, "Object has been deleted"}
 
-      {:scheme, _} ->
-        {:error, "Unsupported URI scheme"}
-
       {:error, e} ->
         {:error, e}
 
@@ -214,8 +246,6 @@ defmodule Pleroma.Object.Fetcher do
     end
   end
 
-  def fetch_and_contain_remote_object_from_id(%{"id" => id}),
-    do: fetch_and_contain_remote_object_from_id(id)
-
-  def fetch_and_contain_remote_object_from_id(_id), do: {:error, "id must be a string"}
+  defp safe_json_decode(nil), do: {:ok, nil}
+  defp safe_json_decode(json), do: Jason.decode(json)
 end
diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex
index 3aa6909d2..e388993b7 100644
--- a/lib/pleroma/signature.ex
+++ b/lib/pleroma/signature.ex
@@ -39,7 +39,7 @@ defmodule Pleroma.Signature do
   def fetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
       {:ok, public_key}
     else
       e ->
@@ -50,8 +50,8 @@ defmodule Pleroma.Signature do
   def refetch_public_key(conn) do
     with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn),
          {:ok, actor_id} <- key_id_to_actor_id(kid),
-         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do
+         {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id, force_http: true),
+         {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id, force_http: true) do
       {:ok, public_key}
     else
       e ->
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 1ffe60dfc..d92484a40 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1820,12 +1820,12 @@ defmodule Pleroma.User do
 
   def html_filter_policy(_), do: Config.get([:markup, :scrub_policy])
 
-  def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
+  def fetch_by_ap_id(ap_id, opts \\ []), do: ActivityPub.make_user_from_ap_id(ap_id, opts)
 
-  def get_or_fetch_by_ap_id(ap_id) do
+  def get_or_fetch_by_ap_id(ap_id, opts \\ []) do
     cached_user = get_cached_by_ap_id(ap_id)
 
-    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id)
+    maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id, opts)
 
     case {cached_user, maybe_fetched_user} do
       {_, {:ok, %User{} = user}} ->
@@ -1898,8 +1898,8 @@ defmodule Pleroma.User do
 
   def public_key(_), do: {:error, "key not found"}
 
-  def get_public_key_for_ap_id(ap_id) do
-    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id),
+  def get_public_key_for_ap_id(ap_id, opts \\ []) do
+    with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id, opts),
          {:ok, public_key} <- public_key(user) do
       {:ok, public_key}
     else
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 92fc1e422..06e8e1a7c 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1270,10 +1270,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   def fetch_follow_information_for_user(user) do
     with {:ok, following_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address),
+           Fetcher.fetch_and_contain_remote_object_from_id(user.following_address,
+             force_http: true
+           ),
          {:ok, hide_follows} <- collection_private(following_data),
          {:ok, followers_data} <-
-           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address),
+           Fetcher.fetch_and_contain_remote_object_from_id(user.follower_address, force_http: true),
          {:ok, hide_followers} <- collection_private(followers_data) do
       {:ok,
        %{
@@ -1347,8 +1349,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id),
+  def fetch_and_prepare_user_from_ap_id(ap_id, opts \\ []) do
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id, opts),
          {:ok, data} <- user_data_from_user_object(data) do
       {:ok, maybe_update_follow_information(data)}
     else
@@ -1390,13 +1392,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def make_user_from_ap_id(ap_id) do
+  def make_user_from_ap_id(ap_id, opts \\ []) do
     user = User.get_cached_by_ap_id(ap_id)
 
     if user && !User.ap_enabled?(user) do
       Transmogrifier.upgrade_user_from_ap_id(ap_id)
     else
-      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id) do
+      with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, opts) do
         if user do
           user
           |> User.remote_user_changeset(data)
diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex
index d88f7f3ee..9c3956683 100644
--- a/lib/pleroma/web/activity_pub/publisher.ex
+++ b/lib/pleroma/web/activity_pub/publisher.ex
@@ -13,6 +13,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
   alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.FedSockets
 
   require Pleroma.Constants
 
@@ -50,15 +51,35 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
     Logger.debug("Federating #{id} to #{inbox}")
 
-    uri = URI.parse(inbox)
+    case FedSockets.get_or_create_fed_socket(inbox) do
+      {:ok, fedsocket} ->
+        Logger.debug("publishing via fedsockets - #{inspect(inbox)}")
+        FedSockets.publish(fedsocket, json)
 
+      _ ->
+        Logger.debug("publishing via http - #{inspect(inbox)}")
+        http_publish(inbox, actor, json, params)
+    end
+  end
+
+  def publish_one(%{actor_id: actor_id} = params) do
+    actor = User.get_cached_by_id(actor_id)
+
+    params
+    |> Map.delete(:actor_id)
+    |> Map.put(:actor, actor)
+    |> publish_one()
+  end
+
+  defp http_publish(inbox, actor, json, params) do
+    uri = %{path: path} = URI.parse(inbox)
     digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
 
     date = Pleroma.Signature.signed_date()
 
     signature =
       Pleroma.Signature.sign(actor, %{
-        "(request-target)": "post #{uri.path}",
+        "(request-target)": "post #{path}",
         host: signature_host(uri),
         "content-length": byte_size(json),
         digest: digest,
@@ -89,15 +110,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     end
   end
 
-  def publish_one(%{actor_id: actor_id} = params) do
-    actor = User.get_cached_by_id(actor_id)
-
-    params
-    |> Map.delete(:actor_id)
-    |> Map.put(:actor, actor)
-    |> publish_one()
-  end
-
   defp signature_host(%URI{port: port, scheme: scheme, host: host}) do
     if port == URI.default_port(scheme) do
       host
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index fcca014f0..aa6a69463 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -1000,7 +1000,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def upgrade_user_from_ap_id(ap_id) do
     with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id),
-         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id),
+         {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id, force_http: true),
          {:ok, user} <- update_user(user, data) do
       TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id})
       {:ok, user}
diff --git a/lib/pleroma/web/fed_sockets/fed_registry.ex b/lib/pleroma/web/fed_sockets/fed_registry.ex
new file mode 100644
index 000000000..e00ea69c0
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/fed_registry.ex
@@ -0,0 +1,185 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.FedRegistry do
+  @moduledoc """
+  The FedRegistry stores the active FedSockets for quick retrieval.
+
+  The storage and retrieval portion of the FedRegistry is done in process through
+  elixir's `Registry` module for speed and its ability to monitor for terminated processes.
+
+  Dropped connections will be caught by `Registry` and deleted. Since the next
+  message will initiate a new connection there is no reason to try and reconnect at that point.
+
+  Normally outside modules should have no need to call or use the FedRegistry themselves.
+  """
+
+  alias Pleroma.Web.FedSockets.FedSocket
+  alias Pleroma.Web.FedSockets.SocketInfo
+
+  require Logger
+
+  @default_rejection_duration 15 * 60 * 1000
+  @rejections :fed_socket_rejections
+
+  @doc """
+  Retrieves a FedSocket from the Registry given it's origin.
+
+  The origin is expected to be a string identifying the endpoint "example.com" or "example2.com:8080"
+
+  Will return:
+    * {:ok, fed_socket} for working FedSockets
+    * {:error, :rejected} for origins that have been tried and refused within the rejection duration interval
+    * {:error, some_reason} usually :missing for unknown origins
+  """
+  def get_fed_socket(origin) do
+    case get_registry_data(origin) do
+      {:error, reason} ->
+        {:error, reason}
+
+      {:ok, %{state: :connected} = socket_info} ->
+        {:ok, socket_info}
+    end
+  end
+
+  @doc """
+  Adds a connected FedSocket to the Registry.
+
+  Always returns {:ok, fed_socket}
+  """
+  def add_fed_socket(origin, pid \\ nil) do
+    origin
+    |> SocketInfo.build(pid)
+    |> SocketInfo.connect()
+    |> add_socket_info
+  end
+
+  defp add_socket_info(%{origin: origin, state: :connected} = socket_info) do
+    case Registry.register(FedSockets.Registry, origin, socket_info) do
+      {:ok, _owner} ->
+        clear_prior_rejection(origin)
+        Logger.debug("fedsocket added: #{inspect(origin)}")
+
+        {:ok, socket_info}
+
+      {:error, {:already_registered, _pid}} ->
+        FedSocket.close(socket_info)
+        existing_socket_info = Registry.lookup(FedSockets.Registry, origin)
+
+        {:ok, existing_socket_info}
+
+      _ ->
+        {:error, :error_adding_socket}
+    end
+  end
+
+  @doc """
+  Mark this origin as having rejected a connection attempt.
+  This will keep it from getting additional connection attempts
+  for a period of time specified in the config.
+
+  Always returns {:ok, new_reg_data}
+  """
+  def set_host_rejected(uri) do
+    new_reg_data =
+      uri
+      |> SocketInfo.origin()
+      |> get_or_create_registry_data()
+      |> set_to_rejected()
+      |> save_registry_data()
+
+    {:ok, new_reg_data}
+  end
+
+  @doc """
+  Retrieves the FedRegistryData from the Registry given it's origin.
+
+  The origin is expected to be a string identifying the endpoint "example.com" or "example2.com:8080"
+
+  Will return:
+    * {:ok, fed_registry_data} for known origins
+    * {:error, :missing} for uniknown origins
+    * {:error, :cache_error} indicating some low level runtime issues
+  """
+  def get_registry_data(origin) do
+    case Registry.lookup(FedSockets.Registry, origin) do
+      [] ->
+        if is_rejected?(origin) do
+          Logger.debug("previously rejected fedsocket requested")
+          {:error, :rejected}
+        else
+          {:error, :missing}
+        end
+
+      [{_pid, %{state: :connected} = socket_info}] ->
+        {:ok, socket_info}
+
+      _ ->
+        {:error, :cache_error}
+    end
+  end
+
+  @doc """
+  Retrieves a map of all sockets from the Registry. The keys are the origins and the values are the corresponding SocketInfo
+  """
+  def list_all do
+    (list_all_connected() ++ list_all_rejected())
+    |> Enum.into(%{})
+  end
+
+  defp list_all_connected do
+    FedSockets.Registry
+    |> Registry.select([{{:"$1", :_, :"$3"}, [], [{{:"$1", :"$3"}}]}])
+  end
+
+  defp list_all_rejected do
+    {:ok, keys} = Cachex.keys(@rejections)
+
+    {:ok, registry_data} =
+      Cachex.execute(@rejections, fn worker ->
+        Enum.map(keys, fn k -> {k, Cachex.get!(worker, k)} end)
+      end)
+
+    registry_data
+  end
+
+  defp clear_prior_rejection(origin),
+    do: Cachex.del(@rejections, origin)
+
+  defp is_rejected?(origin) do
+    case Cachex.get(@rejections, origin) do
+      {:ok, nil} ->
+        false
+
+      {:ok, _} ->
+        true
+    end
+  end
+
+  defp get_or_create_registry_data(origin) do
+    case get_registry_data(origin) do
+      {:error, :missing} ->
+        %SocketInfo{origin: origin}
+
+      {:ok, socket_info} ->
+        socket_info
+    end
+  end
+
+  defp save_registry_data(%SocketInfo{origin: origin, state: :connected} = socket_info) do
+    {:ok, true} = Registry.update_value(FedSockets.Registry, origin, fn _ -> socket_info end)
+    socket_info
+  end
+
+  defp save_registry_data(%SocketInfo{origin: origin, state: :rejected} = socket_info) do
+    rejection_expiration =
+      Pleroma.Config.get([:fed_sockets, :rejection_duration], @default_rejection_duration)
+
+    {:ok, true} = Cachex.put(@rejections, origin, socket_info, ttl: rejection_expiration)
+    socket_info
+  end
+
+  defp set_to_rejected(%SocketInfo{} = socket_info),
+    do: %SocketInfo{socket_info | state: :rejected}
+end
diff --git a/lib/pleroma/web/fed_sockets/fed_socket.ex b/lib/pleroma/web/fed_sockets/fed_socket.ex
new file mode 100644
index 000000000..98d64e65a
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/fed_socket.ex
@@ -0,0 +1,137 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.FedSocket do
+  @moduledoc """
+  The FedSocket module abstracts the actions to be taken taken on connections regardless of
+  whether the connection started as inbound or outbound.
+
+
+  Normally outside modules will have no need to call the FedSocket module directly.
+  """
+
+  alias Pleroma.Object
+  alias Pleroma.Object.Containment
+  alias Pleroma.User
+  alias Pleroma.Web.ActivityPub.ObjectView
+  alias Pleroma.Web.ActivityPub.UserView
+  alias Pleroma.Web.ActivityPub.Visibility
+  alias Pleroma.Web.FedSockets.FetchRegistry
+  alias Pleroma.Web.FedSockets.IngesterWorker
+  alias Pleroma.Web.FedSockets.OutgoingHandler
+  alias Pleroma.Web.FedSockets.SocketInfo
+
+  require Logger
+
+  @shake "61dd18f7-f1e6-49a4-939a-a749fcdc1103"
+
+  def connect_to_host(uri) do
+    case OutgoingHandler.start_link(uri) do
+      {:ok, pid} ->
+        {:ok, pid}
+
+      error ->
+        {:error, error}
+    end
+  end
+
+  def close(%SocketInfo{pid: socket_pid}),
+    do: Process.send(socket_pid, :close, [])
+
+  def publish(%SocketInfo{pid: socket_pid}, json) do
+    %{action: :publish, data: json}
+    |> Jason.encode!()
+    |> send_packet(socket_pid)
+  end
+
+  def fetch(%SocketInfo{pid: socket_pid}, id) do
+    fetch_uuid = FetchRegistry.register_fetch(id)
+
+    %{action: :fetch, data: id, uuid: fetch_uuid}
+    |> Jason.encode!()
+    |> send_packet(socket_pid)
+
+    wait_for_fetch_to_return(fetch_uuid, 0)
+  end
+
+  def receive_package(%SocketInfo{} = fed_socket, json) do
+    json
+    |> Jason.decode!()
+    |> process_package(fed_socket)
+  end
+
+  defp wait_for_fetch_to_return(uuid, cntr) do
+    case FetchRegistry.check_fetch(uuid) do
+      {:error, :waiting} ->
+        Process.sleep(:math.pow(cntr, 3) |> Kernel.trunc())
+        wait_for_fetch_to_return(uuid, cntr + 1)
+
+      {:error, :missing} ->
+        Logger.error("FedSocket fetch timed out - #{inspect(uuid)}")
+        {:error, :timeout}
+
+      {:ok, _fr} ->
+        FetchRegistry.pop_fetch(uuid)
+    end
+  end
+
+  defp process_package(%{"action" => "publish", "data" => data}, %{origin: origin} = _fed_socket) do
+    if Containment.contain_origin(origin, data) do
+      IngesterWorker.enqueue("ingest", %{"object" => data})
+    end
+
+    {:reply, %{"action" => "publish_reply", "status" => "processed"}}
+  end
+
+  defp process_package(%{"action" => "fetch_reply", "uuid" => uuid, "data" => data}, _fed_socket) do
+    FetchRegistry.register_fetch_received(uuid, data)
+    {:noreply, nil}
+  end
+
+  defp process_package(%{"action" => "fetch", "uuid" => uuid, "data" => ap_id}, _fed_socket) do
+    {:ok, data} = render_fetched_data(ap_id, uuid)
+    {:reply, data}
+  end
+
+  defp process_package(%{"action" => "publish_reply"}, _fed_socket) do
+    {:noreply, nil}
+  end
+
+  defp process_package(other, _fed_socket) do
+    Logger.warn("unknown json packages received #{inspect(other)}")
+    {:noreply, nil}
+  end
+
+  defp render_fetched_data(ap_id, uuid) do
+    {:ok,
+     %{
+       "action" => "fetch_reply",
+       "status" => "processed",
+       "uuid" => uuid,
+       "data" => represent_item(ap_id)
+     }}
+  end
+
+  defp represent_item(ap_id) do
+    case User.get_by_ap_id(ap_id) do
+      nil ->
+        object = Object.get_cached_by_ap_id(ap_id)
+
+        if Visibility.is_public?(object) do
+          Phoenix.View.render_to_string(ObjectView, "object.json", object: object)
+        else
+          nil
+        end
+
+      user ->
+        Phoenix.View.render_to_string(UserView, "user.json", user: user)
+    end
+  end
+
+  defp send_packet(data, socket_pid) do
+    Process.send(socket_pid, {:send, data}, [])
+  end
+
+  def shake, do: @shake
+end
diff --git a/lib/pleroma/web/fed_sockets/fed_sockets.ex b/lib/pleroma/web/fed_sockets/fed_sockets.ex
new file mode 100644
index 000000000..035d54796
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/fed_sockets.ex
@@ -0,0 +1,182 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets do
+  @moduledoc """
+  This documents the FedSockets framework. A framework for federating
+  ActivityPub objects between servers via persistant WebSocket connections.
+
+  FedSockets allow servers to authenticate on first contact and maintain that
+  connection, eliminating the need to authenticate every time data needs to be shared.
+
+  ## Protocol
+  FedSockets currently support 2 types of data transfer:
+    * `publish` method which doesn't require a response
+    * `fetch` method requires a response be sent
+
+    ### Publish
+    The publish operation sends a json encoded map of the shape:
+      %{action: :publish, data: json}
+    and accepts (but does not require) a reply of form:
+      %{"action" => "publish_reply"}
+
+    The outgoing params represent
+      * data: ActivityPub object encoded into json
+
+
+    ### Fetch
+    The fetch operation sends a json encoded map of the shape:
+      %{action: :fetch, data: id, uuid: fetch_uuid}
+    and requires a reply of form:
+      %{"action" => "fetch_reply", "uuid" => uuid, "data" => data}
+
+    The outgoing params represent
+      * id: an ActivityPub object URI
+      * uuid: a unique uuid generated by the sender
+
+    The reply params represent
+      * data: an ActivityPub object encoded into json
+      * uuid: the uuid sent along with the fetch request
+
+  ## Examples
+  Clients of FedSocket transfers shouldn't need to use any of the functions outside of this module.
+
+  A typical publish operation can be performed through the following code, and a fetch operation in a similar manner.
+
+    case FedSockets.get_or_create_fed_socket(inbox) do
+      {:ok, fedsocket} ->
+        FedSockets.publish(fedsocket, json)
+
+      _ ->
+        alternative_publish(inbox, actor, json, params)
+    end
+
+  ## Configuration
+  FedSockets have the following config settings
+
+  config :pleroma, :fed_sockets,
+  enabled: true,
+  ping_interval: :timer.seconds(15),
+  connection_duration: :timer.hours(1),
+  rejection_duration: :timer.hours(1),
+  fed_socket_fetches: [
+    default: 12_000,
+    interval: 3_000,
+    lazy: false
+  ]
+    * enabled - turn FedSockets on or off with this flag. Can be toggled at runtime.
+    * connection_duration - How long a FedSocket can sit idle before it's culled.
+    * rejection_duration - After failing to make a FedSocket connection a host will be excluded
+    from further connections for this amount of time
+    * fed_socket_fetches - Use these parameters to pass options to the Cachex queue backing the FetchRegistry
+    * fed_socket_rejections - Use these parameters to pass options to the Cachex queue backing the FedRegistry
+
+    Cachex options are
+      * default: the minimum amount of time a fetch can wait before it times out.
+      * interval: the interval between checks for timed out entries. This plus the default represent the maximum time allowed
+      * lazy: leave at false for consistant and fast lookups, set to true for stricter timeout enforcement
+
+  """
+  require Logger
+
+  alias Pleroma.Web.FedSockets.FedRegistry
+  alias Pleroma.Web.FedSockets.FedSocket
+  alias Pleroma.Web.FedSockets.SocketInfo
+
+  @doc """
+  returns a FedSocket for the given origin. Will reuse an existing one or create a new one.
+
+  address is expected to be a fully formed URL such as:
+  "http://www.example.com" or "http://www.example.com:8080"
+
+  It can and usually does include additional path parameters,
+  but these are ignored as the FedSockets are organized by host and port info alone.
+  """
+  def get_or_create_fed_socket(address) do
+    with {:cache, {:error, :missing}} <- {:cache, get_fed_socket(address)},
+         {:connect, {:ok, _pid}} <- {:connect, FedSocket.connect_to_host(address)},
+         {:cache, {:ok, fed_socket}} <- {:cache, get_fed_socket(address)} do
+      Logger.debug("fedsocket created for - #{inspect(address)}")
+      {:ok, fed_socket}
+    else
+      {:cache, {:ok, socket}} ->
+        Logger.debug("fedsocket found in cache - #{inspect(address)}")
+        {:ok, socket}
+
+      {:connect, {:error, _host}} ->
+        Logger.debug("set host rejected for - #{inspect(address)}")
+        FedRegistry.set_host_rejected(address)
+        {:error, :rejected}
+
+      {_, {:error, :disabled}} ->
+        {:error, :disabled}
+
+      {_, {:error, reason}} ->
+        Logger.warn("get_or_create_fed_socket error - #{inspect(reason)}")
+        {:error, reason}
+    end
+  end
+
+  @doc """
+  returns a FedSocket for the given origin. Will not create a new FedSocket if one does not exist.
+
+  address is expected to be a fully formed URL such as:
+    "http://www.example.com" or "http://www.example.com:8080"
+  """
+  def get_fed_socket(address) do
+    origin = SocketInfo.origin(address)
+
+    with {:config, true} <- {:config, Pleroma.Config.get([:fed_sockets, :enabled], false)},
+         {:ok, socket} <- FedRegistry.get_fed_socket(origin) do
+      {:ok, socket}
+    else
+      {:config, _} ->
+        {:error, :disabled}
+
+      {:error, :rejected} ->
+        Logger.debug("FedSocket previously rejected - #{inspect(origin)}")
+        {:error, :rejected}
+
+      {:error, reason} ->
+        {:error, reason}
+    end
+  end
+
+  @doc """
+  Sends the supplied data via the publish protocol.
+  It will not block waiting for a reply.
+  Returns :ok but this is not an indication of a successful transfer.
+
+  the data is expected to be JSON encoded binary data.
+  """
+  def publish(%SocketInfo{} = fed_socket, json) do
+    FedSocket.publish(fed_socket, json)
+  end
+
+  @doc """
+  Sends the supplied data via the fetch protocol.
+  It will block waiting for a reply or timeout.
+
+  Returns {:ok, object} where object is the requested object (or nil)
+          {:error, :timeout} in the event the message was not responded to
+
+  the id is expected to be the URI of an ActivityPub object.
+  """
+  def fetch(%SocketInfo{} = fed_socket, id) do
+    FedSocket.fetch(fed_socket, id)
+  end
+
+  @doc """
+  Disconnect all and restart FedSockets.
+  This is mainly used in development and testing but could be useful in production.
+  """
+  def reset do
+    FedRegistry
+    |> Process.whereis()
+    |> Process.exit(:testing)
+  end
+
+  def uri_for_origin(origin),
+    do: "ws://#{origin}/api/fedsocket/v1"
+end
diff --git a/lib/pleroma/web/fed_sockets/fetch_registry.ex b/lib/pleroma/web/fed_sockets/fetch_registry.ex
new file mode 100644
index 000000000..7897f0fc6
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/fetch_registry.ex
@@ -0,0 +1,151 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.FetchRegistry do
+  @moduledoc """
+  The FetchRegistry acts as a broker for fetch requests and return values.
+  This allows calling processes to block while waiting for a reply.
+  It doesn't impose it's own process instead using `Cachex` to handle fetches in process, allowing
+  multi threaded processes to avoid bottlenecking.
+
+  Normally outside modules will have no need to call or use the FetchRegistry themselves.
+
+  The `Cachex` parameters can be controlled from the config. Since exact timeout intervals
+  aren't necessary the following settings are used by default:
+
+  config :pleroma, :fed_sockets,
+    fed_socket_fetches: [
+      default: 12_000,
+      interval: 3_000,
+      lazy: false
+    ]
+
+  """
+
+  defmodule FetchRegistryData do
+    defstruct uuid: nil,
+              sent_json: nil,
+              received_json: nil,
+              sent_at: nil,
+              received_at: nil
+  end
+
+  alias Ecto.UUID
+
+  require Logger
+
+  @fetches :fed_socket_fetches
+
+  @doc """
+  Registers a json request wth the FetchRegistry and returns the identifying UUID.
+  """
+  def register_fetch(json) do
+    %FetchRegistryData{uuid: uuid} =
+      json
+      |> new_registry_data
+      |> save_registry_data
+
+    uuid
+  end
+
+  @doc """
+  Reports on the status of a Fetch given the identifying UUID.
+
+  Will return
+    * {:ok, fetched_object} if a fetch has completed
+    * {:error, :waiting} if a fetch is still pending
+    * {:error, other_error} usually :missing to indicate a fetch that has timed out
+  """
+  def check_fetch(uuid) do
+    case get_registry_data(uuid) do
+      {:ok, %FetchRegistryData{received_at: nil}} ->
+        {:error, :waiting}
+
+      {:ok, %FetchRegistryData{} = reg_data} ->
+        {:ok, reg_data}
+
+      e ->
+        e
+    end
+  end
+
+  @doc """
+  Retrieves the response to a fetch given the identifying UUID.
+  The completed fetch will be deleted from the FetchRegistry
+
+  Will return
+    * {:ok, fetched_object} if a fetch has completed
+    * {:error, :waiting} if a fetch is still pending
+    * {:error, other_error} usually :missing to indicate a fetch that has timed out
+  """
+  def pop_fetch(uuid) do
+    case check_fetch(uuid) do
+      {:ok, %FetchRegistryData{received_json: received_json}} ->
+        delete_registry_data(uuid)
+        {:ok, received_json}
+
+      e ->
+        e
+    end
+  end
+
+  @doc """
+  This is called to register a fetch has returned.
+  It expects the result data along with the UUID that was sent in the request
+
+  Will return the fetched object or :error
+  """
+  def register_fetch_received(uuid, data) do
+    case get_registry_data(uuid) do
+      {:ok, %FetchRegistryData{received_at: nil} = reg_data} ->
+        reg_data
+        |> set_fetch_received(data)
+        |> save_registry_data()
+
+      {:ok, %FetchRegistryData{} = reg_data} ->
+        Logger.warn("tried to add fetched data twice - #{uuid}")
+        reg_data
+
+      {:error, _} ->
+        Logger.warn("Error adding fetch to registry - #{uuid}")
+        :error
+    end
+  end
+
+  defp new_registry_data(json) do
+    %FetchRegistryData{
+      uuid: UUID.generate(),
+      sent_json: json,
+      sent_at: :erlang.monotonic_time(:millisecond)
+    }
+  end
+
+  defp get_registry_data(origin) do
+    case Cachex.get(@fetches, origin) do
+      {:ok, nil} ->
+        {:error, :missing}
+
+      {:ok, reg_data} ->
+        {:ok, reg_data}
+
+      _ ->
+        {:error, :cache_error}
+    end
+  end
+
+  defp set_fetch_received(%FetchRegistryData{} = reg_data, data),
+    do: %FetchRegistryData{
+      reg_data
+      | received_at: :erlang.monotonic_time(:millisecond),
+        received_json: data
+    }
+
+  defp save_registry_data(%FetchRegistryData{uuid: uuid} = reg_data) do
+    {:ok, true} = Cachex.put(@fetches, uuid, reg_data)
+    reg_data
+  end
+
+  defp delete_registry_data(origin),
+    do: {:ok, true} = Cachex.del(@fetches, origin)
+end
diff --git a/lib/pleroma/web/fed_sockets/incoming_handler.ex b/lib/pleroma/web/fed_sockets/incoming_handler.ex
new file mode 100644
index 000000000..49d0d9d84
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/incoming_handler.ex
@@ -0,0 +1,88 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.IncomingHandler do
+  require Logger
+
+  alias Pleroma.Web.FedSockets.FedRegistry
+  alias Pleroma.Web.FedSockets.FedSocket
+  alias Pleroma.Web.FedSockets.SocketInfo
+
+  import HTTPSignatures, only: [validate_conn: 1, split_signature: 1]
+
+  @behaviour :cowboy_websocket
+
+  def init(req, state) do
+    shake = FedSocket.shake()
+
+    with true <- Pleroma.Config.get([:fed_sockets, :enabled]),
+         sec_protocol <- :cowboy_req.header("sec-websocket-protocol", req, nil),
+         headers = %{"(request-target)" => ^shake} <- :cowboy_req.headers(req),
+         true <- validate_conn(%{req_headers: headers}),
+         %{"keyId" => origin} <- split_signature(headers["signature"]) do
+      req =
+        if is_nil(sec_protocol) do
+          req
+        else
+          :cowboy_req.set_resp_header("sec-websocket-protocol", sec_protocol, req)
+        end
+
+      {:cowboy_websocket, req, %{origin: origin}, %{}}
+    else
+      _ ->
+        {:ok, req, state}
+    end
+  end
+
+  def websocket_init(%{origin: origin}) do
+    case FedRegistry.add_fed_socket(origin) do
+      {:ok, socket_info} ->
+        {:ok, socket_info}
+
+      e ->
+        Logger.error("FedSocket websocket_init failed - #{inspect(e)}")
+        {:error, inspect(e)}
+    end
+  end
+
+  # Use the ping to  check if the connection should be expired
+  def websocket_handle(:ping, socket_info) do
+    if SocketInfo.expired?(socket_info) do
+      {:stop, socket_info}
+    else
+      {:ok, socket_info, :hibernate}
+    end
+  end
+
+  def websocket_handle({:text, data}, socket_info) do
+    socket_info = SocketInfo.touch(socket_info)
+
+    case FedSocket.receive_package(socket_info, data) do
+      {:noreply, _} ->
+        {:ok, socket_info}
+
+      {:reply, reply} ->
+        {:reply, {:text, Jason.encode!(reply)}, socket_info}
+
+      {:error, reason} ->
+        Logger.error("incoming error - receive_package: #{inspect(reason)}")
+        {:ok, socket_info}
+    end
+  end
+
+  def websocket_info({:send, message}, socket_info) do
+    socket_info = SocketInfo.touch(socket_info)
+
+    {:reply, {:text, message}, socket_info}
+  end
+
+  def websocket_info(:close, state) do
+    {:stop, state}
+  end
+
+  def websocket_info(message, state) do
+    Logger.debug("#{__MODULE__} unknown message #{inspect(message)}")
+    {:ok, state}
+  end
+end
diff --git a/lib/pleroma/web/fed_sockets/ingester_worker.ex b/lib/pleroma/web/fed_sockets/ingester_worker.ex
new file mode 100644
index 000000000..325f2a4ab
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/ingester_worker.ex
@@ -0,0 +1,33 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.IngesterWorker do
+  use Pleroma.Workers.WorkerHelper, queue: "ingestion_queue"
+  require Logger
+
+  alias Pleroma.Web.Federator
+
+  @impl Oban.Worker
+  def perform(%Job{args: %{"op" => "ingest", "object" => ingestee}}) do
+    try do
+      ingestee
+      |> Jason.decode!()
+      |> do_ingestion()
+    rescue
+      e ->
+        Logger.error("IngesterWorker error - #{inspect(e)}")
+        e
+    end
+  end
+
+  defp do_ingestion(params) do
+    case Federator.incoming_ap_doc(params) do
+      {:error, reason} ->
+        {:error, reason}
+
+      {:ok, object} ->
+        {:ok, object}
+    end
+  end
+end
diff --git a/lib/pleroma/web/fed_sockets/outgoing_handler.ex b/lib/pleroma/web/fed_sockets/outgoing_handler.ex
new file mode 100644
index 000000000..6ddef17fe
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/outgoing_handler.ex
@@ -0,0 +1,146 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.OutgoingHandler do
+  use GenServer
+
+  require Logger
+
+  alias Pleroma.Web.ActivityPub.InternalFetchActor
+  alias Pleroma.Web.FedSockets
+  alias Pleroma.Web.FedSockets.FedRegistry
+  alias Pleroma.Web.FedSockets.FedSocket
+  alias Pleroma.Web.FedSockets.SocketInfo
+
+  def start_link(uri) do
+    GenServer.start_link(__MODULE__, %{uri: uri})
+  end
+
+  def init(%{uri: uri}) do
+    case initiate_connection(uri) do
+      {:ok, ws_origin, conn_pid} ->
+        FedRegistry.add_fed_socket(ws_origin, conn_pid)
+
+      {:error, reason} ->
+        Logger.debug("Outgoing connection failed - #{inspect(reason)}")
+        :ignore
+    end
+  end
+
+  def handle_info({:gun_ws, conn_pid, _ref, {:text, data}}, socket_info) do
+    socket_info = SocketInfo.touch(socket_info)
+
+    case FedSocket.receive_package(socket_info, data) do
+      {:noreply, _} ->
+        {:noreply, socket_info}
+
+      {:reply, reply} ->
+        :gun.ws_send(conn_pid, {:text, Jason.encode!(reply)})
+        {:noreply, socket_info}
+
+      {:error, reason} ->
+        Logger.error("incoming error - receive_package: #{inspect(reason)}")
+        {:noreply, socket_info}
+    end
+  end
+
+  def handle_info(:close, state) do
+    Logger.debug("Sending close frame !!!!!!!")
+    {:close, state}
+  end
+
+  def handle_info({:gun_down, _pid, _prot, :closed, _}, state) do
+    {:stop, :normal, state}
+  end
+
+  def handle_info({:send, data}, %{conn_pid: conn_pid} = socket_info) do
+    socket_info = SocketInfo.touch(socket_info)
+    :gun.ws_send(conn_pid, {:text, data})
+    {:noreply, socket_info}
+  end
+
+  def handle_info({:gun_ws, _, _, :pong}, state) do
+    {:noreply, state, :hibernate}
+  end
+
+  def handle_info(msg, state) do
+    Logger.debug("#{__MODULE__} unhandled event #{inspect(msg)}")
+    {:noreply, state}
+  end
+
+  def terminate(reason, state) do
+    Logger.debug(
+      "#{__MODULE__} terminating outgoing connection for #{inspect(state)} for #{inspect(reason)}"
+    )
+
+    {:ok, state}
+  end
+
+  def initiate_connection(uri) do
+    ws_uri =
+      uri
+      |> SocketInfo.origin()
+      |> FedSockets.uri_for_origin()
+
+    %{host: host, port: port, path: path} = URI.parse(ws_uri)
+
+    with {:ok, conn_pid} <- :gun.open(to_charlist(host), port),
+         {:ok, _} <- :gun.await_up(conn_pid),
+         reference <- :gun.get(conn_pid, to_charlist(path)),
+         {:response, :fin, 204, _} <- :gun.await(conn_pid, reference),
+         headers <- build_headers(uri),
+         ref <- :gun.ws_upgrade(conn_pid, to_charlist(path), headers, %{silence_pings: false}) do
+      receive do
+        {:gun_upgrade, ^conn_pid, ^ref, [<<"websocket">>], _} ->
+          {:ok, ws_uri, conn_pid}
+      after
+        15_000 ->
+          Logger.debug("Fedsocket timeout connecting to #{inspect(uri)}")
+          {:error, :timeout}
+      end
+    else
+      {:response, :nofin, 404, _} ->
+        {:error, :fedsockets_not_supported}
+
+      e ->
+        Logger.debug("Fedsocket error connecting to #{inspect(uri)}")
+        {:error, e}
+    end
+  end
+
+  defp build_headers(uri) do
+    host_for_sig = uri |> URI.parse() |> host_signature()
+
+    shake = FedSocket.shake()
+    digest = "SHA-256=" <> (:crypto.hash(:sha256, shake) |> Base.encode64())
+    date = Pleroma.Signature.signed_date()
+    shake_size = byte_size(shake)
+
+    signature_opts = %{
+      "(request-target)": shake,
+      "content-length": to_charlist("#{shake_size}"),
+      date: date,
+      digest: digest,
+      host: host_for_sig
+    }
+
+    signature = Pleroma.Signature.sign(InternalFetchActor.get_actor(), signature_opts)
+
+    [
+      {'signature', to_charlist(signature)},
+      {'date', date},
+      {'digest', to_charlist(digest)},
+      {'content-length', to_charlist("#{shake_size}")},
+      {to_charlist("(request-target)"), to_charlist(shake)}
+    ]
+  end
+
+  defp host_signature(%{host: host, scheme: scheme, port: port}) do
+    if port == URI.default_port(scheme) do
+      host
+    else
+      "#{host}:#{port}"
+    end
+  end
+end
diff --git a/lib/pleroma/web/fed_sockets/socket_info.ex b/lib/pleroma/web/fed_sockets/socket_info.ex
new file mode 100644
index 000000000..d6fdffe1a
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/socket_info.ex
@@ -0,0 +1,52 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.SocketInfo do
+  defstruct origin: nil,
+            pid: nil,
+            conn_pid: nil,
+            state: :default,
+            connected_until: nil
+
+  alias Pleroma.Web.FedSockets.SocketInfo
+  @default_connection_duration 15 * 60 * 1000
+
+  def build(uri, conn_pid \\ nil) do
+    uri
+    |> build_origin()
+    |> build_pids(conn_pid)
+    |> touch()
+  end
+
+  def touch(%SocketInfo{} = socket_info),
+    do: %{socket_info | connected_until: new_ttl()}
+
+  def connect(%SocketInfo{} = socket_info),
+    do: %{socket_info | state: :connected}
+
+  def expired?(%{connected_until: connected_until}),
+    do: connected_until < :erlang.monotonic_time(:millisecond)
+
+  def origin(uri),
+    do: build_origin(uri).origin
+
+  defp build_pids(socket_info, conn_pid),
+    do: struct(socket_info, pid: self(), conn_pid: conn_pid)
+
+  defp build_origin(uri) when is_binary(uri),
+    do: uri |> URI.parse() |> build_origin
+
+  defp build_origin(%{host: host, port: nil, scheme: scheme}),
+    do: build_origin(%{host: host, port: URI.default_port(scheme)})
+
+  defp build_origin(%{host: host, port: port}),
+    do: %SocketInfo{origin: "#{host}:#{port}"}
+
+  defp new_ttl do
+    connection_duration =
+      Pleroma.Config.get([:fed_sockets, :connection_duration], @default_connection_duration)
+
+    :erlang.monotonic_time(:millisecond) + connection_duration
+  end
+end
diff --git a/lib/pleroma/web/fed_sockets/supervisor.ex b/lib/pleroma/web/fed_sockets/supervisor.ex
new file mode 100644
index 000000000..a5f4bebfb
--- /dev/null
+++ b/lib/pleroma/web/fed_sockets/supervisor.ex
@@ -0,0 +1,59 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.FedSockets.Supervisor do
+  use Supervisor
+  import Cachex.Spec
+
+  def start_link(opts) do
+    Supervisor.start_link(__MODULE__, opts, name: __MODULE__)
+  end
+
+  def init(args) do
+    children = [
+      build_cache(:fed_socket_fetches, args),
+      build_cache(:fed_socket_rejections, args),
+      {Registry, keys: :unique, name: FedSockets.Registry, meta: [rejected: %{}]}
+    ]
+
+    opts = [strategy: :one_for_all, name: Pleroma.Web.Streamer.Supervisor]
+    Supervisor.init(children, opts)
+  end
+
+  defp build_cache(name, args) do
+    opts = get_opts(name, args)
+
+    %{
+      id: String.to_atom("#{name}_cache"),
+      start: {Cachex, :start_link, [name, opts]},
+      type: :worker
+    }
+  end
+
+  defp get_opts(cache_name, args)
+       when cache_name in [:fed_socket_fetches, :fed_socket_rejections] do
+    default = get_opts_or_config(args, cache_name, :default, 15_000)
+    interval = get_opts_or_config(args, cache_name, :interval, 3_000)
+    lazy = get_opts_or_config(args, cache_name, :lazy, false)
+
+    [expiration: expiration(default: default, interval: interval, lazy: lazy)]
+  end
+
+  defp get_opts(name, args) do
+    Keyword.get(args, name, [])
+  end
+
+  defp get_opts_or_config(args, name, key, default) do
+    args
+    |> Keyword.get(name, [])
+    |> Keyword.get(key)
+    |> case do
+      nil ->
+        Pleroma.Config.get([:fed_sockets, name, key], default)
+
+      value ->
+        value
+    end
+  end
+end
-- 
cgit v1.2.3


From 26c571df339ebfdadf760f75871b9343131a2d81 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 18 Sep 2020 16:09:37 +0300
Subject: FedSockets: fix log spam on cached rejects

---
 lib/pleroma/web/fed_sockets/fed_sockets.ex | 3 +++
 1 file changed, 3 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/fed_sockets/fed_sockets.ex b/lib/pleroma/web/fed_sockets/fed_sockets.ex
index 035d54796..1fd5899c8 100644
--- a/lib/pleroma/web/fed_sockets/fed_sockets.ex
+++ b/lib/pleroma/web/fed_sockets/fed_sockets.ex
@@ -104,6 +104,9 @@ defmodule Pleroma.Web.FedSockets do
         Logger.debug("fedsocket found in cache - #{inspect(address)}")
         {:ok, socket}
 
+      {:cache, {:error, :rejected} = e} ->
+        e
+
       {:connect, {:error, _host}} ->
         Logger.debug("set host rejected for - #{inspect(address)}")
         FedRegistry.set_host_rejected(address)
-- 
cgit v1.2.3


From 26859c549c58901bbcc9dde009fd1cb81f6eb292 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 18 Sep 2020 17:49:15 -0500
Subject: Add user agent to fedsocket requests

---
 lib/pleroma/web/fed_sockets/outgoing_handler.ex | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/fed_sockets/outgoing_handler.ex b/lib/pleroma/web/fed_sockets/outgoing_handler.ex
index 6ddef17fe..3ff4a85e3 100644
--- a/lib/pleroma/web/fed_sockets/outgoing_handler.ex
+++ b/lib/pleroma/web/fed_sockets/outgoing_handler.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.FedSockets.OutgoingHandler do
 
   require Logger
 
+  alias Pleroma.Application
   alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.FedSockets
   alias Pleroma.Web.FedSockets.FedRegistry
@@ -87,7 +88,10 @@ defmodule Pleroma.Web.FedSockets.OutgoingHandler do
 
     with {:ok, conn_pid} <- :gun.open(to_charlist(host), port),
          {:ok, _} <- :gun.await_up(conn_pid),
-         reference <- :gun.get(conn_pid, to_charlist(path)),
+         reference <-
+           :gun.get(conn_pid, to_charlist(path), [
+             {'user-agent', to_charlist(Application.user_agent())}
+           ]),
          {:response, :fin, 204, _} <- :gun.await(conn_pid, reference),
          headers <- build_headers(uri),
          ref <- :gun.ws_upgrade(conn_pid, to_charlist(path), headers, %{silence_pings: false}) do
@@ -132,7 +136,8 @@ defmodule Pleroma.Web.FedSockets.OutgoingHandler do
       {'date', date},
       {'digest', to_charlist(digest)},
       {'content-length', to_charlist("#{shake_size}")},
-      {to_charlist("(request-target)"), to_charlist(shake)}
+      {to_charlist("(request-target)"), to_charlist(shake)},
+      {'user-agent', to_charlist(Application.user_agent())}
     ]
   end
 
-- 
cgit v1.2.3


From 51116b539201eb99fa89cabc88a08f92c67bfeca Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 19 Sep 2020 00:50:00 +0300
Subject: OpenAPI: fix various errors pointed out by editor.swagger.io

---
 lib/pleroma/web/api_spec/helpers.ex                             | 6 +++++-
 lib/pleroma/web/api_spec/operations/account_operation.ex        | 7 +++++--
 lib/pleroma/web/api_spec/operations/custom_emoji_operation.ex   | 2 +-
 lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex | 2 +-
 lib/pleroma/web/api_spec/operations/list_operation.ex           | 3 +--
 lib/pleroma/web/api_spec/schemas/chat_message.ex                | 3 ++-
 lib/pleroma/web/api_spec/schemas/scheduled_status.ex            | 4 ++--
 7 files changed, 17 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex
index 2a7f1a706..34de2ed57 100644
--- a/lib/pleroma/web/api_spec/helpers.ex
+++ b/lib/pleroma/web/api_spec/helpers.ex
@@ -72,7 +72,11 @@ defmodule Pleroma.Web.ApiSpec.Helpers do
   end
 
   def empty_array_response do
-    Operation.response("Empty array", "application/json", %Schema{type: :array, example: []})
+    Operation.response("Empty array", "application/json", %Schema{
+      type: :array,
+      items: %Schema{type: :object, example: %{}},
+      example: []
+    })
   end
 
   def no_content_response do
diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index aaebc9b5c..d90ddb787 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -372,6 +372,10 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
       tags: ["accounts"],
       summary: "Identity proofs",
       operationId: "AccountController.identity_proofs",
+      # Validators complains about unused path params otherwise
+      parameters: [
+        %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}
+      ],
       description: "Not implemented",
       responses: %{
         200 => empty_array_response()
@@ -469,7 +473,6 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
         identifier: %Schema{type: :string},
         message: %Schema{type: :string}
       },
-      required: [],
       # Note: example of successful registration with failed login response:
       # example: %{
       #   "identifier" => "missing_confirmed_email",
@@ -530,7 +533,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
           nullable: true,
           oneOf: [
             %Schema{type: :array, items: attribute_field()},
-            %Schema{type: :object, additionalProperties: %Schema{type: attribute_field()}}
+            %Schema{type: :object, additionalProperties: attribute_field()}
           ]
         },
         # NOTE: `source` field is not supported
diff --git a/lib/pleroma/web/api_spec/operations/custom_emoji_operation.ex b/lib/pleroma/web/api_spec/operations/custom_emoji_operation.ex
index 2f812ac77..5ff263ceb 100644
--- a/lib/pleroma/web/api_spec/operations/custom_emoji_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/custom_emoji_operation.ex
@@ -69,7 +69,7 @@ defmodule Pleroma.Web.ApiSpec.CustomEmojiOperation do
           type: :object,
           properties: %{
             category: %Schema{type: :string},
-            tags: %Schema{type: :array}
+            tags: %Schema{type: :array, items: %Schema{type: :string}}
           }
         }
       ],
diff --git a/lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex b/lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex
index 1a49fece0..745d41f88 100644
--- a/lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/emoji_reaction_operation.ex
@@ -23,7 +23,7 @@ defmodule Pleroma.Web.ApiSpec.EmojiReactionOperation do
       parameters: [
         Operation.parameter(:id, :path, FlakeID, "Status ID", required: true),
         Operation.parameter(:emoji, :path, :string, "Filter by a single unicode emoji",
-          required: false
+          required: nil
         )
       ],
       security: [%{"oAuth" => ["read:statuses"]}],
diff --git a/lib/pleroma/web/api_spec/operations/list_operation.ex b/lib/pleroma/web/api_spec/operations/list_operation.ex
index 15039052e..f6e73968a 100644
--- a/lib/pleroma/web/api_spec/operations/list_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/list_operation.ex
@@ -187,8 +187,7 @@ defmodule Pleroma.Web.ApiSpec.ListOperation do
         type: :object,
         properties: %{
           account_ids: %Schema{type: :array, description: "Array of account IDs", items: FlakeID}
-        },
-        required: required && [:account_ids]
+        }
       },
       required: required
     )
diff --git a/lib/pleroma/web/api_spec/schemas/chat_message.ex b/lib/pleroma/web/api_spec/schemas/chat_message.ex
index bbf2a4427..9d2799618 100644
--- a/lib/pleroma/web/api_spec/schemas/chat_message.ex
+++ b/lib/pleroma/web/api_spec/schemas/chat_message.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
   alias OpenApiSpex.Schema
+  alias Pleroma.Web.ApiSpec.Schemas.Emoji
 
   require OpenApiSpex
 
@@ -18,7 +19,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ChatMessage do
       chat_id: %Schema{type: :string},
       content: %Schema{type: :string, nullable: true},
       created_at: %Schema{type: :string, format: :"date-time"},
-      emojis: %Schema{type: :array},
+      emojis: %Schema{type: :array, items: Emoji},
       attachment: %Schema{type: :object, nullable: true},
       card: %Schema{
         type: :object,
diff --git a/lib/pleroma/web/api_spec/schemas/scheduled_status.ex b/lib/pleroma/web/api_spec/schemas/scheduled_status.ex
index 0520d0848..addefa9d3 100644
--- a/lib/pleroma/web/api_spec/schemas/scheduled_status.ex
+++ b/lib/pleroma/web/api_spec/schemas/scheduled_status.ex
@@ -27,9 +27,9 @@ defmodule Pleroma.Web.ApiSpec.Schemas.ScheduledStatus do
           media_ids: %Schema{type: :array, nullable: true, items: %Schema{type: :string}},
           sensitive: %Schema{type: :boolean, nullable: true},
           spoiler_text: %Schema{type: :string, nullable: true},
-          visibility: %Schema{type: VisibilityScope, nullable: true},
+          visibility: %Schema{allOf: [VisibilityScope], nullable: true},
           scheduled_at: %Schema{type: :string, format: :"date-time", nullable: true},
-          poll: %Schema{type: Poll, nullable: true},
+          poll: %Schema{allOf: [Poll], nullable: true},
           in_reply_to_id: %Schema{type: :string, nullable: true}
         }
       }
-- 
cgit v1.2.3


From 4b12e071ac7ea87c91f6192cc0da90c32e846ca4 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 19 Sep 2020 00:50:38 +0300
Subject: OpenAPI: make it possible to generate a spec without starting the app

Needed for api docs generation on pleroma.social that will come later
---
 lib/pleroma/web/api_spec.ex | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex
index 79fd5f871..93a5273e3 100644
--- a/lib/pleroma/web/api_spec.ex
+++ b/lib/pleroma/web/api_spec.ex
@@ -13,10 +13,15 @@ defmodule Pleroma.Web.ApiSpec do
   @impl OpenApi
   def spec do
     %OpenApi{
-      servers: [
-        # Populate the Server info from a phoenix endpoint
-        OpenApiSpex.Server.from_endpoint(Endpoint)
-      ],
+      servers:
+        if Phoenix.Endpoint.server?(:pleroma, Endpoint) do
+          [
+            # Populate the Server info from a phoenix endpoint
+            OpenApiSpex.Server.from_endpoint(Endpoint)
+          ]
+        else
+          []
+        end,
       info: %OpenApiSpex.Info{
         title: "Pleroma",
         description: Application.spec(:pleroma, :description) |> to_string(),
-- 
cgit v1.2.3


From 60b025b782eb27b86a791451149b6690431371dc Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Sat, 19 Sep 2020 19:16:55 +0300
Subject: [#2074] OAuth scope checking in Streaming API.

---
 lib/pleroma/plugs/oauth_scopes_plug.ex            |  2 +-
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 10 ++--
 lib/pleroma/web/streamer/streamer.ex              | 70 +++++++++++++++++------
 3 files changed, 59 insertions(+), 23 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/plugs/oauth_scopes_plug.ex b/lib/pleroma/plugs/oauth_scopes_plug.ex
index efc25b79f..b1a736d78 100644
--- a/lib/pleroma/plugs/oauth_scopes_plug.ex
+++ b/lib/pleroma/plugs/oauth_scopes_plug.ex
@@ -53,7 +53,7 @@ defmodule Pleroma.Plugs.OAuthScopesPlug do
     |> assign(:token, nil)
   end
 
-  @doc "Filters descendants of supported scopes"
+  @doc "Keeps those of `scopes` which are descendants of `supported_scopes`"
   def filter_descendants(scopes, supported_scopes) do
     Enum.filter(
       scopes,
diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index cf923ded8..439cdd716 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -23,8 +23,8 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     with params <- Enum.into(:cow_qs.parse_qs(qs), %{}),
          sec_websocket <- :cowboy_req.header("sec-websocket-protocol", req, nil),
          access_token <- Map.get(params, "access_token"),
-         {:ok, user} <- authenticate_request(access_token, sec_websocket),
-         {:ok, topic} <- Streamer.get_topic(Map.get(params, "stream"), user, params) do
+         {:ok, user, oauth_token} <- authenticate_request(access_token, sec_websocket),
+         {:ok, topic} <- Streamer.get_topic(params["stream"], user, oauth_token, params) do
       req =
         if sec_websocket do
           :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
@@ -117,7 +117,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
 
   # Public streams without authentication.
   defp authenticate_request(nil, nil) do
-    {:ok, nil}
+    {:ok, nil, nil}
   end
 
   # Authenticated streams.
@@ -125,9 +125,9 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
     token = access_token || sec_websocket
 
     with true <- is_bitstring(token),
-         %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
+         oauth_token = %Token{user_id: user_id} <- Repo.get_by(Token, token: token),
          user = %User{} <- User.get_cached_by_id(user_id) do
-      {:ok, user}
+      {:ok, user, oauth_token}
     else
       _ -> {:error, :unauthorized}
     end
diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index d1d70e556..5475f18a6 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -11,10 +11,12 @@ defmodule Pleroma.Web.Streamer do
   alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Plugs.OAuthScopesPlug
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.CommonAPI
+  alias Pleroma.Web.OAuth.Token
   alias Pleroma.Web.StreamerView
 
   @mix_env Mix.env()
@@ -26,53 +28,87 @@ defmodule Pleroma.Web.Streamer do
   @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"]
 
   @doc "Expands and authorizes a stream, and registers the process for streaming."
-  @spec get_topic_and_add_socket(stream :: String.t(), User.t() | nil, Map.t() | nil) ::
+  @spec get_topic_and_add_socket(
+          stream :: String.t(),
+          User.t() | nil,
+          Token.t() | nil,
+          Map.t() | nil
+        ) ::
           {:ok, topic :: String.t()} | {:error, :bad_topic} | {:error, :unauthorized}
-  def get_topic_and_add_socket(stream, user, params \\ %{}) do
-    case get_topic(stream, user, params) do
+  def get_topic_and_add_socket(stream, user, oauth_token, params \\ %{}) do
+    case get_topic(stream, user, oauth_token, params) do
       {:ok, topic} -> add_socket(topic, user)
       error -> error
     end
   end
 
   @doc "Expand and authorizes a stream"
-  @spec get_topic(stream :: String.t(), User.t() | nil, Map.t()) ::
+  @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) ::
           {:ok, topic :: String.t()} | {:error, :bad_topic}
-  def get_topic(stream, user, params \\ %{})
+  def get_topic(stream, user, oauth_token, params \\ %{})
 
   # Allow all public steams.
-  def get_topic(stream, _, _) when stream in @public_streams do
+  def get_topic(stream, _user, _oauth_token, _params) when stream in @public_streams do
     {:ok, stream}
   end
 
   # Allow all hashtags streams.
-  def get_topic("hashtag", _, %{"tag" => tag}) do
+  def get_topic("hashtag", _user, _oauth_token, %{"tag" => tag} = _params) do
     {:ok, "hashtag:" <> tag}
   end
 
   # Expand user streams.
-  def get_topic(stream, %User{} = user, _) when stream in @user_streams do
-    {:ok, stream <> ":" <> to_string(user.id)}
+  def get_topic(
+        stream,
+        %User{id: user_id} = user,
+        %Token{user_id: token_user_id} = oauth_token,
+        _params
+      )
+      when stream in @user_streams and user_id == token_user_id do
+    # Note: "read" works for all user streams (not mentioning it since it's an ancestor scope)
+    required_scopes =
+      if stream == "user:notification" do
+        ["read:notifications"]
+      else
+        ["read:statuses"]
+      end
+
+    if OAuthScopesPlug.filter_descendants(required_scopes, oauth_token.scopes) == [] do
+      {:error, :unauthorized}
+    else
+      {:ok, stream <> ":" <> to_string(user.id)}
+    end
   end
 
-  def get_topic(stream, _, _) when stream in @user_streams do
+  def get_topic(stream, _user, _oauth_token, _params) when stream in @user_streams do
     {:error, :unauthorized}
   end
 
   # List streams.
-  def get_topic("list", %User{} = user, %{"list" => id}) do
-    if Pleroma.List.get(id, user) do
-      {:ok, "list:" <> to_string(id)}
-    else
-      {:error, :bad_topic}
+  def get_topic(
+        "list",
+        %User{id: user_id} = user,
+        %Token{user_id: token_user_id} = oauth_token,
+        %{"list" => id}
+      )
+      when user_id == token_user_id do
+    cond do
+      OAuthScopesPlug.filter_descendants(["read", "read:lists"], oauth_token.scopes) == [] ->
+        {:error, :unauthorized}
+
+      Pleroma.List.get(id, user) ->
+        {:ok, "list:" <> to_string(id)}
+
+      true ->
+        {:error, :bad_topic}
     end
   end
 
-  def get_topic("list", _, _) do
+  def get_topic("list", _user, _oauth_token, _params) do
     {:error, :unauthorized}
   end
 
-  def get_topic(_, _, _) do
+  def get_topic(_stream, _user, _oauth_token, _params) do
     {:error, :bad_topic}
   end
 
-- 
cgit v1.2.3


From f2f0a0260f00e316f62d42e766787b20cc92601a Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Mon, 21 Sep 2020 16:08:38 +0200
Subject: ActivityPub: Don't block-filter your own posts

We are filtering out replies to people you block, but that should
not include your own posts.
---
 lib/pleroma/web/activity_pub/activity_pub.ex | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 06e8e1a7c..aacd58d03 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -841,7 +841,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     from(
       [activity, object: o] in query,
       where: fragment("not (? = ANY(?))", activity.actor, ^blocked_ap_ids),
-      where: fragment("not (? && ?)", activity.recipients, ^blocked_ap_ids),
+      where:
+        fragment(
+          "((not (? && ?)) or ? = ?)",
+          activity.recipients,
+          ^blocked_ap_ids,
+          activity.actor,
+          ^user.ap_id
+        ),
       where:
         fragment(
           "recipients_contain_blocked_domains(?, ?) = false",
-- 
cgit v1.2.3


From 8afdbcdb1c012c37cde3a3d2bea147f95c5111b0 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Mon, 21 Sep 2020 10:13:41 -0500
Subject: Force HTTP for fedsockets

---
 lib/pleroma/web/fed_sockets/outgoing_handler.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/fed_sockets/outgoing_handler.ex b/lib/pleroma/web/fed_sockets/outgoing_handler.ex
index 3ff4a85e3..e235a7c43 100644
--- a/lib/pleroma/web/fed_sockets/outgoing_handler.ex
+++ b/lib/pleroma/web/fed_sockets/outgoing_handler.ex
@@ -86,7 +86,7 @@ defmodule Pleroma.Web.FedSockets.OutgoingHandler do
 
     %{host: host, port: port, path: path} = URI.parse(ws_uri)
 
-    with {:ok, conn_pid} <- :gun.open(to_charlist(host), port),
+    with {:ok, conn_pid} <- :gun.open(to_charlist(host), port, %{protocols: [:http]}),
          {:ok, _} <- :gun.await_up(conn_pid),
          reference <-
            :gun.get(conn_pid, to_charlist(path), [
-- 
cgit v1.2.3