From 9eea80002673eb1359a2d4369c65a89c42c2f707 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 28 May 2020 10:16:09 -0500
Subject: Refactor notification settings

---
 lib/pleroma/notification.ex                 | 29 +++++++++--------------------
 lib/pleroma/user/notification_setting.ex    | 14 ++++++--------
 lib/pleroma/web/api_spec/schemas/account.ex | 14 ++++++--------
 3 files changed, 21 insertions(+), 36 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 7eca55ac9..ca556f0bb 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -459,10 +459,9 @@ defmodule Pleroma.Notification do
   def skip?(%Activity{} = activity, %User{} = user) do
     [
       :self,
-      :followers,
-      :follows,
-      :non_followers,
-      :non_follows,
+      :from_followers,
+      :from_following,
+      :from_strangers,
       :recently_followed
     ]
     |> Enum.find(&skip?(&1, activity, user))
@@ -476,9 +475,9 @@ defmodule Pleroma.Notification do
   end
 
   def skip?(
-        :followers,
+        :from_followers,
         %Activity{} = activity,
-        %User{notification_settings: %{followers: false}} = user
+        %User{notification_settings: %{from_followers: false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
@@ -486,9 +485,9 @@ defmodule Pleroma.Notification do
   end
 
   def skip?(
-        :non_followers,
+        :from_strangers,
         %Activity{} = activity,
-        %User{notification_settings: %{non_followers: false}} = user
+        %User{notification_settings: %{from_strangers: false}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
@@ -496,25 +495,15 @@ defmodule Pleroma.Notification do
   end
 
   def skip?(
-        :follows,
+        :from_following,
         %Activity{} = activity,
-        %User{notification_settings: %{follows: false}} = user
+        %User{notification_settings: %{from_following: false}} = user
       ) do
     actor = activity.data["actor"]
     followed = User.get_cached_by_ap_id(actor)
     User.following?(user, followed)
   end
 
-  def skip?(
-        :non_follows,
-        %Activity{} = activity,
-        %User{notification_settings: %{non_follows: false}} = user
-      ) do
-    actor = activity.data["actor"]
-    followed = User.get_cached_by_ap_id(actor)
-    !User.following?(user, followed)
-  end
-
   # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL
   def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do
     actor = activity.data["actor"]
diff --git a/lib/pleroma/user/notification_setting.ex b/lib/pleroma/user/notification_setting.ex
index 4bd55e139..e47ac4cab 100644
--- a/lib/pleroma/user/notification_setting.ex
+++ b/lib/pleroma/user/notification_setting.ex
@@ -10,20 +10,18 @@ defmodule Pleroma.User.NotificationSetting do
   @primary_key false
 
   embedded_schema do
-    field(:followers, :boolean, default: true)
-    field(:follows, :boolean, default: true)
-    field(:non_follows, :boolean, default: true)
-    field(:non_followers, :boolean, default: true)
+    field(:from_followers, :boolean, default: true)
+    field(:from_following, :boolean, default: true)
+    field(:from_strangers, :boolean, default: true)
     field(:privacy_option, :boolean, default: false)
   end
 
   def changeset(schema, params) do
     schema
     |> cast(prepare_attrs(params), [
-      :followers,
-      :follows,
-      :non_follows,
-      :non_followers,
+      :from_followers,
+      :from_following,
+      :from_strangers,
       :privacy_option
     ])
   end
diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex
index d54e2158d..ed90ef3db 100644
--- a/lib/pleroma/web/api_spec/schemas/account.ex
+++ b/lib/pleroma/web/api_spec/schemas/account.ex
@@ -57,10 +57,9 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
           notification_settings: %Schema{
             type: :object,
             properties: %{
-              followers: %Schema{type: :boolean},
-              follows: %Schema{type: :boolean},
-              non_followers: %Schema{type: :boolean},
-              non_follows: %Schema{type: :boolean},
+              from_followers: %Schema{type: :boolean},
+              from_following: %Schema{type: :boolean},
+              from_strangers: %Schema{type: :boolean},
               privacy_option: %Schema{type: :boolean}
             }
           },
@@ -123,10 +122,9 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
         "unread_conversation_count" => 0,
         "tags" => [],
         "notification_settings" => %{
-          "followers" => true,
-          "follows" => true,
-          "non_followers" => true,
-          "non_follows" => true,
+          "from_followers" => true,
+          "from_following" => true,
+          "from_strangers" => true,
           "privacy_option" => false
         },
         "relationship" => %{
-- 
cgit v1.2.3


From fd5e797379155e5a85deb88dc79f8fbca483948e Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 26 Jun 2020 11:24:28 -0500
Subject: Simplify notification filtering settings further

---
 lib/pleroma/notification.ex                 | 28 +++-------------------------
 lib/pleroma/user/notification_setting.ex    |  8 ++------
 lib/pleroma/web/api_spec/schemas/account.ex |  8 ++------
 3 files changed, 7 insertions(+), 37 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index 9d09cf082..8a28a1821 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -550,9 +550,7 @@ defmodule Pleroma.Notification do
     [
       :self,
       :invisible,
-      :from_followers,
-      :from_following,
-      :from_strangers,
+      :block_from_strangers,
       :recently_followed
     ]
     |> Enum.find(&skip?(&1, activity, user))
@@ -572,35 +570,15 @@ defmodule Pleroma.Notification do
   end
 
   def skip?(
-        :from_followers,
+        :block_from_strangers,
         %Activity{} = activity,
-        %User{notification_settings: %{from_followers: false}} = user
-      ) do
-    actor = activity.data["actor"]
-    follower = User.get_cached_by_ap_id(actor)
-    User.following?(follower, user)
-  end
-
-  def skip?(
-        :from_strangers,
-        %Activity{} = activity,
-        %User{notification_settings: %{from_strangers: false}} = user
+        %User{notification_settings: %{block_from_strangers: true}} = user
       ) do
     actor = activity.data["actor"]
     follower = User.get_cached_by_ap_id(actor)
     !User.following?(follower, user)
   end
 
-  def skip?(
-        :from_following,
-        %Activity{} = activity,
-        %User{notification_settings: %{from_following: false}} = user
-      ) do
-    actor = activity.data["actor"]
-    followed = User.get_cached_by_ap_id(actor)
-    User.following?(user, followed)
-  end
-
   # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL
   def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do
     actor = activity.data["actor"]
diff --git a/lib/pleroma/user/notification_setting.ex b/lib/pleroma/user/notification_setting.ex
index e47ac4cab..ffe9860de 100644
--- a/lib/pleroma/user/notification_setting.ex
+++ b/lib/pleroma/user/notification_setting.ex
@@ -10,18 +10,14 @@ defmodule Pleroma.User.NotificationSetting do
   @primary_key false
 
   embedded_schema do
-    field(:from_followers, :boolean, default: true)
-    field(:from_following, :boolean, default: true)
-    field(:from_strangers, :boolean, default: true)
+    field(:block_from_strangers, :boolean, default: false)
     field(:privacy_option, :boolean, default: false)
   end
 
   def changeset(schema, params) do
     schema
     |> cast(prepare_attrs(params), [
-      :from_followers,
-      :from_following,
-      :from_strangers,
+      :block_from_strangers,
       :privacy_option
     ])
   end
diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex
index ed90ef3db..91bb1ba88 100644
--- a/lib/pleroma/web/api_spec/schemas/account.ex
+++ b/lib/pleroma/web/api_spec/schemas/account.ex
@@ -57,9 +57,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
           notification_settings: %Schema{
             type: :object,
             properties: %{
-              from_followers: %Schema{type: :boolean},
-              from_following: %Schema{type: :boolean},
-              from_strangers: %Schema{type: :boolean},
+              block_from_strangers: %Schema{type: :boolean},
               privacy_option: %Schema{type: :boolean}
             }
           },
@@ -122,9 +120,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
         "unread_conversation_count" => 0,
         "tags" => [],
         "notification_settings" => %{
-          "from_followers" => true,
-          "from_following" => true,
-          "from_strangers" => true,
+          "block_from_strangers" => false,
           "privacy_option" => false
         },
         "relationship" => %{
-- 
cgit v1.2.3


From 69848d5c97c9e5d4c14fb5613eb174cb81d5026d Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 26 Jun 2020 12:45:46 -0500
Subject: Rename notification "privacy_option" setting

---
 lib/mix/tasks/pleroma/notification_settings.ex | 18 +++++++++---------
 lib/pleroma/user/notification_setting.ex       |  4 ++--
 lib/pleroma/web/api_spec/schemas/account.ex    |  4 ++--
 lib/pleroma/web/push/impl.ex                   |  2 +-
 4 files changed, 14 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/notification_settings.ex b/lib/mix/tasks/pleroma/notification_settings.ex
index 7d65f0587..00f5ba7bf 100644
--- a/lib/mix/tasks/pleroma/notification_settings.ex
+++ b/lib/mix/tasks/pleroma/notification_settings.ex
@@ -3,8 +3,8 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do
   @moduledoc """
   Example:
 
-  > mix pleroma.notification_settings --privacy-option=false --nickname-users="parallel588"  # set false only for parallel588 user
-  > mix pleroma.notification_settings --privacy-option=true # set true for all users
+  > mix pleroma.notification_settings --hide-notification-contents=false --nickname-users="parallel588"  # set false only for parallel588 user
+  > mix pleroma.notification_settings --hide-notification-contents=true # set true for all users
 
   """
 
@@ -19,16 +19,16 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do
       OptionParser.parse(
         args,
         strict: [
-          privacy_option: :boolean,
+          hide_notification_contents: :boolean,
           email_users: :string,
           nickname_users: :string
         ]
       )
 
-    privacy_option = Keyword.get(options, :privacy_option)
+    hide_notification_contents = Keyword.get(options, :hide_notification_contents)
 
-    if not is_nil(privacy_option) do
-      privacy_option
+    if not is_nil(hide_notification_contents) do
+      hide_notification_contents
       |> build_query(options)
       |> Pleroma.Repo.update_all([])
     end
@@ -36,15 +36,15 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do
     shell_info("Done")
   end
 
-  defp build_query(privacy_option, options) do
+  defp build_query(hide_notification_contents, options) do
     query =
       from(u in Pleroma.User,
         update: [
           set: [
             notification_settings:
               fragment(
-                "jsonb_set(notification_settings, '{privacy_option}', ?)",
-                ^privacy_option
+                "jsonb_set(notification_settings, '{hide_notification_contents}', ?)",
+                ^hide_notification_contents
               )
           ]
         ]
diff --git a/lib/pleroma/user/notification_setting.ex b/lib/pleroma/user/notification_setting.ex
index ffe9860de..7d9e8a000 100644
--- a/lib/pleroma/user/notification_setting.ex
+++ b/lib/pleroma/user/notification_setting.ex
@@ -11,14 +11,14 @@ defmodule Pleroma.User.NotificationSetting do
 
   embedded_schema do
     field(:block_from_strangers, :boolean, default: false)
-    field(:privacy_option, :boolean, default: false)
+    field(:hide_notification_contents, :boolean, default: false)
   end
 
   def changeset(schema, params) do
     schema
     |> cast(prepare_attrs(params), [
       :block_from_strangers,
-      :privacy_option
+      :hide_notification_contents
     ])
   end
 
diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex
index 91bb1ba88..71d402b18 100644
--- a/lib/pleroma/web/api_spec/schemas/account.ex
+++ b/lib/pleroma/web/api_spec/schemas/account.ex
@@ -58,7 +58,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
             type: :object,
             properties: %{
               block_from_strangers: %Schema{type: :boolean},
-              privacy_option: %Schema{type: :boolean}
+              hide_notification_contents: %Schema{type: :boolean}
             }
           },
           relationship: AccountRelationship,
@@ -121,7 +121,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do
         "tags" => [],
         "notification_settings" => %{
           "block_from_strangers" => false,
-          "privacy_option" => false
+          "hide_notification_contents" => false
         },
         "relationship" => %{
           "blocked_by" => false,
diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex
index cdb827e76..16368485e 100644
--- a/lib/pleroma/web/push/impl.ex
+++ b/lib/pleroma/web/push/impl.ex
@@ -104,7 +104,7 @@ defmodule Pleroma.Web.Push.Impl do
 
   def build_content(
         %{
-          user: %{notification_settings: %{privacy_option: true}}
+          user: %{notification_settings: %{hide_notification_contents: true}}
         } = notification,
         _actor,
         _object,
-- 
cgit v1.2.3


From 8daacc911498d827fd68ea3d34eb1be9ae4a1ffe Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Tue, 23 Jun 2020 14:17:23 -0500
Subject: AutoLinker --> Linkify, update to latest version
 https://git.pleroma.social/pleroma/elixir-libraries/linkify

---
 lib/pleroma/config/config_db.ex       |  2 +-
 lib/pleroma/formatter.ex              | 26 +++++++++++++++-----------
 lib/pleroma/web/rich_media/helpers.ex |  4 ++--
 3 files changed, 18 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index 1a89d8895..f8141ced8 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -156,7 +156,7 @@ defmodule Pleroma.ConfigDB do
       {:quack, :meta},
       {:mime, :types},
       {:cors_plug, [:max_age, :methods, :expose, :headers]},
-      {:auto_linker, :opts},
+      {:linkify, :opts},
       {:swarm, :node_blacklist},
       {:logger, :backends}
     ]
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 02a93a8dc..0c450eae4 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -10,11 +10,15 @@ defmodule Pleroma.Formatter do
   @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
 
-  @auto_linker_config hashtag: true,
-                      hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
-                      mention: true,
-                      mention_handler: &Pleroma.Formatter.mention_handler/4,
-                      scheme: true
+  defp linkify_opts do
+    Pleroma.Config.get(Pleroma.Formatter) ++
+      [
+        hashtag: true,
+        hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
+        mention: true,
+        mention_handler: &Pleroma.Formatter.mention_handler/4
+      ]
+  end
 
   def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do
     case User.get_cached_by_nickname(nickname) do
@@ -80,19 +84,19 @@ defmodule Pleroma.Formatter do
   @spec linkify(String.t(), keyword()) ::
           {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
   def linkify(text, options \\ []) do
-    options = options ++ @auto_linker_config
+    options = linkify_opts() ++ options
 
     if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
       %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
 
-      {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options)
-      {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options)
+      {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options)
+      {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options)
 
       {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)}
     else
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
-      {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
+      {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options)
 
       {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
     end
@@ -111,9 +115,9 @@ defmodule Pleroma.Formatter do
 
     if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
       %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
-      AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options)
+      Linkify.link(mentions, options) <> Linkify.link(rest, options)
     else
-      AutoLinker.link(text, options)
+      Linkify.link(text, options)
     end
   end
 
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 1729141e9..747f2dc6b 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do
 
   @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
-    validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
+    validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
 
     page_url
-    |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld)
+    |> Linkify.Parser.url?(validate_tld: validate_tld)
     |> parse_uri(page_url)
   end
 
-- 
cgit v1.2.3


From 62fc8eab0dfd3f4c60c8f36fd3a544d6785ff2c6 Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Sat, 11 Jul 2020 07:20:35 +0300
Subject: fix reset confirmation email in admin section

---
 lib/pleroma/application_requirements.ex            | 18 +++++++++++++++++
 lib/pleroma/user.ex                                | 22 ++++++++++++---------
 .../admin_api/controllers/admin_api_controller.ex  | 23 +++++++++-------------
 3 files changed, 40 insertions(+), 23 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex
index 88575a498..f0f34734e 100644
--- a/lib/pleroma/application_requirements.ex
+++ b/lib/pleroma/application_requirements.ex
@@ -16,6 +16,7 @@ defmodule Pleroma.ApplicationRequirements do
   @spec verify!() :: :ok | VerifyError.t()
   def verify! do
     :ok
+    |> check_confirmation_accounts!
     |> check_migrations_applied!()
     |> check_rum!()
     |> handle_result()
@@ -24,6 +25,23 @@ defmodule Pleroma.ApplicationRequirements do
   defp handle_result(:ok), do: :ok
   defp handle_result({:error, message}), do: raise(VerifyError, message: message)
 
+  # Checks account confirmation email
+  #
+  def check_confirmation_accounts!(:ok) do
+    if Pleroma.Config.get([:instance, :account_activation_required]) &&
+         not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
+      Logger.error(
+        "To use confirmation an user account need to enable and setting mailer.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable mailer."
+      )
+
+      {:error, "Confirmation account: Mailer is disabled"}
+    else
+      :ok
+    end
+  end
+
+  def check_confirmation_accounts!(result), do: result
+
   # Checks for pending migrations.
   #
   def check_migrations_applied!(:ok) do
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index b9989f901..711258ac7 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -709,21 +709,25 @@ defmodule Pleroma.User do
     end
   end
 
-  def try_send_confirmation_email(%User{} = user) do
-    if user.confirmation_pending &&
-         Config.get([:instance, :account_activation_required]) do
-      user
-      |> Pleroma.Emails.UserEmail.account_confirmation_email()
-      |> Pleroma.Emails.Mailer.deliver_async()
-
+  @spec try_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop}
+  def try_send_confirmation_email(%User{confirmation_pending: true} = user) do
+    if Config.get([:instance, :account_activation_required]) do
+      send_confirmation_email(user)
       {:ok, :enqueued}
     else
       {:ok, :noop}
     end
   end
 
-  def try_send_confirmation_email(users) do
-    Enum.each(users, &try_send_confirmation_email/1)
+  def try_send_confirmation_email(_), do: {:ok, :noop}
+
+  @spec send_confirmation_email(Uset.t()) :: User.t()
+  def send_confirmation_email(%User{} = user) do
+    user
+    |> Pleroma.Emails.UserEmail.account_confirmation_email()
+    |> Pleroma.Emails.Mailer.deliver_async()
+
+    user
   end
 
   def needs_update?(%User{local: true}), do: false
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index e5f14269a..c10181bae 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -616,29 +616,24 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
   end
 
   def confirm_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
 
     User.toggle_confirmation(users)
 
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "confirm_email"
-    })
+    ModerationLog.insert_log(%{actor: admin, subject: users, action: "confirm_email"})
 
     json(conn, "")
   end
 
   def resend_confirmation_email(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
-    users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
-
-    User.try_send_confirmation_email(users)
+    users =
+      Enum.map(nicknames, fn nickname ->
+        nickname
+        |> User.get_cached_by_nickname()
+        |> User.send_confirmation_email()
+      end)
 
-    ModerationLog.insert_log(%{
-      actor: admin,
-      subject: users,
-      action: "resend_confirmation_email"
-    })
+    ModerationLog.insert_log(%{actor: admin, subject: users, action: "resend_confirmation_email"})
 
     json(conn, "")
   end
-- 
cgit v1.2.3


From 2aac92e9e05ba76903795cdddea652d7e444e701 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 13 Jul 2020 14:27:25 +0200
Subject: Transmogrifier.fix_in_reply_to/2: Use warn for non-fatal fail to get
 replied-to post

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 884646ceb..168422c93 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -176,7 +176,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         |> Map.drop(["conversation"])
       else
         e ->
-          Logger.error("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
+          Logger.warn("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}")
           object
       end
     else
-- 
cgit v1.2.3


From ce243b107ffaf79fee0377998320d90c30dd77e0 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 13 Jul 2020 14:23:03 +0200
Subject: Use Logger.info for {:reject, reason}

---
 lib/pleroma/object/fetcher.ex                | 4 ++++
 lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++++
 2 files changed, 8 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 3e2949ee2..e74c87269 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -124,6 +124,10 @@ defmodule Pleroma.Object.Fetcher do
       {:error, "Object has been deleted"} ->
         nil
 
+      {:reject, reason} ->
+        Logger.info("Rejected #{id} while fetching: #{inspect(reason)}")
+        nil
+
       e ->
         Logger.error("Error while fetching #{id}: #{inspect(e)}")
         nil
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index bc7b5d95a..a4db1d87c 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1370,6 +1370,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         Logger.debug("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
 
+      {:error, {:reject, reason} = e} ->
+        Logger.info("Rejected user #{ap_id}: #{inspect(reason)}")
+        {:error, e}
+
       {:error, e} ->
         Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
         {:error, e}
-- 
cgit v1.2.3


From 858d9fc7e8e722604676c90cf2707f0209f935ec Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Mon, 13 Jul 2020 15:47:13 +0200
Subject: MRF Policies: Return a {:reject, reason} instead of {:reject, nil}

---
 .../web/activity_pub/mrf/anti_followbot_policy.ex      |  2 +-
 .../web/activity_pub/mrf/anti_link_spam_policy.ex      |  7 +++----
 lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex  |  4 ++--
 lib/pleroma/web/activity_pub/mrf/keyword_policy.ex     |  7 ++++---
 lib/pleroma/web/activity_pub/mrf/mention_policy.ex     |  5 +++--
 lib/pleroma/web/activity_pub/mrf/object_age_policy.ex  |  8 +++-----
 lib/pleroma/web/activity_pub/mrf/reject_non_public.ex  |  2 +-
 lib/pleroma/web/activity_pub/mrf/simple_policy.ex      | 16 ++++++++++------
 lib/pleroma/web/activity_pub/mrf/tag_policy.ex         |  7 ++++---
 .../web/activity_pub/mrf/user_allow_list_policy.ex     |  2 +-
 lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex  | 18 +++++++++++-------
 11 files changed, 43 insertions(+), 35 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
index 0270b96ae..b96388489 100644
--- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex
@@ -60,7 +60,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do
     if score < 0.8 do
       {:ok, message}
     else
-      {:reject, nil}
+      {:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
index a7e187b5e..b22464111 100644
--- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex
@@ -39,14 +39,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do
         {:ok, message}
 
       {:old_user, false} ->
-        {:reject, nil}
+        {:reject, "[AntiLinkSpamPolicy] User has no posts nor followers"}
 
       {:error, _} ->
-        {:reject, nil}
+        {:reject, "[AntiLinkSpamPolicy] Failed to get or fetch user by ap_id"}
 
       e ->
-        Logger.warn("[MRF anti-link-spam] WTF: unhandled error #{inspect(e)}")
-        {:reject, nil}
+        {:reject, "[AntiLinkSpamPolicy] Unhandled error #{inspect(e)}"}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
index f6b2c4415..9ba07b4e3 100644
--- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex
@@ -43,7 +43,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
   defp reject_message(message, threshold) when threshold > 0 do
     with {_, recipients} <- get_recipient_count(message) do
       if recipients > threshold do
-        {:reject, nil}
+        {:reject, "[HellthreadPolicy] #{recipients} recipients is over the limit of #{threshold}"}
       else
         {:ok, message}
       end
@@ -87,7 +87,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do
          {:ok, message} <- delist_message(message, delist_threshold) do
       {:ok, message}
     else
-      _e -> {:reject, nil}
+      e -> e
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
index 88b0d2b39..15e09dcf0 100644
--- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
     if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern ->
          string_matches?(content, pattern) or string_matches?(summary, pattern)
        end) do
-      {:reject, nil}
+      {:reject, "[KeywordPolicy] Matches with rejected keyword"}
     else
       {:ok, message}
     end
@@ -89,8 +89,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do
          {:ok, message} <- check_replace(message) do
       {:ok, message}
     else
-      _e ->
-        {:reject, nil}
+      {:reject, nil} -> {:reject, "[KeywordPolicy] "}
+      {:reject, _} = e -> e
+      _e -> {:reject, "[KeywordPolicy] "}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
index 06f003921..7910ca131 100644
--- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex
@@ -12,8 +12,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do
     reject_actors = Pleroma.Config.get([:mrf_mention, :actors], [])
     recipients = (message["to"] || []) ++ (message["cc"] || [])
 
-    if Enum.any?(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do
-      {:reject, nil}
+    if rejected_mention =
+         Enum.find(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do
+      {:reject, "[MentionPolicy] Rejected for mention of #{rejected_mention}"}
     else
       {:ok, message}
     end
diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
index a62914135..5f111c72f 100644
--- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex
@@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
 
   defp check_reject(message, actions) do
     if :reject in actions do
-      {:reject, nil}
+      {:reject, "[ObjectAgePolicy]"}
     else
       {:ok, message}
     end
@@ -47,9 +47,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
 
         {:ok, message}
       else
-        # Unhandleable error: somebody is messing around, just drop the message.
         _e ->
-          {:reject, nil}
+          {:reject, "[ObjectAgePolicy] Unhandled error"}
       end
     else
       {:ok, message}
@@ -69,9 +68,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do
 
         {:ok, message}
       else
-        # Unhandleable error: somebody is messing around, just drop the message.
         _e ->
-          {:reject, nil}
+          {:reject, "[ObjectAgePolicy] Unhandled error"}
       end
     else
       {:ok, message}
diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex
index 4fd63106d..0b9ed2224 100644
--- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex
+++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex
@@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do
         {:ok, object}
 
       true ->
-        {:reject, nil}
+        {:reject, "[RejectNonPublic] visibility: #{visibility}"}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 70a2ca053..b77b8c7b4 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -21,7 +21,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
       accepts == [] -> {:ok, object}
       actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object}
       MRF.subdomain_match?(accepts, actor_host) -> {:ok, object}
-      true -> {:reject, nil}
+      true -> {:reject, "[SimplePolicy] host not in accept list"}
     end
   end
 
@@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
       |> MRF.subdomains_regex()
 
     if MRF.subdomain_match?(rejects, actor_host) do
-      {:reject, nil}
+      {:reject, "[SimplePolicy] host in reject list"}
     else
       {:ok, object}
     end
@@ -114,7 +114,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
       |> MRF.subdomains_regex()
 
     if MRF.subdomain_match?(report_removal, actor_host) do
-      {:reject, nil}
+      {:reject, "[SimplePolicy] host in report_removal list"}
     else
       {:ok, object}
     end
@@ -159,7 +159,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
       |> MRF.subdomains_regex()
 
     if MRF.subdomain_match?(reject_deletes, actor_host) do
-      {:reject, nil}
+      {:reject, "[SimplePolicy] host in reject_deletes list"}
     else
       {:ok, object}
     end
@@ -177,7 +177,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
          {:ok, object} <- check_report_removal(actor_info, object) do
       {:ok, object}
     else
-      _e -> {:reject, nil}
+      {:reject, nil} -> {:reject, "[SimplePolicy]"}
+      {:reject, _} = e -> e
+      _ -> {:reject, "[SimplePolicy]"}
     end
   end
 
@@ -191,7 +193,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
          {:ok, object} <- check_banner_removal(actor_info, object) do
       {:ok, object}
     else
-      _e -> {:reject, nil}
+      {:reject, nil} -> {:reject, "[SimplePolicy]"}
+      {:reject, _} = e -> e
+      _ -> {:reject, "[SimplePolicy]"}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex
index c310462cb..febabda08 100644
--- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex
@@ -134,12 +134,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do
     if user.local == true do
       {:ok, message}
     else
-      {:reject, nil}
+      {:reject,
+       "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-remote-subscription"}
     end
   end
 
-  defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow"}),
-    do: {:reject, nil}
+  defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow", "actor" => actor}),
+    do: {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-any-subscription"}
 
   defp process_tag(_, message), do: {:ok, message}
 
diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
index 651aed70f..1a28f2ba2 100644
--- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do
     if actor in allow_list do
       {:ok, object}
     else
-      {:reject, nil}
+      {:reject, "[UserAllowListPolicy] #{actor} not in the list"}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex
index 6167a74e2..a6c545570 100644
--- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex
@@ -11,22 +11,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do
     with {:ok, _} <- filter(child_message) do
       {:ok, message}
     else
-      {:reject, nil} ->
-        {:reject, nil}
+      {:reject, _} = e -> e
     end
   end
 
   def filter(%{"type" => message_type} = message) do
     with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]),
          rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]),
-         true <-
-           Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type),
-         false <-
-           length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type),
+         {_, true} <-
+           {:accepted,
+            Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type)},
+         {_, false} <-
+           {:rejected,
+            length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type)},
          {:ok, _} <- filter(message["object"]) do
       {:ok, message}
     else
-      _ -> {:reject, nil}
+      {:reject, _} = e -> e
+      {:accepted, _} -> {:reject, "[VocabularyPolicy] #{message_type} not in accept list"}
+      {:rejected, _} -> {:reject, "[VocabularyPolicy] #{message_type} in reject list"}
+      _ -> {:reject, "[VocabularyPolicy]"}
     end
   end
 
-- 
cgit v1.2.3


From 37297a8482eedbb0a3adab2748b3e76401d87e4a Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 14 Jul 2020 13:12:16 -0500
Subject: Improve error messages

---
 lib/pleroma/application_requirements.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex
index f0f34734e..d51160b82 100644
--- a/lib/pleroma/application_requirements.ex
+++ b/lib/pleroma/application_requirements.ex
@@ -31,10 +31,10 @@ defmodule Pleroma.ApplicationRequirements do
     if Pleroma.Config.get([:instance, :account_activation_required]) &&
          not Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do
       Logger.error(
-        "To use confirmation an user account need to enable and setting mailer.\nIf you want to start Pleroma anyway, set\nconfig :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable mailer."
+        "Account activation enabled, but no Mailer settings enabled.\nPlease set config :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable Mailer."
       )
 
-      {:error, "Confirmation account: Mailer is disabled"}
+      {:error, "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."}
     else
       :ok
     end
-- 
cgit v1.2.3


From 777a7edc6b4bf8b9e0ff3b86bdb780f8f2ae2610 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 14 Jul 2020 13:15:37 -0500
Subject: Lint and fix test to match new log message

---
 lib/pleroma/application_requirements.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex
index d51160b82..ee88c3346 100644
--- a/lib/pleroma/application_requirements.ex
+++ b/lib/pleroma/application_requirements.ex
@@ -34,7 +34,8 @@ defmodule Pleroma.ApplicationRequirements do
         "Account activation enabled, but no Mailer settings enabled.\nPlease set config :pleroma, :instance, account_activation_required: false\nOtherwise setup and enable Mailer."
       )
 
-      {:error, "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."}
+      {:error,
+       "Account activation enabled, but Mailer is disabled. Cannot send confirmation emails."}
     else
       :ok
     end
-- 
cgit v1.2.3


From 1dd767b8c7b7565ad94ccb85324e97fa9885923e Mon Sep 17 00:00:00 2001
From: Maksim Pechnikov <parallel588@gmail.com>
Date: Tue, 14 Jul 2020 21:44:08 +0300
Subject: Include port in host for signatures

---
 lib/pleroma/web/activity_pub/publisher.ex | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex
index b70cbd043..d88f7f3ee 100644
--- a/lib/pleroma/web/activity_pub/publisher.ex
+++ b/lib/pleroma/web/activity_pub/publisher.ex
@@ -49,7 +49,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
   """
   def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do
     Logger.debug("Federating #{id} to #{inbox}")
-    %{host: host, path: path} = URI.parse(inbox)
+
+    uri = URI.parse(inbox)
 
     digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64())
 
@@ -57,8 +58,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
 
     signature =
       Pleroma.Signature.sign(actor, %{
-        "(request-target)": "post #{path}",
-        host: host,
+        "(request-target)": "post #{uri.path}",
+        host: signature_host(uri),
         "content-length": byte_size(json),
         digest: digest,
         date: date
@@ -76,8 +77,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
                  {"digest", digest}
                ]
              ) do
-      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
-        do: Instances.set_reachable(inbox)
+      if not Map.has_key?(params, :unreachable_since) || params[:unreachable_since] do
+        Instances.set_reachable(inbox)
+      end
 
       result
     else
@@ -96,6 +98,14 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
     |> publish_one()
   end
 
+  defp signature_host(%URI{port: port, scheme: scheme, host: host}) do
+    if port == URI.default_port(scheme) do
+      host
+    else
+      "#{host}:#{port}"
+    end
+  end
+
   defp should_federate?(inbox, public) do
     if public do
       true
-- 
cgit v1.2.3


From 58a4f350a8bc361d793cb96442f856362c18f195 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 6 May 2020 01:51:10 +0300
Subject: Refactor gun pooling and simplify adapter option insertion

This patch refactors gun pooling to use Elixir process registry and
simplifies adapter option insertion.

Having the pool use process registry instead of a GenServer has a number of advantages:
- Simpler code: the initial implementation adds about half the lines of code it deletes
- Concurrency: unlike a GenServer, ETS-based registry can handle multiple checkout/checkin
requests at the same time
- Precise and easy idle connection clousure: current proposal for closing idle connections in
the GenServer-based pool needs to filter through all connections once a minute and compare their
last active time with closing time. With Elixir process registry this can be done
by just using `Process.send_after`/`Process.cancel_timer` in the worker process.
- Lower memory footprint: In my tests `gun-memory-leak` branch uses about 290mb on peak load (250 connections)
and 235mb on idle (5-10 connections). Registry-based pool uses 210mb on idle and 240mb on peak load
---
 lib/pleroma/application.ex                 |   8 +-
 lib/pleroma/gun/conn.ex                    |  78 +-------
 lib/pleroma/gun/connection_pool.ex         | 129 +++++++++++++
 lib/pleroma/gun/connection_pool/worker.ex  |  95 ++++++++++
 lib/pleroma/http/adapter_helper.ex         | 133 ++++++++++++--
 lib/pleroma/http/adapter_helper/default.ex |  17 ++
 lib/pleroma/http/adapter_helper/gun.ex     |  32 +---
 lib/pleroma/http/adapter_helper/hackney.ex |   3 +
 lib/pleroma/http/connection.ex             | 124 -------------
 lib/pleroma/http/http.ex                   |  53 ++----
 lib/pleroma/pool/connections.ex            | 283 -----------------------------
 lib/pleroma/pool/pool.ex                   |  22 ---
 lib/pleroma/pool/request.ex                |  65 -------
 lib/pleroma/pool/supervisor.ex             |  42 -----
 lib/pleroma/reverse_proxy/client/tesla.ex  |   2 +-
 15 files changed, 400 insertions(+), 686 deletions(-)
 create mode 100644 lib/pleroma/gun/connection_pool.ex
 create mode 100644 lib/pleroma/gun/connection_pool/worker.ex
 create mode 100644 lib/pleroma/http/adapter_helper/default.ex
 delete mode 100644 lib/pleroma/http/connection.ex
 delete mode 100644 lib/pleroma/pool/connections.ex
 delete mode 100644 lib/pleroma/pool/pool.ex
 delete mode 100644 lib/pleroma/pool/request.ex
 delete mode 100644 lib/pleroma/pool/supervisor.ex

(limited to 'lib')

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 3282c6882..be14c1f9f 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -223,9 +223,7 @@ defmodule Pleroma.Application do
 
   # start hackney and gun pools in tests
   defp http_children(_, :test) do
-    hackney_options = Config.get([:hackney_pools, :federation])
-    hackney_pool = :hackney_pool.child_spec(:federation, hackney_options)
-    [hackney_pool, Pleroma.Pool.Supervisor]
+    http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil)
   end
 
   defp http_children(Tesla.Adapter.Hackney, _) do
@@ -244,7 +242,9 @@ defmodule Pleroma.Application do
     end
   end
 
-  defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor]
+  defp http_children(Tesla.Adapter.Gun, _) do
+    [{Registry, keys: :unique, name: Pleroma.Gun.ConnectionPool}]
+  end
 
   defp http_children(_, _), do: []
 end
diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index cd25a2e74..77f78c7ff 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -3,40 +3,11 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.Gun.Conn do
-  @moduledoc """
-  Struct for gun connection data
-  """
   alias Pleroma.Gun
-  alias Pleroma.Pool.Connections
 
   require Logger
 
-  @type gun_state :: :up | :down
-  @type conn_state :: :active | :idle
-
-  @type t :: %__MODULE__{
-          conn: pid(),
-          gun_state: gun_state(),
-          conn_state: conn_state(),
-          used_by: [pid()],
-          last_reference: pos_integer(),
-          crf: float(),
-          retries: pos_integer()
-        }
-
-  defstruct conn: nil,
-            gun_state: :open,
-            conn_state: :init,
-            used_by: [],
-            last_reference: 0,
-            crf: 1,
-            retries: 0
-
-  @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil
-  def open(url, name, opts \\ [])
-  def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts)
-
-  def open(%URI{} = uri, name, opts) do
+  def open(%URI{} = uri, opts) do
     pool_opts = Pleroma.Config.get([:connections_pool], [])
 
     opts =
@@ -45,30 +16,10 @@ defmodule Pleroma.Gun.Conn do
       |> Map.put_new(:retry, pool_opts[:retry] || 1)
       |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
       |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
+      |> Map.put_new(:supervise, false)
       |> maybe_add_tls_opts(uri)
 
-    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
-
-    max_connections = pool_opts[:max_connections] || 250
-
-    conn_pid =
-      if Connections.count(name) < max_connections do
-        do_open(uri, opts)
-      else
-        close_least_used_and_do_open(name, uri, opts)
-      end
-
-    if is_pid(conn_pid) do
-      conn = %Pleroma.Gun.Conn{
-        conn: conn_pid,
-        gun_state: :up,
-        conn_state: :active,
-        last_reference: :os.system_time(:second)
-      }
-
-      :ok = Gun.set_owner(conn_pid, Process.whereis(name))
-      Connections.add_conn(name, key, conn)
-    end
+    do_open(uri, opts)
   end
 
   defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
@@ -81,7 +32,7 @@ defmodule Pleroma.Gun.Conn do
       reuse_sessions: false,
       verify_fun:
         {&:ssl_verify_hostname.verify_fun/3,
-         [check_hostname: Pleroma.HTTP.Connection.format_host(host)]}
+         [check_hostname: Pleroma.HTTP.AdapterHelper.format_host(host)]}
     ]
 
     tls_opts =
@@ -105,7 +56,7 @@ defmodule Pleroma.Gun.Conn do
          {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]),
          stream <- Gun.connect(conn, connect_opts),
          {:response, :fin, 200, _} <- Gun.await(conn, stream) do
-      conn
+      {:ok, conn}
     else
       error ->
         Logger.warn(
@@ -141,7 +92,7 @@ defmodule Pleroma.Gun.Conn do
 
     with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
          {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
-      conn
+      {:ok, conn}
     else
       error ->
         Logger.warn(
@@ -155,11 +106,11 @@ defmodule Pleroma.Gun.Conn do
   end
 
   defp do_open(%URI{host: host, port: port} = uri, opts) do
-    host = Pleroma.HTTP.Connection.parse_host(host)
+    host = Pleroma.HTTP.AdapterHelper.parse_host(host)
 
     with {:ok, conn} <- Gun.open(host, port, opts),
          {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do
-      conn
+      {:ok, conn}
     else
       error ->
         Logger.warn(
@@ -171,7 +122,7 @@ defmodule Pleroma.Gun.Conn do
   end
 
   defp destination_opts(%URI{host: host, port: port}) do
-    host = Pleroma.HTTP.Connection.parse_host(host)
+    host = Pleroma.HTTP.AdapterHelper.parse_host(host)
     %{host: host, port: port}
   end
 
@@ -181,17 +132,6 @@ defmodule Pleroma.Gun.Conn do
 
   defp add_http2_opts(opts, _, _), do: opts
 
-  defp close_least_used_and_do_open(name, uri, opts) do
-    with [{key, conn} | _conns] <- Connections.get_unused_conns(name),
-         :ok <- Gun.close(conn.conn) do
-      Connections.remove_conn(name, key)
-
-      do_open(uri, opts)
-    else
-      [] -> {:error, :pool_overflowed}
-    end
-  end
-
   def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do
     "#{scheme}://#{host}#{path}"
   end
diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex
new file mode 100644
index 000000000..e6abee69c
--- /dev/null
+++ b/lib/pleroma/gun/connection_pool.ex
@@ -0,0 +1,129 @@
+defmodule Pleroma.Gun.ConnectionPool do
+  @registry __MODULE__
+
+  def get_conn(uri, opts) do
+    case enforce_pool_limits() do
+      :ok ->
+        key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
+
+        case Registry.lookup(@registry, key) do
+          # The key has already been registered, but connection is not up yet
+          [{worker_pid, {nil, _used_by, _crf, _last_reference}}] ->
+            get_gun_pid_from_worker(worker_pid)
+
+          [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
+            GenServer.cast(worker_pid, {:add_client, self(), false})
+            {:ok, gun_pid}
+
+          [] ->
+            # :gun.set_owner fails in :connected state for whatevever reason,
+            # so we open the connection in the process directly and send it's pid back
+            # We trust gun to handle timeouts by itself
+            case GenServer.start(Pleroma.Gun.ConnectionPool.Worker, [uri, key, opts, self()],
+                   timeout: :infinity
+                 ) do
+              {:ok, _worker_pid} ->
+                receive do
+                  {:conn_pid, pid} -> {:ok, pid}
+                end
+
+              {:error, {:error, {:already_registered, worker_pid}}} ->
+                get_gun_pid_from_worker(worker_pid)
+
+              err ->
+                err
+            end
+        end
+
+      :error ->
+        {:error, :pool_full}
+    end
+  end
+
+  @enforcer_key "enforcer"
+  defp enforce_pool_limits() do
+    max_connections = Pleroma.Config.get([:connections_pool, :max_connections])
+
+    if Registry.count(@registry) >= max_connections do
+      case Registry.lookup(@registry, @enforcer_key) do
+        [] ->
+          pid =
+            spawn(fn ->
+              {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil)
+
+              reclaim_max =
+                [:connections_pool, :reclaim_multiplier]
+                |> Pleroma.Config.get()
+                |> Kernel.*(max_connections)
+                |> round
+                |> max(1)
+
+              unused_conns =
+                Registry.select(
+                  @registry,
+                  [
+                    {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}],
+                     [{{:"$1", :"$3", :"$4"}}]}
+                  ]
+                )
+
+              case unused_conns do
+                [] ->
+                  exit(:pool_full)
+
+                unused_conns ->
+                  unused_conns
+                  |> Enum.sort(fn {_pid1, crf1, last_reference1},
+                                  {_pid2, crf2, last_reference2} ->
+                    crf1 <= crf2 and last_reference1 <= last_reference2
+                  end)
+                  |> Enum.take(reclaim_max)
+                  |> Enum.each(fn {pid, _, _} -> GenServer.call(pid, :idle_close) end)
+              end
+            end)
+
+          wait_for_enforcer_finish(pid)
+
+        [{pid, _}] ->
+          wait_for_enforcer_finish(pid)
+      end
+    else
+      :ok
+    end
+  end
+
+  defp wait_for_enforcer_finish(pid) do
+    ref = Process.monitor(pid)
+
+    receive do
+      {:DOWN, ^ref, :process, ^pid, :pool_full} ->
+        :error
+
+      {:DOWN, ^ref, :process, ^pid, :normal} ->
+        :ok
+    end
+  end
+
+  defp get_gun_pid_from_worker(worker_pid) do
+    # GenServer.call will block the process for timeout length if
+    # the server crashes on startup (which will happen if gun fails to connect)
+    # so instead we use cast + monitor
+
+    ref = Process.monitor(worker_pid)
+    GenServer.cast(worker_pid, {:add_client, self(), true})
+
+    receive do
+      {:conn_pid, pid} -> {:ok, pid}
+      {:DOWN, ^ref, :process, ^worker_pid, reason} -> reason
+    end
+  end
+
+  def release_conn(conn_pid) do
+    [worker_pid] =
+      Registry.select(@registry, [
+        {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
+      ])
+
+    GenServer.cast(worker_pid, {:remove_client, self()})
+  end
+end
diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
new file mode 100644
index 000000000..ebde4bbf6
--- /dev/null
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -0,0 +1,95 @@
+defmodule Pleroma.Gun.ConnectionPool.Worker do
+  alias Pleroma.Gun
+  use GenServer
+
+  @registry Pleroma.Gun.ConnectionPool
+
+  @impl true
+  def init([uri, key, opts, client_pid]) do
+    time = :os.system_time(:second)
+    # Register before opening connection to prevent race conditions
+    with {:ok, _owner} <- Registry.register(@registry, key, {nil, [client_pid], 1, time}),
+         {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
+         Process.link(conn_pid) do
+      {_, _} =
+        Registry.update_value(@registry, key, fn {_, used_by, crf, last_reference} ->
+          {conn_pid, used_by, crf, last_reference}
+        end)
+
+      send(client_pid, {:conn_pid, conn_pid})
+      {:ok, %{key: key, timer: nil}, :hibernate}
+    else
+      err -> {:stop, err}
+    end
+  end
+
+  @impl true
+  def handle_cast({:add_client, client_pid, send_pid_back}, %{key: key} = state) do
+    time = :os.system_time(:second)
+
+    {{conn_pid, _, _, _}, _} =
+      Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
+        {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
+      end)
+
+    if send_pid_back, do: send(client_pid, {:conn_pid, conn_pid})
+
+    state =
+      if state.timer != nil do
+        Process.cancel_timer(state[:timer])
+        %{state | timer: nil}
+      else
+        state
+      end
+
+    {:noreply, state, :hibernate}
+  end
+
+  @impl true
+  def handle_cast({:remove_client, client_pid}, %{key: key} = state) do
+    {{_conn_pid, used_by, _crf, _last_reference}, _} =
+      Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
+        {conn_pid, List.delete(used_by, client_pid), crf, last_reference}
+      end)
+
+    timer =
+      if used_by == [] do
+        max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
+        Process.send_after(self(), :idle_close, max_idle)
+      else
+        nil
+      end
+
+    {:noreply, %{state | timer: timer}, :hibernate}
+  end
+
+  @impl true
+  def handle_info(:idle_close, state) do
+    # Gun monitors the owner process, and will close the connection automatically
+    # when it's terminated
+    {:stop, :normal, state}
+  end
+
+  # Gracefully shutdown if the connection got closed without any streams left
+  @impl true
+  def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do
+    {:stop, :normal, state}
+  end
+
+  # Otherwise, shutdown with an error
+  @impl true
+  def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do
+    {:stop, {:error, down_message}, state}
+  end
+
+  @impl true
+  def handle_call(:idle_close, _, %{key: key} = state) do
+    Registry.unregister(@registry, key)
+    {:stop, :normal, state}
+  end
+
+  # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478
+  defp crf(time_delta, prev_crf) do
+    1 + :math.pow(0.5, time_delta / 100) * prev_crf
+  end
+end
diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index 510722ff9..0532ea31d 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -3,7 +3,21 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 
 defmodule Pleroma.HTTP.AdapterHelper do
-  alias Pleroma.HTTP.Connection
+  @moduledoc """
+  Configure Tesla.Client with default and customized adapter options.
+  """
+  @defaults [pool: :federation]
+
+  @type ip_address :: ipv4_address() | ipv6_address()
+  @type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
+  @type ipv6_address ::
+          {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
+  @type proxy_type() :: :socks4 | :socks5
+  @type host() :: charlist() | ip_address()
+
+  alias Pleroma.Config
+  alias Pleroma.HTTP.AdapterHelper
+  require Logger
 
   @type proxy ::
           {Connection.host(), pos_integer()}
@@ -11,24 +25,13 @@ defmodule Pleroma.HTTP.AdapterHelper do
 
   @callback options(keyword(), URI.t()) :: keyword()
   @callback after_request(keyword()) :: :ok
-
-  @spec options(keyword(), URI.t()) :: keyword()
-  def options(opts, _uri) do
-    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
-    maybe_add_proxy(opts, format_proxy(proxy))
-  end
-
-  @spec maybe_get_conn(URI.t(), keyword()) :: keyword()
-  def maybe_get_conn(_uri, opts), do: opts
-
-  @spec after_request(keyword()) :: :ok
-  def after_request(_opts), do: :ok
+  @callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()}
 
   @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
   def format_proxy(nil), do: nil
 
   def format_proxy(proxy_url) do
-    case Connection.parse_proxy(proxy_url) do
+    case parse_proxy(proxy_url) do
       {:ok, host, port} -> {host, port}
       {:ok, type, host, port} -> {type, host, port}
       _ -> nil
@@ -38,4 +41,106 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword()
   def maybe_add_proxy(opts, nil), do: opts
   def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy)
+
+  @doc """
+  Merge default connection & adapter options with received ones.
+  """
+
+  @spec options(URI.t(), keyword()) :: keyword()
+  def options(%URI{} = uri, opts \\ []) do
+    @defaults
+    |> pool_timeout()
+    |> Keyword.merge(opts)
+    |> adapter_helper().options(uri)
+  end
+
+  defp pool_timeout(opts) do
+    {config_key, default} =
+      if adapter() == Tesla.Adapter.Gun do
+        {:pools, Config.get([:pools, :default, :timeout])}
+      else
+        {:hackney_pools, 10_000}
+      end
+
+    timeout = Config.get([config_key, opts[:pool], :timeout], default)
+
+    Keyword.merge(opts, timeout: timeout)
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(opts), do: adapter_helper().after_request(opts)
+
+  def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
+  defp adapter, do: Application.get_env(:tesla, :adapter)
+
+  defp adapter_helper do
+    case adapter() do
+      Tesla.Adapter.Gun -> AdapterHelper.Gun
+      Tesla.Adapter.Hackney -> AdapterHelper.Hackney
+      _ -> AdapterHelper.Default
+    end
+  end
+
+  @spec parse_proxy(String.t() | tuple() | nil) ::
+          {:ok, host(), pos_integer()}
+          | {:ok, proxy_type(), host(), pos_integer()}
+          | {:error, atom()}
+          | nil
+
+  def parse_proxy(nil), do: nil
+
+  def parse_proxy(proxy) when is_binary(proxy) do
+    with [host, port] <- String.split(proxy, ":"),
+         {port, ""} <- Integer.parse(port) do
+      {:ok, parse_host(host), port}
+    else
+      {_, _} ->
+        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        {:error, :invalid_proxy_port}
+
+      :error ->
+        Logger.warn("Parsing port failed #{inspect(proxy)}")
+        {:error, :invalid_proxy_port}
+
+      _ ->
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
+        {:error, :invalid_proxy}
+    end
+  end
+
+  def parse_proxy(proxy) when is_tuple(proxy) do
+    with {type, host, port} <- proxy do
+      {:ok, type, parse_host(host), port}
+    else
+      _ ->
+        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
+        {:error, :invalid_proxy}
+    end
+  end
+
+  @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
+  def parse_host(host) when is_list(host), do: host
+  def parse_host(host) when is_atom(host), do: to_charlist(host)
+
+  def parse_host(host) when is_binary(host) do
+    host = to_charlist(host)
+
+    case :inet.parse_address(host) do
+      {:error, :einval} -> host
+      {:ok, ip} -> ip
+    end
+  end
+
+  @spec format_host(String.t()) :: charlist()
+  def format_host(host) do
+    host_charlist = to_charlist(host)
+
+    case :inet.parse_address(host_charlist) do
+      {:error, :einval} ->
+        :idna.encode(host_charlist)
+
+      {:ok, _ip} ->
+        host_charlist
+    end
+  end
 end
diff --git a/lib/pleroma/http/adapter_helper/default.ex b/lib/pleroma/http/adapter_helper/default.ex
new file mode 100644
index 000000000..218cfacc0
--- /dev/null
+++ b/lib/pleroma/http/adapter_helper/default.ex
@@ -0,0 +1,17 @@
+defmodule Pleroma.HTTP.AdapterHelper.Default do
+  alias Pleroma.HTTP.AdapterHelper
+
+  @behaviour Pleroma.HTTP.AdapterHelper
+
+  @spec options(keyword(), URI.t()) :: keyword()
+  def options(opts, _uri) do
+    proxy = Pleroma.Config.get([:http, :proxy_url], nil)
+    AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy))
+  end
+
+  @spec after_request(keyword()) :: :ok
+  def after_request(_opts), do: :ok
+
+  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()}
+  def get_conn(_uri, opts), do: {:ok, opts}
+end
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index ead7cdc6b..6f7cc9784 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -5,8 +5,8 @@
 defmodule Pleroma.HTTP.AdapterHelper.Gun do
   @behaviour Pleroma.HTTP.AdapterHelper
 
+  alias Pleroma.Gun.ConnectionPool
   alias Pleroma.HTTP.AdapterHelper
-  alias Pleroma.Pool.Connections
 
   require Logger
 
@@ -31,13 +31,13 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     |> Keyword.merge(config_opts)
     |> add_scheme_opts(uri)
     |> AdapterHelper.maybe_add_proxy(proxy)
-    |> maybe_get_conn(uri, incoming_opts)
+    |> Keyword.merge(incoming_opts)
   end
 
   @spec after_request(keyword()) :: :ok
   def after_request(opts) do
     if opts[:conn] && opts[:body_as] != :chunks do
-      Connections.checkout(opts[:conn], self(), :gun_connections)
+      ConnectionPool.release_conn(opts[:conn])
     end
 
     :ok
@@ -51,27 +51,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     |> Keyword.put(:tls_opts, log_level: :warning)
   end
 
-  defp maybe_get_conn(adapter_opts, uri, incoming_opts) do
-    {receive_conn?, opts} =
-      adapter_opts
-      |> Keyword.merge(incoming_opts)
-      |> Keyword.pop(:receive_conn, true)
-
-    if Connections.alive?(:gun_connections) and receive_conn? do
-      checkin_conn(uri, opts)
-    else
-      opts
-    end
-  end
-
-  defp checkin_conn(uri, opts) do
-    case Connections.checkin(uri, :gun_connections) do
-      nil ->
-        Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts])
-        opts
-
-      conn when is_pid(conn) ->
-        Keyword.merge(opts, conn: conn, close_conn: false)
+  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
+  def get_conn(uri, opts) do
+    case ConnectionPool.get_conn(uri, opts) do
+      {:ok, conn_pid} -> {:ok, Keyword.merge(opts, conn: conn_pid, close_conn: false)}
+      err -> err
     end
   end
 end
diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex
index 3972a03a9..42d552740 100644
--- a/lib/pleroma/http/adapter_helper/hackney.ex
+++ b/lib/pleroma/http/adapter_helper/hackney.ex
@@ -25,4 +25,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
   defp add_scheme_opts(opts, _), do: opts
 
   def after_request(_), do: :ok
+
+  @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()}
+  def get_conn(_uri, opts), do: {:ok, opts}
 end
diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex
deleted file mode 100644
index ebacf7902..000000000
--- a/lib/pleroma/http/connection.ex
+++ /dev/null
@@ -1,124 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.HTTP.Connection do
-  @moduledoc """
-  Configure Tesla.Client with default and customized adapter options.
-  """
-
-  alias Pleroma.Config
-  alias Pleroma.HTTP.AdapterHelper
-
-  require Logger
-
-  @defaults [pool: :federation]
-
-  @type ip_address :: ipv4_address() | ipv6_address()
-  @type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
-  @type ipv6_address ::
-          {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
-  @type proxy_type() :: :socks4 | :socks5
-  @type host() :: charlist() | ip_address()
-
-  @doc """
-  Merge default connection & adapter options with received ones.
-  """
-
-  @spec options(URI.t(), keyword()) :: keyword()
-  def options(%URI{} = uri, opts \\ []) do
-    @defaults
-    |> pool_timeout()
-    |> Keyword.merge(opts)
-    |> adapter_helper().options(uri)
-  end
-
-  defp pool_timeout(opts) do
-    {config_key, default} =
-      if adapter() == Tesla.Adapter.Gun do
-        {:pools, Config.get([:pools, :default, :timeout])}
-      else
-        {:hackney_pools, 10_000}
-      end
-
-    timeout = Config.get([config_key, opts[:pool], :timeout], default)
-
-    Keyword.merge(opts, timeout: timeout)
-  end
-
-  @spec after_request(keyword()) :: :ok
-  def after_request(opts), do: adapter_helper().after_request(opts)
-
-  defp adapter, do: Application.get_env(:tesla, :adapter)
-
-  defp adapter_helper do
-    case adapter() do
-      Tesla.Adapter.Gun -> AdapterHelper.Gun
-      Tesla.Adapter.Hackney -> AdapterHelper.Hackney
-      _ -> AdapterHelper
-    end
-  end
-
-  @spec parse_proxy(String.t() | tuple() | nil) ::
-          {:ok, host(), pos_integer()}
-          | {:ok, proxy_type(), host(), pos_integer()}
-          | {:error, atom()}
-          | nil
-
-  def parse_proxy(nil), do: nil
-
-  def parse_proxy(proxy) when is_binary(proxy) do
-    with [host, port] <- String.split(proxy, ":"),
-         {port, ""} <- Integer.parse(port) do
-      {:ok, parse_host(host), port}
-    else
-      {_, _} ->
-        Logger.warn("Parsing port failed #{inspect(proxy)}")
-        {:error, :invalid_proxy_port}
-
-      :error ->
-        Logger.warn("Parsing port failed #{inspect(proxy)}")
-        {:error, :invalid_proxy_port}
-
-      _ ->
-        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
-        {:error, :invalid_proxy}
-    end
-  end
-
-  def parse_proxy(proxy) when is_tuple(proxy) do
-    with {type, host, port} <- proxy do
-      {:ok, type, parse_host(host), port}
-    else
-      _ ->
-        Logger.warn("Parsing proxy failed #{inspect(proxy)}")
-        {:error, :invalid_proxy}
-    end
-  end
-
-  @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
-  def parse_host(host) when is_list(host), do: host
-  def parse_host(host) when is_atom(host), do: to_charlist(host)
-
-  def parse_host(host) when is_binary(host) do
-    host = to_charlist(host)
-
-    case :inet.parse_address(host) do
-      {:error, :einval} -> host
-      {:ok, ip} -> ip
-    end
-  end
-
-  @spec format_host(String.t()) :: charlist()
-  def format_host(host) do
-    host_charlist = to_charlist(host)
-
-    case :inet.parse_address(host_charlist) do
-      {:error, :einval} ->
-        :idna.encode(host_charlist)
-
-      {:ok, _ip} ->
-        host_charlist
-    end
-  end
-end
diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
index 66ca75367..8ded76601 100644
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@@ -7,7 +7,7 @@ defmodule Pleroma.HTTP do
     Wrapper for `Tesla.request/2`.
   """
 
-  alias Pleroma.HTTP.Connection
+  alias Pleroma.HTTP.AdapterHelper
   alias Pleroma.HTTP.Request
   alias Pleroma.HTTP.RequestBuilder, as: Builder
   alias Tesla.Client
@@ -60,49 +60,26 @@ defmodule Pleroma.HTTP do
           {:ok, Env.t()} | {:error, any()}
   def request(method, url, body, headers, options) when is_binary(url) do
     uri = URI.parse(url)
-    adapter_opts = Connection.options(uri, options[:adapter] || [])
-    options = put_in(options[:adapter], adapter_opts)
-    params = options[:params] || []
-    request = build_request(method, headers, options, url, body, params)
+    adapter_opts = AdapterHelper.options(uri, options[:adapter] || [])
 
-    adapter = Application.get_env(:tesla, :adapter)
-    client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
+    case AdapterHelper.get_conn(uri, adapter_opts) do
+      {:ok, adapter_opts} ->
+        options = put_in(options[:adapter], adapter_opts)
+        params = options[:params] || []
+        request = build_request(method, headers, options, url, body, params)
 
-    pid = Process.whereis(adapter_opts[:pool])
+        adapter = Application.get_env(:tesla, :adapter)
+        client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
 
-    pool_alive? =
-      if adapter == Tesla.Adapter.Gun && pid do
-        Process.alive?(pid)
-      else
-        false
-      end
+        response = request(client, request)
 
-    request_opts =
-      adapter_opts
-      |> Enum.into(%{})
-      |> Map.put(:env, Pleroma.Config.get([:env]))
-      |> Map.put(:pool_alive?, pool_alive?)
+        AdapterHelper.after_request(adapter_opts)
 
-    response = request(client, request, request_opts)
+        response
 
-    Connection.after_request(adapter_opts)
-
-    response
-  end
-
-  @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()}
-  def request(%Client{} = client, request, %{env: :test}), do: request(client, request)
-
-  def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request)
-
-  def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request)
-
-  def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do
-    :poolboy.transaction(
-      pool,
-      &Pleroma.Pool.Request.execute(&1, client, request, timeout),
-      timeout
-    )
+      err ->
+        err
+    end
   end
 
   @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()}
diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex
deleted file mode 100644
index acafe1bea..000000000
--- a/lib/pleroma/pool/connections.ex
+++ /dev/null
@@ -1,283 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Pool.Connections do
-  use GenServer
-
-  alias Pleroma.Config
-  alias Pleroma.Gun
-
-  require Logger
-
-  @type domain :: String.t()
-  @type conn :: Pleroma.Gun.Conn.t()
-
-  @type t :: %__MODULE__{
-          conns: %{domain() => conn()},
-          opts: keyword()
-        }
-
-  defstruct conns: %{}, opts: []
-
-  @spec start_link({atom(), keyword()}) :: {:ok, pid()}
-  def start_link({name, opts}) do
-    GenServer.start_link(__MODULE__, opts, name: name)
-  end
-
-  @impl true
-  def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}}
-
-  @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil
-  def checkin(url, name)
-  def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name)
-
-  def checkin(%URI{} = uri, name) do
-    timeout = Config.get([:connections_pool, :checkin_timeout], 250)
-
-    GenServer.call(name, {:checkin, uri}, timeout)
-  end
-
-  @spec alive?(atom()) :: boolean()
-  def alive?(name) do
-    if pid = Process.whereis(name) do
-      Process.alive?(pid)
-    else
-      false
-    end
-  end
-
-  @spec get_state(atom()) :: t()
-  def get_state(name) do
-    GenServer.call(name, :state)
-  end
-
-  @spec count(atom()) :: pos_integer()
-  def count(name) do
-    GenServer.call(name, :count)
-  end
-
-  @spec get_unused_conns(atom()) :: [{domain(), conn()}]
-  def get_unused_conns(name) do
-    GenServer.call(name, :unused_conns)
-  end
-
-  @spec checkout(pid(), pid(), atom()) :: :ok
-  def checkout(conn, pid, name) do
-    GenServer.cast(name, {:checkout, conn, pid})
-  end
-
-  @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok
-  def add_conn(name, key, conn) do
-    GenServer.cast(name, {:add_conn, key, conn})
-  end
-
-  @spec remove_conn(atom(), String.t()) :: :ok
-  def remove_conn(name, key) do
-    GenServer.cast(name, {:remove_conn, key})
-  end
-
-  @impl true
-  def handle_cast({:add_conn, key, conn}, state) do
-    state = put_in(state.conns[key], conn)
-
-    Process.monitor(conn.conn)
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_cast({:checkout, conn_pid, pid}, state) do
-    state =
-      with true <- Process.alive?(conn_pid),
-           {key, conn} <- find_conn(state.conns, conn_pid),
-           used_by <- List.keydelete(conn.used_by, pid, 0) do
-        conn_state = if used_by == [], do: :idle, else: conn.conn_state
-
-        put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by})
-      else
-        false ->
-          Logger.debug("checkout for closed conn #{inspect(conn_pid)}")
-          state
-
-        nil ->
-          Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state")
-          state
-      end
-
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_cast({:remove_conn, key}, state) do
-    state = put_in(state.conns, Map.delete(state.conns, key))
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_call({:checkin, uri}, from, state) do
-    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
-
-    case state.conns[key] do
-      %{conn: pid, gun_state: :up} = conn ->
-        time = :os.system_time(:second)
-        last_reference = time - conn.last_reference
-        crf = crf(last_reference, 100, conn.crf)
-
-        state =
-          put_in(state.conns[key], %{
-            conn
-            | last_reference: time,
-              crf: crf,
-              conn_state: :active,
-              used_by: [from | conn.used_by]
-          })
-
-        {:reply, pid, state}
-
-      %{gun_state: :down} ->
-        {:reply, nil, state}
-
-      nil ->
-        {:reply, nil, state}
-    end
-  end
-
-  @impl true
-  def handle_call(:state, _from, state), do: {:reply, state, state}
-
-  @impl true
-  def handle_call(:count, _from, state) do
-    {:reply, Enum.count(state.conns), state}
-  end
-
-  @impl true
-  def handle_call(:unused_conns, _from, state) do
-    unused_conns =
-      state.conns
-      |> Enum.filter(&filter_conns/1)
-      |> Enum.sort(&sort_conns/2)
-
-    {:reply, unused_conns, state}
-  end
-
-  defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true
-  defp filter_conns(_), do: false
-
-  defp sort_conns({_, c1}, {_, c2}) do
-    c1.crf <= c2.crf and c1.last_reference <= c2.last_reference
-  end
-
-  @impl true
-  def handle_info({:gun_up, conn_pid, _protocol}, state) do
-    %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid)
-
-    host =
-      case :inet.ntoa(host) do
-        {:error, :einval} -> host
-        ip -> ip
-      end
-
-    key = "#{scheme}:#{host}:#{port}"
-
-    state =
-      with {key, conn} <- find_conn(state.conns, conn_pid, key),
-           {true, key} <- {Process.alive?(conn_pid), key} do
-        put_in(state.conns[key], %{
-          conn
-          | gun_state: :up,
-            conn_state: :active,
-            retries: 0
-        })
-      else
-        {false, key} ->
-          put_in(
-            state.conns,
-            Map.delete(state.conns, key)
-          )
-
-        nil ->
-          :ok = Gun.close(conn_pid)
-
-          state
-      end
-
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do
-    retries = Config.get([:connections_pool, :retry], 1)
-    # we can't get info on this pid, because pid is dead
-    state =
-      with {key, conn} <- find_conn(state.conns, conn_pid),
-           {true, key} <- {Process.alive?(conn_pid), key} do
-        if conn.retries == retries do
-          :ok = Gun.close(conn.conn)
-
-          put_in(
-            state.conns,
-            Map.delete(state.conns, key)
-          )
-        else
-          put_in(state.conns[key], %{
-            conn
-            | gun_state: :down,
-              retries: conn.retries + 1
-          })
-        end
-      else
-        {false, key} ->
-          put_in(
-            state.conns,
-            Map.delete(state.conns, key)
-          )
-
-        nil ->
-          Logger.debug(":gun_down for conn which isn't found in state")
-
-          state
-      end
-
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do
-    Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}")
-
-    state =
-      with {key, conn} <- find_conn(state.conns, conn_pid) do
-        Enum.each(conn.used_by, fn {pid, _ref} ->
-          Process.exit(pid, reason)
-        end)
-
-        put_in(
-          state.conns,
-          Map.delete(state.conns, key)
-        )
-      else
-        nil ->
-          Logger.debug(":DOWN for conn which isn't found in state")
-
-          state
-      end
-
-    {:noreply, state}
-  end
-
-  defp find_conn(conns, conn_pid) do
-    Enum.find(conns, fn {_key, conn} ->
-      conn.conn == conn_pid
-    end)
-  end
-
-  defp find_conn(conns, conn_pid, conn_key) do
-    Enum.find(conns, fn {key, conn} ->
-      key == conn_key and conn.conn == conn_pid
-    end)
-  end
-
-  def crf(current, steps, crf) do
-    1 + :math.pow(0.5, current / steps) * crf
-  end
-end
diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex
deleted file mode 100644
index 21a6fbbc5..000000000
--- a/lib/pleroma/pool/pool.ex
+++ /dev/null
@@ -1,22 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Pool do
-  def child_spec(opts) do
-    poolboy_opts =
-      opts
-      |> Keyword.put(:worker_module, Pleroma.Pool.Request)
-      |> Keyword.put(:name, {:local, opts[:name]})
-      |> Keyword.put(:size, opts[:size])
-      |> Keyword.put(:max_overflow, opts[:max_overflow])
-
-    %{
-      id: opts[:id] || {__MODULE__, make_ref()},
-      start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]},
-      restart: :permanent,
-      shutdown: 5000,
-      type: :worker
-    }
-  end
-end
diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex
deleted file mode 100644
index 3fb930db7..000000000
--- a/lib/pleroma/pool/request.ex
+++ /dev/null
@@ -1,65 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Pool.Request do
-  use GenServer
-
-  require Logger
-
-  def start_link(args) do
-    GenServer.start_link(__MODULE__, args)
-  end
-
-  @impl true
-  def init(_), do: {:ok, []}
-
-  @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) ::
-          {:ok, Tesla.Env.t()} | {:error, any()}
-  def execute(pid, client, request, timeout) do
-    GenServer.call(pid, {:execute, client, request}, timeout)
-  end
-
-  @impl true
-  def handle_call({:execute, client, request}, _from, state) do
-    response = Pleroma.HTTP.request(client, request)
-
-    {:reply, response, state}
-  end
-
-  @impl true
-  def handle_info({:gun_data, _conn, _stream, _, _}, state) do
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_info({:gun_up, _conn, _protocol}, state) do
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_info({:gun_error, _conn, _stream, _error}, state) do
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do
-    {:noreply, state}
-  end
-
-  @impl true
-  def handle_info(msg, state) do
-    Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}")
-    {:noreply, state}
-  end
-end
diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex
deleted file mode 100644
index faf646cb2..000000000
--- a/lib/pleroma/pool/supervisor.ex
+++ /dev/null
@@ -1,42 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Pool.Supervisor do
-  use Supervisor
-
-  alias Pleroma.Config
-  alias Pleroma.Pool
-
-  def start_link(args) do
-    Supervisor.start_link(__MODULE__, args, name: __MODULE__)
-  end
-
-  def init(_) do
-    conns_child = %{
-      id: Pool.Connections,
-      start:
-        {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]}
-    }
-
-    Supervisor.init([conns_child | pools()], strategy: :one_for_one)
-  end
-
-  defp pools do
-    pools = Config.get(:pools)
-
-    pools =
-      if Config.get([Pleroma.Upload, :proxy_remote]) == false do
-        Keyword.delete(pools, :upload)
-      else
-        pools
-      end
-
-    for {pool_name, pool_opts} <- pools do
-      pool_opts
-      |> Keyword.put(:id, {Pool, pool_name})
-      |> Keyword.put(:name, pool_name)
-      |> Pool.child_spec()
-    end
-  end
-end
diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex
index e81ea8bde..65785445d 100644
--- a/lib/pleroma/reverse_proxy/client/tesla.ex
+++ b/lib/pleroma/reverse_proxy/client/tesla.ex
@@ -48,7 +48,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do
     # if there were redirects we need to checkout old conn
     conn = opts[:old_conn] || opts[:conn]
 
-    if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections)
+    if conn, do: :ok = Pleroma.Gun.ConnectionPool.release_conn(conn)
 
     :done
   end
-- 
cgit v1.2.3


From fffbcffb8c9ce1e96de5d1a5e15005e271deacd4 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 6 May 2020 21:41:34 +0300
Subject: Connection Pool: don't enforce pool limits if no new connection needs
 to be opened

---
 lib/pleroma/gun/connection_pool.ex | 28 ++++++++++++++--------------
 1 file changed, 14 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex
index e6abee69c..ed7ddff81 100644
--- a/lib/pleroma/gun/connection_pool.ex
+++ b/lib/pleroma/gun/connection_pool.ex
@@ -2,20 +2,20 @@ defmodule Pleroma.Gun.ConnectionPool do
   @registry __MODULE__
 
   def get_conn(uri, opts) do
-    case enforce_pool_limits() do
-      :ok ->
-        key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
+    key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
 
-        case Registry.lookup(@registry, key) do
-          # The key has already been registered, but connection is not up yet
-          [{worker_pid, {nil, _used_by, _crf, _last_reference}}] ->
-            get_gun_pid_from_worker(worker_pid)
+    case Registry.lookup(@registry, key) do
+      # The key has already been registered, but connection is not up yet
+      [{worker_pid, {nil, _used_by, _crf, _last_reference}}] ->
+        get_gun_pid_from_worker(worker_pid)
 
-          [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
-            GenServer.cast(worker_pid, {:add_client, self(), false})
-            {:ok, gun_pid}
+      [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
+        GenServer.cast(worker_pid, {:add_client, self(), false})
+        {:ok, gun_pid}
 
-          [] ->
+      [] ->
+        case enforce_pool_limits() do
+          :ok ->
             # :gun.set_owner fails in :connected state for whatevever reason,
             # so we open the connection in the process directly and send it's pid back
             # We trust gun to handle timeouts by itself
@@ -33,10 +33,10 @@ defmodule Pleroma.Gun.ConnectionPool do
               err ->
                 err
             end
-        end
 
-      :error ->
-        {:error, :pool_full}
+          :error ->
+            {:error, :pool_full}
+        end
     end
   end
 
-- 
cgit v1.2.3


From d08b1576990ca33ac4178fb757ec03a777c55b5b Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 6 May 2020 21:51:10 +0300
Subject: Connection pool: check that there actually is a result

Sometimes connections died before being released to the pool, resulting
in MatchErrors
---
 lib/pleroma/gun/connection_pool.ex | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex
index ed7ddff81..0daf1da44 100644
--- a/lib/pleroma/gun/connection_pool.ex
+++ b/lib/pleroma/gun/connection_pool.ex
@@ -119,11 +119,17 @@ defmodule Pleroma.Gun.ConnectionPool do
   end
 
   def release_conn(conn_pid) do
-    [worker_pid] =
+    query_result =
       Registry.select(@registry, [
         {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
       ])
 
-    GenServer.cast(worker_pid, {:remove_client, self()})
+    case query_result do
+      [worker_pid] ->
+        GenServer.cast(worker_pid, {:remove_client, self()})
+
+      [] ->
+        :ok
+    end
   end
 end
-- 
cgit v1.2.3


From ec9d0d146b4ec6752f8f2896ace9bb5585469773 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 6 May 2020 23:14:24 +0300
Subject: Connection pool: Fix race conditions in limit enforcement

Fixes race conditions in limit enforcement by putting worker processes
in a DynamicSupervisor
---
 lib/pleroma/application.ex                         |   2 +-
 lib/pleroma/gun/connection_pool.ex                 | 105 +++++----------------
 lib/pleroma/gun/connection_pool/worker.ex          |  12 +--
 .../gun/connection_pool/worker_supervisor.ex       |  91 ++++++++++++++++++
 4 files changed, 118 insertions(+), 92 deletions(-)
 create mode 100644 lib/pleroma/gun/connection_pool/worker_supervisor.ex

(limited to 'lib')

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index be14c1f9f..cfdaf1770 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -243,7 +243,7 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(Tesla.Adapter.Gun, _) do
-    [{Registry, keys: :unique, name: Pleroma.Gun.ConnectionPool}]
+    Pleroma.Gun.ConnectionPool.children()
   end
 
   defp http_children(_, _), do: []
diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex
index 0daf1da44..545bfaf7f 100644
--- a/lib/pleroma/gun/connection_pool.ex
+++ b/lib/pleroma/gun/connection_pool.ex
@@ -1,6 +1,15 @@
 defmodule Pleroma.Gun.ConnectionPool do
   @registry __MODULE__
 
+  alias Pleroma.Gun.ConnectionPool.WorkerSupervisor
+
+  def children do
+    [
+      {Registry, keys: :unique, name: @registry},
+      Pleroma.Gun.ConnectionPool.WorkerSupervisor
+    ]
+  end
+
   def get_conn(uri, opts) do
     key = "#{uri.scheme}:#{uri.host}:#{uri.port}"
 
@@ -14,93 +23,21 @@ defmodule Pleroma.Gun.ConnectionPool do
         {:ok, gun_pid}
 
       [] ->
-        case enforce_pool_limits() do
-          :ok ->
-            # :gun.set_owner fails in :connected state for whatevever reason,
-            # so we open the connection in the process directly and send it's pid back
-            # We trust gun to handle timeouts by itself
-            case GenServer.start(Pleroma.Gun.ConnectionPool.Worker, [uri, key, opts, self()],
-                   timeout: :infinity
-                 ) do
-              {:ok, _worker_pid} ->
-                receive do
-                  {:conn_pid, pid} -> {:ok, pid}
-                end
-
-              {:error, {:error, {:already_registered, worker_pid}}} ->
-                get_gun_pid_from_worker(worker_pid)
-
-              err ->
-                err
+        # :gun.set_owner fails in :connected state for whatevever reason,
+        # so we open the connection in the process directly and send it's pid back
+        # We trust gun to handle timeouts by itself
+        case WorkerSupervisor.start_worker([uri, key, opts, self()]) do
+          {:ok, _worker_pid} ->
+            receive do
+              {:conn_pid, pid} -> {:ok, pid}
             end
 
-          :error ->
-            {:error, :pool_full}
-        end
-    end
-  end
-
-  @enforcer_key "enforcer"
-  defp enforce_pool_limits() do
-    max_connections = Pleroma.Config.get([:connections_pool, :max_connections])
-
-    if Registry.count(@registry) >= max_connections do
-      case Registry.lookup(@registry, @enforcer_key) do
-        [] ->
-          pid =
-            spawn(fn ->
-              {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil)
-
-              reclaim_max =
-                [:connections_pool, :reclaim_multiplier]
-                |> Pleroma.Config.get()
-                |> Kernel.*(max_connections)
-                |> round
-                |> max(1)
-
-              unused_conns =
-                Registry.select(
-                  @registry,
-                  [
-                    {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}],
-                     [{{:"$1", :"$3", :"$4"}}]}
-                  ]
-                )
+          {:error, {:error, {:already_registered, worker_pid}}} ->
+            get_gun_pid_from_worker(worker_pid)
 
-              case unused_conns do
-                [] ->
-                  exit(:pool_full)
-
-                unused_conns ->
-                  unused_conns
-                  |> Enum.sort(fn {_pid1, crf1, last_reference1},
-                                  {_pid2, crf2, last_reference2} ->
-                    crf1 <= crf2 and last_reference1 <= last_reference2
-                  end)
-                  |> Enum.take(reclaim_max)
-                  |> Enum.each(fn {pid, _, _} -> GenServer.call(pid, :idle_close) end)
-              end
-            end)
-
-          wait_for_enforcer_finish(pid)
-
-        [{pid, _}] ->
-          wait_for_enforcer_finish(pid)
-      end
-    else
-      :ok
-    end
-  end
-
-  defp wait_for_enforcer_finish(pid) do
-    ref = Process.monitor(pid)
-
-    receive do
-      {:DOWN, ^ref, :process, ^pid, :pool_full} ->
-        :error
-
-      {:DOWN, ^ref, :process, ^pid, :normal} ->
-        :ok
+          err ->
+            err
+        end
     end
   end
 
diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index ebde4bbf6..25fafc64c 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -1,9 +1,13 @@
 defmodule Pleroma.Gun.ConnectionPool.Worker do
   alias Pleroma.Gun
-  use GenServer
+  use GenServer, restart: :temporary
 
   @registry Pleroma.Gun.ConnectionPool
 
+  def start_link(opts) do
+    GenServer.start_link(__MODULE__, opts)
+  end
+
   @impl true
   def init([uri, key, opts, client_pid]) do
     time = :os.system_time(:second)
@@ -82,12 +86,6 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
     {:stop, {:error, down_message}, state}
   end
 
-  @impl true
-  def handle_call(:idle_close, _, %{key: key} = state) do
-    Registry.unregister(@registry, key)
-    {:stop, :normal, state}
-  end
-
   # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478
   defp crf(time_delta, prev_crf) do
     1 + :math.pow(0.5, time_delta / 100) * prev_crf
diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
new file mode 100644
index 000000000..5b546bd87
--- /dev/null
+++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
@@ -0,0 +1,91 @@
+defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
+  @doc "Supervisor for pool workers. Does not do anything except enforce max connection limit"
+
+  use DynamicSupervisor
+
+  def start_link(opts) do
+    DynamicSupervisor.start_link(__MODULE__, opts, name: __MODULE__)
+  end
+
+  def init(_opts) do
+    DynamicSupervisor.init(
+      strategy: :one_for_one,
+      max_children: Pleroma.Config.get([:connections_pool, :max_connections])
+    )
+  end
+
+  def start_worker(opts) do
+    case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do
+      {:error, :max_children} ->
+        case free_pool() do
+          :ok -> start_worker(opts)
+          :error -> {:error, :pool_full}
+        end
+
+      res ->
+        res
+    end
+  end
+
+  @registry Pleroma.Gun.ConnectionPool
+  @enforcer_key "enforcer"
+  defp free_pool do
+    case Registry.lookup(@registry, @enforcer_key) do
+      [] ->
+        pid =
+          spawn(fn ->
+            {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil)
+
+            max_connections = Pleroma.Config.get([:connections_pool, :max_connections])
+
+            reclaim_max =
+              [:connections_pool, :reclaim_multiplier]
+              |> Pleroma.Config.get()
+              |> Kernel.*(max_connections)
+              |> round
+              |> max(1)
+
+            unused_conns =
+              Registry.select(
+                @registry,
+                [
+                  {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}],
+                   [{{:"$1", :"$3", :"$4"}}]}
+                ]
+              )
+
+            case unused_conns do
+              [] ->
+                exit(:no_unused_conns)
+
+              unused_conns ->
+                unused_conns
+                |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} ->
+                  crf1 <= crf2 and last_reference1 <= last_reference2
+                end)
+                |> Enum.take(reclaim_max)
+                |> Enum.each(fn {pid, _, _} ->
+                  DynamicSupervisor.terminate_child(__MODULE__, pid)
+                end)
+            end
+          end)
+
+        wait_for_enforcer_finish(pid)
+
+      [{pid, _}] ->
+        wait_for_enforcer_finish(pid)
+    end
+  end
+
+  defp wait_for_enforcer_finish(pid) do
+    ref = Process.monitor(pid)
+
+    receive do
+      {:DOWN, ^ref, :process, ^pid, :no_unused_conns} ->
+        :error
+
+      {:DOWN, ^ref, :process, ^pid, :normal} ->
+        :ok
+    end
+  end
+end
-- 
cgit v1.2.3


From 0ffde499b8a8f31c82183253bdd692c75733ca2f Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 15 Jul 2020 15:24:47 +0300
Subject: Connection Pool: register workers using :via

---
 lib/pleroma/gun/connection_pool.ex                   |  8 +++++---
 lib/pleroma/gun/connection_pool/worker.ex            | 17 ++++++++---------
 lib/pleroma/gun/connection_pool/worker_supervisor.ex |  3 +--
 3 files changed, 14 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex
index 545bfaf7f..e951872fe 100644
--- a/lib/pleroma/gun/connection_pool.ex
+++ b/lib/pleroma/gun/connection_pool.ex
@@ -15,7 +15,7 @@ defmodule Pleroma.Gun.ConnectionPool do
 
     case Registry.lookup(@registry, key) do
       # The key has already been registered, but connection is not up yet
-      [{worker_pid, {nil, _used_by, _crf, _last_reference}}] ->
+      [{worker_pid, nil}] ->
         get_gun_pid_from_worker(worker_pid)
 
       [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
@@ -26,13 +26,13 @@ defmodule Pleroma.Gun.ConnectionPool do
         # :gun.set_owner fails in :connected state for whatevever reason,
         # so we open the connection in the process directly and send it's pid back
         # We trust gun to handle timeouts by itself
-        case WorkerSupervisor.start_worker([uri, key, opts, self()]) do
+        case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
           {:ok, _worker_pid} ->
             receive do
               {:conn_pid, pid} -> {:ok, pid}
             end
 
-          {:error, {:error, {:already_registered, worker_pid}}} ->
+          {:error, {:already_started, worker_pid}} ->
             get_gun_pid_from_worker(worker_pid)
 
           err ->
@@ -56,6 +56,8 @@ defmodule Pleroma.Gun.ConnectionPool do
   end
 
   def release_conn(conn_pid) do
+    # :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid ->
+    #    worker_pid end)
     query_result =
       Registry.select(@registry, [
         {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]}
diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index 25fafc64c..0a94f16a2 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -4,20 +4,19 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
 
   @registry Pleroma.Gun.ConnectionPool
 
-  def start_link(opts) do
-    GenServer.start_link(__MODULE__, opts)
+  def start_link([key | _] = opts) do
+    GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {@registry, key}})
   end
 
   @impl true
-  def init([uri, key, opts, client_pid]) do
-    time = :os.system_time(:second)
-    # Register before opening connection to prevent race conditions
-    with {:ok, _owner} <- Registry.register(@registry, key, {nil, [client_pid], 1, time}),
-         {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
+  def init([key, uri, opts, client_pid]) do
+    with {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
          Process.link(conn_pid) do
+      time = :os.system_time(:second)
+
       {_, _} =
-        Registry.update_value(@registry, key, fn {_, used_by, crf, last_reference} ->
-          {conn_pid, used_by, crf, last_reference}
+        Registry.update_value(@registry, key, fn _ ->
+          {conn_pid, [client_pid], 1, time}
         end)
 
       send(client_pid, {:conn_pid, conn_pid})
diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
index 5b546bd87..d090c034e 100644
--- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex
+++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
@@ -1,5 +1,5 @@
 defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
-  @doc "Supervisor for pool workers. Does not do anything except enforce max connection limit"
+  @moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit"
 
   use DynamicSupervisor
 
@@ -35,7 +35,6 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
         pid =
           spawn(fn ->
             {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil)
-
             max_connections = Pleroma.Config.get([:connections_pool, :max_connections])
 
             reclaim_max =
-- 
cgit v1.2.3


From 7738fbbaf5a6fcd6a10b4ef0a2dcea731a3d4192 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 15 Jul 2020 15:26:25 +0300
Subject: Connection pool: implement logging and telemetry events

---
 lib/pleroma/application.ex                         |  1 +
 .../gun/connection_pool/worker_supervisor.ex       | 44 ++++++++++++---
 lib/pleroma/telemetry/logger.ex                    | 62 ++++++++++++++++++++++
 3 files changed, 100 insertions(+), 7 deletions(-)
 create mode 100644 lib/pleroma/telemetry/logger.ex

(limited to 'lib')

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index cfdaf1770..37fcdf293 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -39,6 +39,7 @@ defmodule Pleroma.Application do
     # every time the application is restarted, so we disable module
     # conflicts at runtime
     Code.compiler_options(ignore_module_conflict: true)
+    Pleroma.Telemetry.Logger.attach()
     Config.Holder.save_default()
     Pleroma.HTML.compile_scrubbers()
     Config.DeprecationWarnings.warn()
diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
index d090c034e..4b5d10d2a 100644
--- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex
+++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
@@ -18,8 +18,12 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
     case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do
       {:error, :max_children} ->
         case free_pool() do
-          :ok -> start_worker(opts)
-          :error -> {:error, :pool_full}
+          :ok ->
+            start_worker(opts)
+
+          :error ->
+            :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
+            {:error, :pool_full}
         end
 
       res ->
@@ -44,6 +48,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
               |> round
               |> max(1)
 
+            :telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{
+              max_connections: max_connections,
+              reclaim_max: reclaim_max
+            })
+
+            # :ets.fun2ms(
+            # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
+            #   {worker_pid, crf, last_reference} end)
             unused_conns =
               Registry.select(
                 @registry,
@@ -55,17 +67,35 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
 
             case unused_conns do
               [] ->
+                :telemetry.execute(
+                  [:pleroma, :connection_pool, :reclaim, :stop],
+                  %{reclaimed_count: 0},
+                  %{
+                    max_connections: max_connections
+                  }
+                )
+
                 exit(:no_unused_conns)
 
               unused_conns ->
-                unused_conns
-                |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} ->
-                  crf1 <= crf2 and last_reference1 <= last_reference2
-                end)
-                |> Enum.take(reclaim_max)
+                reclaimed =
+                  unused_conns
+                  |> Enum.sort(fn {_pid1, crf1, last_reference1},
+                                  {_pid2, crf2, last_reference2} ->
+                    crf1 <= crf2 and last_reference1 <= last_reference2
+                  end)
+                  |> Enum.take(reclaim_max)
+
+                reclaimed
                 |> Enum.each(fn {pid, _, _} ->
                   DynamicSupervisor.terminate_child(__MODULE__, pid)
                 end)
+
+                :telemetry.execute(
+                  [:pleroma, :connection_pool, :reclaim, :stop],
+                  %{reclaimed_count: Enum.count(reclaimed)},
+                  %{max_connections: max_connections}
+                )
             end
           end)
 
diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex
new file mode 100644
index 000000000..d76dd37b5
--- /dev/null
+++ b/lib/pleroma/telemetry/logger.ex
@@ -0,0 +1,62 @@
+defmodule Pleroma.Telemetry.Logger do
+  @moduledoc "Transforms Pleroma telemetry events to logs"
+
+  require Logger
+
+  @events [
+    [:pleroma, :connection_pool, :reclaim, :start],
+    [:pleroma, :connection_pool, :reclaim, :stop],
+    [:pleroma, :connection_pool, :provision_failure]
+  ]
+  def attach do
+    :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, [])
+  end
+
+  # Passing anonymous functions instead of strings to logger is intentional,
+  # that way strings won't be concatenated if the message is going to be thrown
+  # out anyway due to higher log level configured
+
+  def handle_event(
+        [:pleroma, :connection_pool, :reclaim, :start],
+        _,
+        %{max_connections: max_connections, reclaim_max: reclaim_max},
+        _
+      ) do
+    Logger.debug(fn ->
+      "Connection pool is exhausted (reached #{max_connections} connections). Starting idle connection cleanup to reclaim as much as #{
+        reclaim_max
+      } connections"
+    end)
+  end
+
+  def handle_event(
+        [:pleroma, :connection_pool, :reclaim, :stop],
+        %{reclaimed_count: 0},
+        _,
+        _
+      ) do
+    Logger.error(fn ->
+      "Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts"
+    end)
+  end
+
+  def handle_event(
+        [:pleroma, :connection_pool, :reclaim, :stop],
+        %{reclaimed_count: reclaimed_count},
+        _,
+        _
+      ) do
+    Logger.debug(fn -> "Connection pool cleaned up #{reclaimed_count} idle connections" end)
+  end
+
+  def handle_event(
+        [:pleroma, :connection_pool, :provision_failure],
+        %{opts: [key | _]},
+        _,
+        _
+      ) do
+    Logger.error(fn ->
+      "Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion"
+    end)
+  end
+end
-- 
cgit v1.2.3


From e94ba05e523d735cd7a357a3aa30e433f60ef9a3 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 7 May 2020 16:11:48 +0300
Subject: Connection pool: Fix a possible infinite recursion if the pool is
 exhausted

---
 lib/pleroma/gun/connection_pool/worker_supervisor.ex | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
index 4b5d10d2a..5cb8d488a 100644
--- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex
+++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
@@ -14,16 +14,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
     )
   end
 
-  def start_worker(opts) do
+  def start_worker(opts, retry \\ false) do
     case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do
       {:error, :max_children} ->
-        case free_pool() do
-          :ok ->
-            start_worker(opts)
-
-          :error ->
-            :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
-            {:error, :pool_full}
+        if retry or free_pool() == :error do
+          :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts})
+          {:error, :pool_full}
+        else
+          start_worker(opts, true)
         end
 
       res ->
-- 
cgit v1.2.3


From 1b15cb066c612c72d106e7e7026819ea14e0ceab Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 8 May 2020 18:18:59 +0300
Subject: Connection pool: Add client death tracking

While running this in production I noticed a number of ghost
processes with all their clients dead before they released the connection,
so let's track them to log it and remove them from clients
---
 lib/pleroma/gun/connection_pool/worker.ex | 31 ++++++++++++++++++++++++++++++-
 lib/pleroma/telemetry/logger.ex           | 16 +++++++++++++++-
 2 files changed, 45 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index 0a94f16a2..8467325f3 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -20,7 +20,10 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
         end)
 
       send(client_pid, {:conn_pid, conn_pid})
-      {:ok, %{key: key, timer: nil}, :hibernate}
+
+      {:ok,
+       %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}},
+       :hibernate}
     else
       err -> {:stop, err}
     end
@@ -45,6 +48,9 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
         state
       end
 
+    ref = Process.monitor(client_pid)
+
+    state = put_in(state.client_monitors[client_pid], ref)
     {:noreply, state, :hibernate}
   end
 
@@ -55,6 +61,9 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
         {conn_pid, List.delete(used_by, client_pid), crf, last_reference}
       end)
 
+    {ref, state} = pop_in(state.client_monitors[client_pid])
+    Process.demonitor(ref)
+
     timer =
       if used_by == [] do
         max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000)
@@ -85,6 +94,26 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
     {:stop, {:error, down_message}, state}
   end
 
+  @impl true
+  def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
+    # Sometimes the client is dead before we demonitor it in :remove_client, so the message
+    # arrives anyway
+
+    case state.client_monitors[pid] do
+      nil ->
+        {:noreply, state, :hibernate}
+
+      _ref ->
+        :telemetry.execute(
+          [:pleroma, :connection_pool, :client_death],
+          %{client_pid: pid, reason: reason},
+          %{key: state.key}
+        )
+
+        handle_cast({:remove_client, pid}, state)
+    end
+  end
+
   # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478
   defp crf(time_delta, prev_crf) do
     1 + :math.pow(0.5, time_delta / 100) * prev_crf
diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex
index d76dd37b5..4cacae02f 100644
--- a/lib/pleroma/telemetry/logger.ex
+++ b/lib/pleroma/telemetry/logger.ex
@@ -6,7 +6,8 @@ defmodule Pleroma.Telemetry.Logger do
   @events [
     [:pleroma, :connection_pool, :reclaim, :start],
     [:pleroma, :connection_pool, :reclaim, :stop],
-    [:pleroma, :connection_pool, :provision_failure]
+    [:pleroma, :connection_pool, :provision_failure],
+    [:pleroma, :connection_pool, :client_death]
   ]
   def attach do
     :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, [])
@@ -59,4 +60,17 @@ defmodule Pleroma.Telemetry.Logger do
       "Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion"
     end)
   end
+
+  def handle_event(
+        [:pleroma, :connection_pool, :client_death],
+        %{client_pid: client_pid, reason: reason},
+        %{key: key},
+        _
+      ) do
+    Logger.warn(fn ->
+      "Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{
+        inspect(reason)
+      }"
+    end)
+  end
 end
-- 
cgit v1.2.3


From 281ddd5e371c5698489774e703106bd7c3ccb56b Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 8 May 2020 19:57:11 +0300
Subject: Connection pool: fix connections being supervised by gun_sup

---
 lib/pleroma/gun/api.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex
index f51cd7db8..09be74392 100644
--- a/lib/pleroma/gun/api.ex
+++ b/lib/pleroma/gun/api.ex
@@ -19,7 +19,8 @@ defmodule Pleroma.Gun.API do
     :tls_opts,
     :tcp_opts,
     :socks_opts,
-    :ws_opts
+    :ws_opts,
+    :supervise
   ]
 
   @impl Gun
-- 
cgit v1.2.3


From 94c8f3cfafb92c6d092549b24bb69f3870e1c0d8 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sat, 16 May 2020 11:49:19 +0300
Subject: Use a custom pool-aware FollowRedirects middleware

---
 lib/pleroma/http/adapter_helper.ex               |   4 -
 lib/pleroma/http/adapter_helper/default.ex       |   3 -
 lib/pleroma/http/adapter_helper/gun.ex           |   9 --
 lib/pleroma/http/adapter_helper/hackney.ex       |   2 -
 lib/pleroma/http/http.ex                         |   9 +-
 lib/pleroma/tesla/middleware/follow_redirects.ex | 106 +++++++++++++++++++++++
 6 files changed, 109 insertions(+), 24 deletions(-)
 create mode 100644 lib/pleroma/tesla/middleware/follow_redirects.ex

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index 0532ea31d..bcb9b2b1e 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -24,7 +24,6 @@ defmodule Pleroma.HTTP.AdapterHelper do
           | {Connection.proxy_type(), Connection.host(), pos_integer()}
 
   @callback options(keyword(), URI.t()) :: keyword()
-  @callback after_request(keyword()) :: :ok
   @callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()}
 
   @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil
@@ -67,9 +66,6 @@ defmodule Pleroma.HTTP.AdapterHelper do
     Keyword.merge(opts, timeout: timeout)
   end
 
-  @spec after_request(keyword()) :: :ok
-  def after_request(opts), do: adapter_helper().after_request(opts)
-
   def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts)
   defp adapter, do: Application.get_env(:tesla, :adapter)
 
diff --git a/lib/pleroma/http/adapter_helper/default.ex b/lib/pleroma/http/adapter_helper/default.ex
index 218cfacc0..e13441316 100644
--- a/lib/pleroma/http/adapter_helper/default.ex
+++ b/lib/pleroma/http/adapter_helper/default.ex
@@ -9,9 +9,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Default do
     AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy))
   end
 
-  @spec after_request(keyword()) :: :ok
-  def after_request(_opts), do: :ok
-
   @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()}
   def get_conn(_uri, opts), do: {:ok, opts}
 end
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index 6f7cc9784..5b4629978 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -34,15 +34,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     |> Keyword.merge(incoming_opts)
   end
 
-  @spec after_request(keyword()) :: :ok
-  def after_request(opts) do
-    if opts[:conn] && opts[:body_as] != :chunks do
-      ConnectionPool.release_conn(opts[:conn])
-    end
-
-    :ok
-  end
-
   defp add_scheme_opts(opts, %{scheme: "http"}), do: opts
 
   defp add_scheme_opts(opts, %{scheme: "https"}) do
diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex
index 42d552740..cd569422b 100644
--- a/lib/pleroma/http/adapter_helper/hackney.ex
+++ b/lib/pleroma/http/adapter_helper/hackney.ex
@@ -24,8 +24,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do
 
   defp add_scheme_opts(opts, _), do: opts
 
-  def after_request(_), do: :ok
-
   @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()}
   def get_conn(_uri, opts), do: {:ok, opts}
 end
diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
index 8ded76601..afcb4d738 100644
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@@ -69,14 +69,11 @@ defmodule Pleroma.HTTP do
         request = build_request(method, headers, options, url, body, params)
 
         adapter = Application.get_env(:tesla, :adapter)
-        client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter)
+        client = Tesla.client([Pleroma.HTTP.Middleware.FollowRedirects], adapter)
 
-        response = request(client, request)
-
-        AdapterHelper.after_request(adapter_opts)
-
-        response
+        request(client, request)
 
+      # Connection release is handled in a custom FollowRedirects middleware
       err ->
         err
     end
diff --git a/lib/pleroma/tesla/middleware/follow_redirects.ex b/lib/pleroma/tesla/middleware/follow_redirects.ex
new file mode 100644
index 000000000..f2c502c69
--- /dev/null
+++ b/lib/pleroma/tesla/middleware/follow_redirects.ex
@@ -0,0 +1,106 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2015-2020 Tymon Tobolski <https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex>
+# Copyright © 2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.HTTP.Middleware.FollowRedirects do
+  @moduledoc """
+  Pool-aware version of https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex
+
+  Follow 3xx redirects
+  ## Options
+  - `:max_redirects` - limit number of redirects (default: `5`)
+  """
+
+  alias Pleroma.Gun.ConnectionPool
+
+  @behaviour Tesla.Middleware
+
+  @max_redirects 5
+  @redirect_statuses [301, 302, 303, 307, 308]
+
+  @impl Tesla.Middleware
+  def call(env, next, opts \\ []) do
+    max = Keyword.get(opts, :max_redirects, @max_redirects)
+
+    redirect(env, next, max)
+  end
+
+  defp redirect(env, next, left) do
+    opts = env.opts[:adapter]
+
+    case Tesla.run(env, next) do
+      {:ok, %{status: status} = res} when status in @redirect_statuses and left > 0 ->
+        release_conn(opts)
+
+        case Tesla.get_header(res, "location") do
+          nil ->
+            {:ok, res}
+
+          location ->
+            location = parse_location(location, res)
+
+            case get_conn(location, opts) do
+              {:ok, opts} ->
+                %{env | opts: Keyword.put(env.opts, :adapter, opts)}
+                |> new_request(res.status, location)
+                |> redirect(next, left - 1)
+
+              e ->
+                e
+            end
+        end
+
+      {:ok, %{status: status}} when status in @redirect_statuses ->
+        release_conn(opts)
+        {:error, {__MODULE__, :too_many_redirects}}
+
+      other ->
+        unless opts[:body_as] == :chunks do
+          release_conn(opts)
+        end
+
+        other
+    end
+  end
+
+  defp get_conn(location, opts) do
+    uri = URI.parse(location)
+
+    case ConnectionPool.get_conn(uri, opts) do
+      {:ok, conn} ->
+        {:ok, Keyword.merge(opts, conn: conn)}
+
+      e ->
+        e
+    end
+  end
+
+  defp release_conn(opts) do
+    ConnectionPool.release_conn(opts[:conn])
+  end
+
+  # The 303 (See Other) redirect was added in HTTP/1.1 to indicate that the originally
+  # requested resource is not available, however a related resource (or another redirect)
+  # available via GET is available at the specified location.
+  # https://tools.ietf.org/html/rfc7231#section-6.4.4
+  defp new_request(env, 303, location), do: %{env | url: location, method: :get, query: []}
+
+  # The 307 (Temporary Redirect) status code indicates that the target
+  # resource resides temporarily under a different URI and the user agent
+  # MUST NOT change the request method (...)
+  # https://tools.ietf.org/html/rfc7231#section-6.4.7
+  defp new_request(env, 307, location), do: %{env | url: location}
+
+  defp new_request(env, _, location), do: %{env | url: location, query: []}
+
+  defp parse_location("https://" <> _rest = location, _env), do: location
+  defp parse_location("http://" <> _rest = location, _env), do: location
+
+  defp parse_location(location, env) do
+    env.url
+    |> URI.parse()
+    |> URI.merge(location)
+    |> URI.to_string()
+  end
+end
-- 
cgit v1.2.3


From 4128e3a84a2b6d75a8f92759e65ee673b47cec01 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Sun, 17 May 2020 22:16:02 +0300
Subject: HTTP: Implement max request limits

---
 lib/pleroma/application.ex             |  3 ++-
 lib/pleroma/http/adapter_helper/gun.ex | 21 +++++++++++++++++++++
 lib/pleroma/http/http.ex               | 17 ++++++++++++++++-
 3 files changed, 39 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 37fcdf293..0ffb55358 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -244,7 +244,8 @@ defmodule Pleroma.Application do
   end
 
   defp http_children(Tesla.Adapter.Gun, _) do
-    Pleroma.Gun.ConnectionPool.children()
+    Pleroma.Gun.ConnectionPool.children() ++
+      [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
   end
 
   defp http_children(_, _), do: []
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index 5b4629978..883f7f6f7 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -49,4 +49,25 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
       err -> err
     end
   end
+
+  @prefix Pleroma.Gun.ConnectionPool
+  def limiter_setup do
+    wait = Pleroma.Config.get([:connections_pool, :connection_acquisition_wait])
+    retries = Pleroma.Config.get([:connections_pool, :connection_acquisition_retries])
+
+    :pools
+    |> Pleroma.Config.get([])
+    |> Enum.each(fn {name, opts} ->
+      max_running = Keyword.get(opts, :size, 50)
+      max_waiting = Keyword.get(opts, :max_waiting, 10)
+
+      :ok =
+        ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting,
+          wait: wait,
+          max_retries: retries
+        )
+    end)
+
+    :ok
+  end
 end
diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
index afcb4d738..6128bc4cf 100644
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@@ -71,7 +71,13 @@ defmodule Pleroma.HTTP do
         adapter = Application.get_env(:tesla, :adapter)
         client = Tesla.client([Pleroma.HTTP.Middleware.FollowRedirects], adapter)
 
-        request(client, request)
+        maybe_limit(
+          fn ->
+            request(client, request)
+          end,
+          adapter,
+          adapter_opts
+        )
 
       # Connection release is handled in a custom FollowRedirects middleware
       err ->
@@ -92,4 +98,13 @@ defmodule Pleroma.HTTP do
     |> Builder.add_param(:query, :query, params)
     |> Builder.convert_to_keyword()
   end
+
+  @prefix Pleroma.Gun.ConnectionPool
+  defp maybe_limit(fun, Tesla.Adapter.Gun, opts) do
+    ConcurrentLimiter.limit(:"#{@prefix}.#{opts[:pool] || :default}", fun)
+  end
+
+  defp maybe_limit(fun, _, _) do
+    fun.()
+  end
 end
-- 
cgit v1.2.3


From 00926a63fb174a8bcb2f496921c5d17e04e44b1d Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 16 Jun 2020 16:20:28 +0300
Subject: Adapter Helper: Use built-in ip address type

---
 lib/pleroma/http/adapter_helper.ex | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index bcb9b2b1e..8ca433732 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -8,12 +8,8 @@ defmodule Pleroma.HTTP.AdapterHelper do
   """
   @defaults [pool: :federation]
 
-  @type ip_address :: ipv4_address() | ipv6_address()
-  @type ipv4_address :: {0..255, 0..255, 0..255, 0..255}
-  @type ipv6_address ::
-          {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535}
   @type proxy_type() :: :socks4 | :socks5
-  @type host() :: charlist() | ip_address()
+  @type host() :: charlist() | :inet.ip_address()
 
   alias Pleroma.Config
   alias Pleroma.HTTP.AdapterHelper
@@ -114,7 +110,7 @@ defmodule Pleroma.HTTP.AdapterHelper do
     end
   end
 
-  @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address()
+  @spec parse_host(String.t() | atom() | charlist()) :: charlist() | :inet.ip_address()
   def parse_host(host) when is_list(host), do: host
   def parse_host(host) when is_atom(host), do: to_charlist(host)
 
-- 
cgit v1.2.3


From 7882f28569bfaee2996d059990eec279415f0785 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Jun 2020 12:54:13 +0300
Subject: Use erlang monotonic time for CRF calculation

---
 lib/pleroma/gun/connection_pool/worker.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index 8467325f3..418cb18c1 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
   def init([key, uri, opts, client_pid]) do
     with {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
          Process.link(conn_pid) do
-      time = :os.system_time(:second)
+      time = :erlang.monotonic_time()
 
       {_, _} =
         Registry.update_value(@registry, key, fn _ ->
@@ -31,7 +31,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
 
   @impl true
   def handle_cast({:add_client, client_pid, send_pid_back}, %{key: key} = state) do
-    time = :os.system_time(:second)
+    time = :erlang.monotonic_time()
 
     {{conn_pid, _, _, _}, _} =
       Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
-- 
cgit v1.2.3


From 007843b75e0c7087dad1ef932224b21327d81793 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 23 Jun 2020 15:38:45 +0300
Subject: Add documentation for new connection pool settings and remove some

`:retry_timeout` and `:retry` got removed because reconnecting on failure is
something the new pool intentionally doesn't do.

`:max_overflow` had to go in favor of `:max_waiting`, I didn't reuse the key because
the settings are very different in their behaviour.

`:checkin_timeout` got removed in favor of `:connection_acquisition_wait`,
I didn't reuse the key because the settings are somewhat different.

I didn't do any migrations/deprecation warnings/changelog entries because
these settings were never in stable.
---
 lib/pleroma/gun/conn.ex | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index 77f78c7ff..9dc8880db 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -13,8 +13,6 @@ defmodule Pleroma.Gun.Conn do
     opts =
       opts
       |> Enum.into(%{})
-      |> Map.put_new(:retry, pool_opts[:retry] || 1)
-      |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000)
       |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000)
       |> Map.put_new(:supervise, false)
       |> maybe_add_tls_opts(uri)
-- 
cgit v1.2.3


From 37f1e781cb19594a6534efbc4d28e793d5960915 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 23 Jun 2020 20:36:21 +0300
Subject: Gun adapter helper: fix wildcard cert issues on OTP 23

See https://bugs.erlang.org/browse/ERL-1260 for more info.

The ssl match function is basically copied from mint, except
that `:string.lowercase/1` was replaced by `:string.casefold`.
It was a TODO in mint's code, so might as well do it since we don't need
to support OTP <20.

Closes #1834
---
 lib/pleroma/http/adapter_helper/gun.ex | 29 ++++++++++++++++++++++++++++-
 1 file changed, 28 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index 883f7f6f7..07aaed7f6 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -39,9 +39,36 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   defp add_scheme_opts(opts, %{scheme: "https"}) do
     opts
     |> Keyword.put(:certificates_verification, true)
-    |> Keyword.put(:tls_opts, log_level: :warning)
+    |> Keyword.put(:tls_opts,
+      log_level: :warning,
+      customize_hostname_check: [match_fun: &ssl_match_fun/2]
+    )
   end
 
+  # ssl_match_fun is adapted from [Mint](https://github.com/elixir-mint/mint)
+  # Copyright 2018 Eric Meadows-Jönsson and Andrea Leopardi
+
+  # Wildcard domain handling for DNS ID entries in the subjectAltName X.509
+  # extension. Note that this is a subset of the wildcard patterns implemented
+  # by OTP when matching against the subject CN attribute, but this is the only
+  # wildcard usage defined by the CA/Browser Forum's Baseline Requirements, and
+  # therefore the only pattern used in commercially issued certificates.
+  defp ssl_match_fun({:dns_id, reference}, {:dNSName, [?*, ?. | presented]}) do
+    case domain_without_host(reference) do
+      '' ->
+        :default
+
+      domain ->
+        :string.casefold(domain) == :string.casefold(presented)
+    end
+  end
+
+  defp ssl_match_fun(_reference, _presented), do: :default
+
+  defp domain_without_host([]), do: []
+  defp domain_without_host([?. | domain]), do: domain
+  defp domain_without_host([_ | more]), do: domain_without_host(more)
+
   @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
   def get_conn(uri, opts) do
     case ConnectionPool.get_conn(uri, opts) do
-- 
cgit v1.2.3


From 12fa5541f01ca5cfe082a62dac3317da78043e8f Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 30 Jun 2020 15:58:53 +0300
Subject: FollowRedirects: Unconditionally release the connection if there is
 an error

There is no need for streaming the body if there is no body
---
 lib/pleroma/tesla/middleware/follow_redirects.ex | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/tesla/middleware/follow_redirects.ex b/lib/pleroma/tesla/middleware/follow_redirects.ex
index f2c502c69..5a7032215 100644
--- a/lib/pleroma/tesla/middleware/follow_redirects.ex
+++ b/lib/pleroma/tesla/middleware/follow_redirects.ex
@@ -55,6 +55,10 @@ defmodule Pleroma.HTTP.Middleware.FollowRedirects do
         release_conn(opts)
         {:error, {__MODULE__, :too_many_redirects}}
 
+      {:error, _} = e ->
+        release_conn(opts)
+        e
+
       other ->
         unless opts[:body_as] == :chunks do
           release_conn(opts)
-- 
cgit v1.2.3


From 9b73c35ca8b051316815461247b802bc8567854f Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Tue, 30 Jun 2020 18:35:15 +0300
Subject: Request limiter setup: consider {:error, :existing} a success

When the application restarts (which happens after certain config
changes), the limiters are not destroyed, so `ConcurrentLimiter.new`
will produce {:error, :existing}
---
 lib/pleroma/http/adapter_helper/gun.ex | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index 07aaed7f6..b8c4cc59c 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -88,11 +88,17 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
       max_running = Keyword.get(opts, :size, 50)
       max_waiting = Keyword.get(opts, :max_waiting, 10)
 
-      :ok =
+      result =
         ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting,
           wait: wait,
           max_retries: retries
         )
+
+      case result do
+        :ok -> :ok
+        {:error, :existing} -> :ok
+        e -> raise e
+      end
     end)
 
     :ok
-- 
cgit v1.2.3


From a705637dcf7ffe063c9c0f3f190f57e44aaa63f2 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 2 Jul 2020 01:53:27 +0300
Subject: Connection Pool: fix LRFU implementation to not actually be LRU

The numbers of the native time unit were so small the CRF was always 1,
making it an LRU. This commit switches the time to miliseconds and changes
the time delta multiplier to the one yielding mostly highest hit rates according
to the paper
---
 lib/pleroma/gun/connection_pool/worker.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index 418cb18c1..ec0502621 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -12,7 +12,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
   def init([key, uri, opts, client_pid]) do
     with {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
          Process.link(conn_pid) do
-      time = :erlang.monotonic_time()
+      time = :erlang.monotonic_time(:millisecond)
 
       {_, _} =
         Registry.update_value(@registry, key, fn _ ->
@@ -31,7 +31,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
 
   @impl true
   def handle_cast({:add_client, client_pid, send_pid_back}, %{key: key} = state) do
-    time = :erlang.monotonic_time()
+    time = :erlang.monotonic_time(:millisecond)
 
     {{conn_pid, _, _, _}, _} =
       Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
@@ -116,6 +116,6 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
 
   # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478
   defp crf(time_delta, prev_crf) do
-    1 + :math.pow(0.5, time_delta / 100) * prev_crf
+    1 + :math.pow(0.5, 0.0001 * time_delta) * prev_crf
   end
 end
-- 
cgit v1.2.3


From 46dd276d686e49676101e2af743aad61393f4b70 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Tue, 7 Jul 2020 18:56:17 +0200
Subject: ConnectionPool.Worker: Open gun conn in continue instead of init

---
 lib/pleroma/gun/connection_pool/worker.ex | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index ec0502621..6ee622fb0 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -9,7 +9,12 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
   end
 
   @impl true
-  def init([key, uri, opts, client_pid]) do
+  def init([_key, _uri, _opts, _client_pid] = opts) do
+    {:ok, nil, {:continue, {:connect, opts}}}
+  end
+
+  @impl true
+  def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
     with {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
          Process.link(conn_pid) do
       time = :erlang.monotonic_time(:millisecond)
@@ -21,7 +26,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
 
       send(client_pid, {:conn_pid, conn_pid})
 
-      {:ok,
+      {:noreply,
        %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}},
        :hibernate}
     else
-- 
cgit v1.2.3


From 6a0f2bdf8ceb4127678cc55406a02d41c7fb0ed7 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Wed, 8 Jul 2020 13:01:02 +0200
Subject: Ensure connections error get known by the caller

---
 lib/pleroma/gun/connection_pool.ex        | 22 ++++++++++++----------
 lib/pleroma/gun/connection_pool/worker.ex |  3 ++-
 lib/pleroma/http/adapter_helper/gun.ex    |  2 +-
 3 files changed, 15 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex
index e951872fe..d3eead7d8 100644
--- a/lib/pleroma/gun/connection_pool.ex
+++ b/lib/pleroma/gun/connection_pool.ex
@@ -16,7 +16,7 @@ defmodule Pleroma.Gun.ConnectionPool do
     case Registry.lookup(@registry, key) do
       # The key has already been registered, but connection is not up yet
       [{worker_pid, nil}] ->
-        get_gun_pid_from_worker(worker_pid)
+        get_gun_pid_from_worker(worker_pid, true)
 
       [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] ->
         GenServer.cast(worker_pid, {:add_client, self(), false})
@@ -27,13 +27,11 @@ defmodule Pleroma.Gun.ConnectionPool do
         # so we open the connection in the process directly and send it's pid back
         # We trust gun to handle timeouts by itself
         case WorkerSupervisor.start_worker([key, uri, opts, self()]) do
-          {:ok, _worker_pid} ->
-            receive do
-              {:conn_pid, pid} -> {:ok, pid}
-            end
+          {:ok, worker_pid} ->
+            get_gun_pid_from_worker(worker_pid, false)
 
           {:error, {:already_started, worker_pid}} ->
-            get_gun_pid_from_worker(worker_pid)
+            get_gun_pid_from_worker(worker_pid, true)
 
           err ->
             err
@@ -41,17 +39,21 @@ defmodule Pleroma.Gun.ConnectionPool do
     end
   end
 
-  defp get_gun_pid_from_worker(worker_pid) do
+  defp get_gun_pid_from_worker(worker_pid, register) do
     # GenServer.call will block the process for timeout length if
     # the server crashes on startup (which will happen if gun fails to connect)
     # so instead we use cast + monitor
 
     ref = Process.monitor(worker_pid)
-    GenServer.cast(worker_pid, {:add_client, self(), true})
+    if register, do: GenServer.cast(worker_pid, {:add_client, self(), true})
 
     receive do
-      {:conn_pid, pid} -> {:ok, pid}
-      {:DOWN, ^ref, :process, ^worker_pid, reason} -> reason
+      {:conn_pid, pid} ->
+        Process.demonitor(ref)
+        {:ok, pid}
+
+      {:DOWN, ^ref, :process, ^worker_pid, reason} ->
+        {:error, reason}
     end
   end
 
diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index 6ee622fb0..16a508ad9 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -30,7 +30,8 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
        %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}},
        :hibernate}
     else
-      err -> {:stop, err}
+      err ->
+        {:stop, err, nil}
     end
   end
 
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index b8c4cc59c..74677ddb5 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -14,7 +14,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
     connect_timeout: 5_000,
     domain_lookup_timeout: 5_000,
     tls_handshake_timeout: 5_000,
-    retry: 1,
+    retry: 0,
     retry_timeout: 1000,
     await_up_timeout: 5_000
   ]
-- 
cgit v1.2.3


From 23d714ed3038a24eb78314d52807c46d8b8de2f3 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Wed, 8 Jul 2020 13:22:42 +0200
Subject: Fix race in enforcer/reclaimer start

---
 lib/pleroma/gun/connection_pool/reclaimer.ex       | 85 ++++++++++++++++++++++
 .../gun/connection_pool/worker_supervisor.ex       | 81 +--------------------
 2 files changed, 89 insertions(+), 77 deletions(-)
 create mode 100644 lib/pleroma/gun/connection_pool/reclaimer.ex

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex
new file mode 100644
index 000000000..1793ac3ee
--- /dev/null
+++ b/lib/pleroma/gun/connection_pool/reclaimer.ex
@@ -0,0 +1,85 @@
+defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
+  use GenServer, restart: :temporary
+
+  @registry Pleroma.Gun.ConnectionPool
+
+  def start_monitor() do
+    pid =
+      case :gen_server.start(__MODULE__, [], name: {:via, Registry, {@registry, "reclaimer"}}) do
+        {:ok, pid} ->
+          pid
+
+        {:error, {:already_registered, pid}} ->
+          pid
+      end
+
+    {pid, Process.monitor(pid)}
+  end
+
+  @impl true
+  def init(_) do
+    {:ok, nil, {:continue, :reclaim}}
+  end
+
+  @impl true
+  def handle_continue(:reclaim, _) do
+    max_connections = Pleroma.Config.get([:connections_pool, :max_connections])
+
+    reclaim_max =
+      [:connections_pool, :reclaim_multiplier]
+      |> Pleroma.Config.get()
+      |> Kernel.*(max_connections)
+      |> round
+      |> max(1)
+
+    :telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{
+      max_connections: max_connections,
+      reclaim_max: reclaim_max
+    })
+
+    # :ets.fun2ms(
+    # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
+    #   {worker_pid, crf, last_reference} end)
+    unused_conns =
+      Registry.select(
+        @registry,
+        [
+          {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]}
+        ]
+      )
+
+    case unused_conns do
+      [] ->
+        :telemetry.execute(
+          [:pleroma, :connection_pool, :reclaim, :stop],
+          %{reclaimed_count: 0},
+          %{
+            max_connections: max_connections
+          }
+        )
+
+        {:stop, :no_unused_conns, nil}
+
+      unused_conns ->
+        reclaimed =
+          unused_conns
+          |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} ->
+            crf1 <= crf2 and last_reference1 <= last_reference2
+          end)
+          |> Enum.take(reclaim_max)
+
+        reclaimed
+        |> Enum.each(fn {pid, _, _} ->
+          DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid)
+        end)
+
+        :telemetry.execute(
+          [:pleroma, :connection_pool, :reclaim, :stop],
+          %{reclaimed_count: Enum.count(reclaimed)},
+          %{max_connections: max_connections}
+        )
+
+        {:stop, :normal, nil}
+    end
+  end
+end
diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
index 5cb8d488a..39615c956 100644
--- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex
+++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex
@@ -29,89 +29,16 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do
     end
   end
 
-  @registry Pleroma.Gun.ConnectionPool
-  @enforcer_key "enforcer"
   defp free_pool do
-    case Registry.lookup(@registry, @enforcer_key) do
-      [] ->
-        pid =
-          spawn(fn ->
-            {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil)
-            max_connections = Pleroma.Config.get([:connections_pool, :max_connections])
-
-            reclaim_max =
-              [:connections_pool, :reclaim_multiplier]
-              |> Pleroma.Config.get()
-              |> Kernel.*(max_connections)
-              |> round
-              |> max(1)
-
-            :telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{
-              max_connections: max_connections,
-              reclaim_max: reclaim_max
-            })
-
-            # :ets.fun2ms(
-            # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] ->
-            #   {worker_pid, crf, last_reference} end)
-            unused_conns =
-              Registry.select(
-                @registry,
-                [
-                  {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}],
-                   [{{:"$1", :"$3", :"$4"}}]}
-                ]
-              )
-
-            case unused_conns do
-              [] ->
-                :telemetry.execute(
-                  [:pleroma, :connection_pool, :reclaim, :stop],
-                  %{reclaimed_count: 0},
-                  %{
-                    max_connections: max_connections
-                  }
-                )
-
-                exit(:no_unused_conns)
-
-              unused_conns ->
-                reclaimed =
-                  unused_conns
-                  |> Enum.sort(fn {_pid1, crf1, last_reference1},
-                                  {_pid2, crf2, last_reference2} ->
-                    crf1 <= crf2 and last_reference1 <= last_reference2
-                  end)
-                  |> Enum.take(reclaim_max)
-
-                reclaimed
-                |> Enum.each(fn {pid, _, _} ->
-                  DynamicSupervisor.terminate_child(__MODULE__, pid)
-                end)
-
-                :telemetry.execute(
-                  [:pleroma, :connection_pool, :reclaim, :stop],
-                  %{reclaimed_count: Enum.count(reclaimed)},
-                  %{max_connections: max_connections}
-                )
-            end
-          end)
-
-        wait_for_enforcer_finish(pid)
-
-      [{pid, _}] ->
-        wait_for_enforcer_finish(pid)
-    end
+    wait_for_reclaimer_finish(Pleroma.Gun.ConnectionPool.Reclaimer.start_monitor())
   end
 
-  defp wait_for_enforcer_finish(pid) do
-    ref = Process.monitor(pid)
-
+  defp wait_for_reclaimer_finish({pid, mon}) do
     receive do
-      {:DOWN, ^ref, :process, ^pid, :no_unused_conns} ->
+      {:DOWN, ^mon, :process, ^pid, :no_unused_conns} ->
         :error
 
-      {:DOWN, ^ref, :process, ^pid, :normal} ->
+      {:DOWN, ^mon, :process, ^pid, :normal} ->
         :ok
     end
   end
-- 
cgit v1.2.3


From 53ba6815b170d7d5c11282933b99730209f526ea Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Wed, 8 Jul 2020 13:58:38 +0200
Subject: parentheses...

---
 lib/pleroma/gun/connection_pool/reclaimer.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex
index 1793ac3ee..cea800882 100644
--- a/lib/pleroma/gun/connection_pool/reclaimer.ex
+++ b/lib/pleroma/gun/connection_pool/reclaimer.ex
@@ -3,7 +3,7 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do
 
   @registry Pleroma.Gun.ConnectionPool
 
-  def start_monitor() do
+  def start_monitor do
     pid =
       case :gen_server.start(__MODULE__, [], name: {:via, Registry, {@registry, "reclaimer"}}) do
         {:ok, pid} ->
-- 
cgit v1.2.3


From ce1a42bd04bcf352ea1565b411444a98261b0a96 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Wed, 8 Jul 2020 15:12:09 +0200
Subject: Simplify TLS opts

- `verify_fun` is not useful now
- use `customize_check_hostname` (OTP 20+ so OK)
- `partial_chain` is useless as of OTP 21.1 (wasn't there, but hackney/..
uses it)
---
 lib/pleroma/gun/conn.ex                |  5 ++---
 lib/pleroma/http/adapter_helper/gun.ex | 28 ----------------------------
 2 files changed, 2 insertions(+), 31 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index 9dc8880db..5c12e8153 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -28,9 +28,8 @@ defmodule Pleroma.Gun.Conn do
       cacertfile: CAStore.file_path(),
       depth: 20,
       reuse_sessions: false,
-      verify_fun:
-        {&:ssl_verify_hostname.verify_fun/3,
-         [check_hostname: Pleroma.HTTP.AdapterHelper.format_host(host)]}
+      log_level: :warning,
+      customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)]
     ]
 
     tls_opts =
diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex
index 74677ddb5..b4ff8306c 100644
--- a/lib/pleroma/http/adapter_helper/gun.ex
+++ b/lib/pleroma/http/adapter_helper/gun.ex
@@ -39,36 +39,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do
   defp add_scheme_opts(opts, %{scheme: "https"}) do
     opts
     |> Keyword.put(:certificates_verification, true)
-    |> Keyword.put(:tls_opts,
-      log_level: :warning,
-      customize_hostname_check: [match_fun: &ssl_match_fun/2]
-    )
   end
 
-  # ssl_match_fun is adapted from [Mint](https://github.com/elixir-mint/mint)
-  # Copyright 2018 Eric Meadows-Jönsson and Andrea Leopardi
-
-  # Wildcard domain handling for DNS ID entries in the subjectAltName X.509
-  # extension. Note that this is a subset of the wildcard patterns implemented
-  # by OTP when matching against the subject CN attribute, but this is the only
-  # wildcard usage defined by the CA/Browser Forum's Baseline Requirements, and
-  # therefore the only pattern used in commercially issued certificates.
-  defp ssl_match_fun({:dns_id, reference}, {:dNSName, [?*, ?. | presented]}) do
-    case domain_without_host(reference) do
-      '' ->
-        :default
-
-      domain ->
-        :string.casefold(domain) == :string.casefold(presented)
-    end
-  end
-
-  defp ssl_match_fun(_reference, _presented), do: :default
-
-  defp domain_without_host([]), do: []
-  defp domain_without_host([?. | domain]), do: domain
-  defp domain_without_host([_ | more]), do: domain_without_host(more)
-
   @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()}
   def get_conn(uri, opts) do
     case ConnectionPool.get_conn(uri, opts) do
-- 
cgit v1.2.3


From afd378f84c4c1b784eba11b35c21e0b6ae3d7915 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Wed, 8 Jul 2020 16:02:57 +0200
Subject: host is now useless

---
 lib/pleroma/gun/conn.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index 5c12e8153..a3f75a4bb 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -22,7 +22,7 @@ defmodule Pleroma.Gun.Conn do
 
   defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts
 
-  defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do
+  defp maybe_add_tls_opts(opts, %URI{scheme: "https"}) do
     tls_opts = [
       verify: :verify_peer,
       cacertfile: CAStore.file_path(),
-- 
cgit v1.2.3


From 6d583bcc3b23c0c16aefa3f34155e7e15b745b01 Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Mon, 13 Jul 2020 10:44:36 +0200
Subject: Set a default timeout for Gun adapter timeout

---
 lib/pleroma/http/adapter_helper.ex | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex
index 8ca433732..9ec3836b0 100644
--- a/lib/pleroma/http/adapter_helper.ex
+++ b/lib/pleroma/http/adapter_helper.ex
@@ -44,15 +44,17 @@ defmodule Pleroma.HTTP.AdapterHelper do
   @spec options(URI.t(), keyword()) :: keyword()
   def options(%URI{} = uri, opts \\ []) do
     @defaults
-    |> pool_timeout()
+    |> put_timeout()
     |> Keyword.merge(opts)
     |> adapter_helper().options(uri)
   end
 
-  defp pool_timeout(opts) do
+  # For Hackney, this is the time a connection can stay idle in the pool.
+  # For Gun, this is the timeout to receive a message from Gun.
+  defp put_timeout(opts) do
     {config_key, default} =
       if adapter() == Tesla.Adapter.Gun do
-        {:pools, Config.get([:pools, :default, :timeout])}
+        {:pools, Config.get([:pools, :default, :timeout], 5_000)}
       else
         {:hackney_pools, 10_000}
       end
-- 
cgit v1.2.3


From 7115c5f82efe1ca1817da3152ba3cbc66e0da1a4 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 15 Jul 2020 15:58:08 +0300
Subject: ConnectionPool.Worker: do not stop with an error when there is a
 timeout

This produced error log messages about GenServer termination
every time the connection was not open due to a timeout.

Instead we stop with `{:shutdown, <gun_error>}` since shutting down
when the connection can't be established is normal behavior.
---
 lib/pleroma/gun/connection_pool.ex        | 5 ++++-
 lib/pleroma/gun/connection_pool/worker.ex | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex
index d3eead7d8..8b41a668c 100644
--- a/lib/pleroma/gun/connection_pool.ex
+++ b/lib/pleroma/gun/connection_pool.ex
@@ -53,7 +53,10 @@ defmodule Pleroma.Gun.ConnectionPool do
         {:ok, pid}
 
       {:DOWN, ^ref, :process, ^worker_pid, reason} ->
-        {:error, reason}
+        case reason do
+          {:shutdown, error} -> error
+          _ -> {:error, reason}
+        end
     end
   end
 
diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index 16a508ad9..f33447cb6 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -31,7 +31,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
        :hibernate}
     else
       err ->
-        {:stop, err, nil}
+        {:stop, {:shutdown, err}, nil}
     end
   end
 
-- 
cgit v1.2.3


From d29b8997f4a3601eac7f2e1e57de27a67df6699c Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Wed, 15 Jul 2020 15:25:33 +0200
Subject: MastoAPI: fix & test giving MRF reject reasons

---
 lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
index 12be530c9..9bb2ef117 100644
--- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
@@ -172,6 +172,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
         with_direct_conversation_id: true
       )
     else
+      {:error, {:reject, message}} ->
+        conn
+        |> put_status(:unprocessable_entity)
+        |> json(%{error: message})
+
       {:error, message} ->
         conn
         |> put_status(:unprocessable_entity)
-- 
cgit v1.2.3


From 3be64556dbe5618de3429a481f41eff917053ce8 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 16 Jul 2020 13:11:03 -0500
Subject: Improve TOTP token and recovery input fields in OAuth login

---
 lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex | 2 +-
 lib/pleroma/web/templates/o_auth/mfa/totp.html.eex     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
index 750f65386..5ab59b57b 100644
--- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
+++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex
@@ -10,7 +10,7 @@
 <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
 <div class="input">
   <%= label f, :code, "Recovery code" %>
-  <%= text_input f, :code %>
+  <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, spellcheck: false] %>
   <%= hidden_input f, :mfa_token, value: @mfa_token %>
   <%= hidden_input f, :state, value: @state %>
   <%= hidden_input f, :redirect_uri, value: @redirect_uri %>
diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
index af6e546b0..8323ff8a1 100644
--- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
+++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
@@ -10,7 +10,7 @@
 <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
 <div class="input">
   <%= label f, :code, "Authentication code" %>
-  <%= text_input f, :code %>
+  <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, maxlength: 6, pattern: "[0-9]{6}", spellcheck: false] %>
   <%= hidden_input f, :mfa_token, value: @mfa_token %>
   <%= hidden_input f, :state, value: @state %>
   <%= hidden_input f, :redirect_uri, value: @redirect_uri %>
-- 
cgit v1.2.3


From 62438530e24d9553b8c1240ad7a39ea0906832b9 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 17 Jul 2020 08:19:49 -0500
Subject: TOTP length is configurable, so we can't hardcode this here.

---
 lib/pleroma/web/templates/o_auth/mfa/totp.html.eex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
index 8323ff8a1..af85777eb 100644
--- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
+++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex
@@ -10,7 +10,7 @@
 <%= form_for @conn, mfa_verify_path(@conn, :verify), [as: "mfa"], fn f -> %>
 <div class="input">
   <%= label f, :code, "Authentication code" %>
-  <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, maxlength: 6, pattern: "[0-9]{6}", spellcheck: false] %>
+  <%= text_input f, :code, [autocomplete: false, autocorrect: "off", autocapitalize: "off", autofocus: true, pattern: "[0-9]*", spellcheck: false] %>
   <%= hidden_input f, :mfa_token, value: @mfa_token %>
   <%= hidden_input f, :state, value: @state %>
   <%= hidden_input f, :redirect_uri, value: @redirect_uri %>
-- 
cgit v1.2.3


From af376cbffbae3ae594e594813873719dfd69664e Mon Sep 17 00:00:00 2001
From: Alexander Strizhakov <alex.strizhakov@gmail.com>
Date: Fri, 17 Jul 2020 18:06:05 +0300
Subject: using atom keys in search params

---
 lib/pleroma/gopher/server.ex | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex
index 3d56d50a9..e9f54c4c0 100644
--- a/lib/pleroma/gopher/server.ex
+++ b/lib/pleroma/gopher/server.ex
@@ -96,16 +96,18 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
 
   def response("/main/public") do
     posts =
-      ActivityPub.fetch_public_activities(%{"type" => ["Create"], "local_only" => true})
-      |> render_activities
+      %{type: ["Create"], local_only: true}
+      |> ActivityPub.fetch_public_activities()
+      |> render_activities()
 
     info("Welcome to the Public Timeline!") <> posts <> ".\r\n"
   end
 
   def response("/main/all") do
     posts =
-      ActivityPub.fetch_public_activities(%{"type" => ["Create"]})
-      |> render_activities
+      %{type: ["Create"]}
+      |> ActivityPub.fetch_public_activities()
+      |> render_activities()
 
     info("Welcome to the Federated Timeline!") <> posts <> ".\r\n"
   end
@@ -130,13 +132,14 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
   def response("/users/" <> nickname) do
     with %User{} = user <- User.get_cached_by_nickname(nickname) do
       params = %{
-        "type" => ["Create"],
-        "actor_id" => user.ap_id
+        type: ["Create"],
+        actor_id: user.ap_id
       }
 
       activities =
-        ActivityPub.fetch_public_activities(params)
-        |> render_activities
+        params
+        |> ActivityPub.fetch_public_activities()
+        |> render_activities()
 
       info("Posts by #{user.nickname}") <> activities <> ".\r\n"
     else
-- 
cgit v1.2.3


From 20a496d2cbea18c563694c7026c0e951e99cfc3b Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 17 Jul 2020 10:45:41 -0500
Subject: Expose the post formats in /api/v1/instance

---
 lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex
index 5deb0d7ed..cd3bc7f00 100644
--- a/lib/pleroma/web/mastodon_api/views/instance_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex
@@ -41,7 +41,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do
           account_activation_required: Keyword.get(instance, :account_activation_required),
           features: features(),
           federation: federation(),
-          fields_limits: fields_limits()
+          fields_limits: fields_limits(),
+          post_formats: Config.get([:instance, :allowed_post_formats])
         },
         vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key)
       }
-- 
cgit v1.2.3


From 48f8b26c92880c0898daac3d691c61be0b891d0b Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Thu, 16 Jul 2020 21:39:10 -0500
Subject: OpenAPI: Add :id to follower/following endpoints, fixes #1958

---
 lib/pleroma/web/api_spec/operations/account_operation.ex | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex
index 952d9347b..50c8e0242 100644
--- a/lib/pleroma/web/api_spec/operations/account_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/account_operation.ex
@@ -159,6 +159,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
         "Accounts which follow the given account, if network is not hidden by the account owner.",
       parameters: [
         %Reference{"$ref": "#/components/parameters/accountIdOrNickname"},
+        Operation.parameter(:id, :query, :string, "ID of the resource owner"),
         with_relationships_param() | pagination_params()
       ],
       responses: %{
@@ -177,6 +178,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do
         "Accounts which the given account is following, if network is not hidden by the account owner.",
       parameters: [
         %Reference{"$ref": "#/components/parameters/accountIdOrNickname"},
+        Operation.parameter(:id, :query, :string, "ID of the resource owner"),
         with_relationships_param() | pagination_params()
       ],
       responses: %{200 => Operation.response("Accounts", "application/json", array_of_accounts())}
-- 
cgit v1.2.3


From 7ce722ce3e3dbc633324ff0ccaeddc467397ac5e Mon Sep 17 00:00:00 2001
From: KokaKiwi <kokakiwi+pleroma@kokakiwi.net>
Date: Sat, 18 Jul 2020 12:55:04 +0200
Subject: Fix /api/pleroma/emoji/packs index endpoint.

---
 lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
index 33ecd1f70..866901344 100644
--- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
@@ -22,7 +22,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
   )
 
   @skip_plugs [Pleroma.Plugs.OAuthScopesPlug, Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug]
-  plug(:skip_plug, @skip_plugs when action in [:archive, :show, :list])
+  plug(:skip_plug, @skip_plugs when action in [:index, :show, :archive])
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaEmojiPackOperation
 
-- 
cgit v1.2.3


From 4bac25e6f5d332b06e481d25b80efb62026c6a1e Mon Sep 17 00:00:00 2001
From: href <href@random.sh>
Date: Sat, 18 Jul 2020 13:17:38 +0200
Subject: Don't enable Pleroma.HTTP.Middleware.FollowRedirects unless Gun is
 used

---
 lib/pleroma/http/http.ex | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex
index 6128bc4cf..b37b3fa89 100644
--- a/lib/pleroma/http/http.ex
+++ b/lib/pleroma/http/http.ex
@@ -69,7 +69,8 @@ defmodule Pleroma.HTTP do
         request = build_request(method, headers, options, url, body, params)
 
         adapter = Application.get_env(:tesla, :adapter)
-        client = Tesla.client([Pleroma.HTTP.Middleware.FollowRedirects], adapter)
+
+        client = Tesla.client(adapter_middlewares(adapter), adapter)
 
         maybe_limit(
           fn ->
@@ -107,4 +108,10 @@ defmodule Pleroma.HTTP do
   defp maybe_limit(fun, _, _) do
     fun.()
   end
+
+  defp adapter_middlewares(Tesla.Adapter.Gun) do
+    [Pleroma.HTTP.Middleware.FollowRedirects]
+  end
+
+  defp adapter_middlewares(_), do: []
 end
-- 
cgit v1.2.3


From 204dddcfaaa5ff1113ef2f772ce5d6fcbbaaec6e Mon Sep 17 00:00:00 2001
From: Alex Gleason <alex@alexgleason.me>
Date: Mon, 20 Jul 2020 13:45:05 -0500
Subject: Pleroma.Formatter can have partial updates

---
 lib/pleroma/config/config_db.ex | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index f8141ced8..e5b7811aa 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -156,7 +156,6 @@ defmodule Pleroma.ConfigDB do
       {:quack, :meta},
       {:mime, :types},
       {:cors_plug, [:max_age, :methods, :expose, :headers]},
-      {:linkify, :opts},
       {:swarm, :node_blacklist},
       {:logger, :backends}
     ]
-- 
cgit v1.2.3


From bdb3375933b17ffd596d9d870d797fcc47a4828b Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Tue, 21 Jul 2020 16:06:46 +0400
Subject: Allow unblocking a domain via query params

---
 lib/pleroma/web/api_spec/operations/domain_block_operation.ex       | 6 +++---
 lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex | 5 +++++
 2 files changed, 8 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex
index 049bcf931..8234394f9 100644
--- a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex
@@ -57,6 +57,7 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do
       description: "Remove a domain block, if it exists in the user's array of blocked domains.",
       operationId: "DomainBlockController.delete",
       requestBody: domain_block_request(),
+      parameters: [Operation.parameter(:domain, :query, %Schema{type: :string}, "Domain name")],
       security: [%{"oAuth" => ["follow", "write:blocks"]}],
       responses: %{
         200 => Operation.response("Empty object", "application/json", %Schema{type: :object})
@@ -71,10 +72,9 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do
         type: :object,
         properties: %{
           domain: %Schema{type: :string}
-        },
-        required: [:domain]
+        }
       },
-      required: true,
+      required: false,
       example: %{
         "domain" => "facebook.com"
       }
diff --git a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex
index 825b231ab..117e89426 100644
--- a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex
@@ -37,4 +37,9 @@ defmodule Pleroma.Web.MastodonAPI.DomainBlockController do
     User.unblock_domain(blocker, domain)
     json(conn, %{})
   end
+
+  def delete(%{assigns: %{user: blocker}} = conn, %{domain: domain}) do
+    User.unblock_domain(blocker, domain)
+    json(conn, %{})
+  end
 end
-- 
cgit v1.2.3


From 696c13ce54aff25737f8f753a94747d79b9c54b0 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 21 Jul 2020 22:17:34 +0000
Subject: Revert "Merge branch 'linkify' into 'develop'"

This reverts merge request !2677
---
 lib/pleroma/config/config_db.ex       |  1 +
 lib/pleroma/formatter.ex              | 26 +++++++++++---------------
 lib/pleroma/web/rich_media/helpers.ex |  4 ++--
 3 files changed, 14 insertions(+), 17 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index e5b7811aa..1a89d8895 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -156,6 +156,7 @@ defmodule Pleroma.ConfigDB do
       {:quack, :meta},
       {:mime, :types},
       {:cors_plug, [:max_age, :methods, :expose, :headers]},
+      {:auto_linker, :opts},
       {:swarm, :node_blacklist},
       {:logger, :backends}
     ]
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 0c450eae4..02a93a8dc 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -10,15 +10,11 @@ defmodule Pleroma.Formatter do
   @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
 
-  defp linkify_opts do
-    Pleroma.Config.get(Pleroma.Formatter) ++
-      [
-        hashtag: true,
-        hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
-        mention: true,
-        mention_handler: &Pleroma.Formatter.mention_handler/4
-      ]
-  end
+  @auto_linker_config hashtag: true,
+                      hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
+                      mention: true,
+                      mention_handler: &Pleroma.Formatter.mention_handler/4,
+                      scheme: true
 
   def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do
     case User.get_cached_by_nickname(nickname) do
@@ -84,19 +80,19 @@ defmodule Pleroma.Formatter do
   @spec linkify(String.t(), keyword()) ::
           {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
   def linkify(text, options \\ []) do
-    options = linkify_opts() ++ options
+    options = options ++ @auto_linker_config
 
     if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
       %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
 
-      {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options)
-      {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options)
+      {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options)
+      {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options)
 
       {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)}
     else
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
-      {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options)
+      {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
 
       {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
     end
@@ -115,9 +111,9 @@ defmodule Pleroma.Formatter do
 
     if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
       %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
-      Linkify.link(mentions, options) <> Linkify.link(rest, options)
+      AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options)
     else
-      Linkify.link(text, options)
+      AutoLinker.link(text, options)
     end
   end
 
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 747f2dc6b..1729141e9 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do
 
   @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
-    validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
+    validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
 
     page_url
-    |> Linkify.Parser.url?(validate_tld: validate_tld)
+    |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld)
     |> parse_uri(page_url)
   end
 
-- 
cgit v1.2.3


From 5b1eeb06d81872696fac89dba457fe62b62d6182 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Tue, 21 Jul 2020 22:18:17 +0000
Subject: Revert "Merge branch 'revert-2b5d9eb1' into 'develop'"

This reverts merge request !2784
---
 lib/pleroma/config/config_db.ex       |  1 -
 lib/pleroma/formatter.ex              | 26 +++++++++++++++-----------
 lib/pleroma/web/rich_media/helpers.ex |  4 ++--
 3 files changed, 17 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex
index 1a89d8895..e5b7811aa 100644
--- a/lib/pleroma/config/config_db.ex
+++ b/lib/pleroma/config/config_db.ex
@@ -156,7 +156,6 @@ defmodule Pleroma.ConfigDB do
       {:quack, :meta},
       {:mime, :types},
       {:cors_plug, [:max_age, :methods, :expose, :headers]},
-      {:auto_linker, :opts},
       {:swarm, :node_blacklist},
       {:logger, :backends}
     ]
diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex
index 02a93a8dc..0c450eae4 100644
--- a/lib/pleroma/formatter.ex
+++ b/lib/pleroma/formatter.ex
@@ -10,11 +10,15 @@ defmodule Pleroma.Formatter do
   @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui
   @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/
 
-  @auto_linker_config hashtag: true,
-                      hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
-                      mention: true,
-                      mention_handler: &Pleroma.Formatter.mention_handler/4,
-                      scheme: true
+  defp linkify_opts do
+    Pleroma.Config.get(Pleroma.Formatter) ++
+      [
+        hashtag: true,
+        hashtag_handler: &Pleroma.Formatter.hashtag_handler/4,
+        mention: true,
+        mention_handler: &Pleroma.Formatter.mention_handler/4
+      ]
+  end
 
   def escape_mention_handler("@" <> nickname = mention, buffer, _, _) do
     case User.get_cached_by_nickname(nickname) do
@@ -80,19 +84,19 @@ defmodule Pleroma.Formatter do
   @spec linkify(String.t(), keyword()) ::
           {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]}
   def linkify(text, options \\ []) do
-    options = options ++ @auto_linker_config
+    options = linkify_opts() ++ options
 
     if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
       %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
 
-      {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options)
-      {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options)
+      {text_mentions, %{mentions: mentions}} = Linkify.link_map(mentions, acc, options)
+      {text_rest, %{tags: tags}} = Linkify.link_map(rest, acc, options)
 
       {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)}
     else
       acc = %{mentions: MapSet.new(), tags: MapSet.new()}
-      {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options)
+      {text, %{mentions: mentions, tags: tags}} = Linkify.link_map(text, acc, options)
 
       {text, MapSet.to_list(mentions), MapSet.to_list(tags)}
     end
@@ -111,9 +115,9 @@ defmodule Pleroma.Formatter do
 
     if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do
       %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text)
-      AutoLinker.link(mentions, options) <> AutoLinker.link(rest, options)
+      Linkify.link(mentions, options) <> Linkify.link(rest, options)
     else
-      AutoLinker.link(text, options)
+      Linkify.link(text, options)
     end
   end
 
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index 1729141e9..747f2dc6b 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -11,10 +11,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do
 
   @spec validate_page_url(URI.t() | binary()) :: :ok | :error
   defp validate_page_url(page_url) when is_binary(page_url) do
-    validate_tld = Application.get_env(:auto_linker, :opts)[:validate_tld]
+    validate_tld = Pleroma.Config.get([Pleroma.Formatter, :validate_tld])
 
     page_url
-    |> AutoLinker.Parser.url?(scheme: true, validate_tld: validate_tld)
+    |> Linkify.Parser.url?(validate_tld: validate_tld)
     |> parse_uri(page_url)
   end
 
-- 
cgit v1.2.3


From 341a8f35002e2ec8b6a91453b40acf0f04ba7631 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Tue, 21 Jul 2020 17:26:59 -0500
Subject: Skip the correct plug

---
 lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
index 866901344..657f46324 100644
--- a/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/emoji_pack_controller.ex
@@ -21,7 +21,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiPackController do
          ]
   )
 
-  @skip_plugs [Pleroma.Plugs.OAuthScopesPlug, Pleroma.Plugs.ExpectPublicOrAuthenticatedCheckPlug]
+  @skip_plugs [Pleroma.Plugs.OAuthScopesPlug, Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug]
   plug(:skip_plug, @skip_plugs when action in [:index, :show, :archive])
 
   defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaEmojiPackOperation
-- 
cgit v1.2.3


From 0cb9e1da746ee5bfb8147cead3944f0e13fb447f Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@soykaf.club>
Date: Wed, 22 Jul 2020 14:44:06 +0200
Subject: StatusView: Handle badly formatted emoji reactions.

---
 lib/pleroma/web/mastodon_api/views/status_view.ex | 24 +++++++++++++++++------
 1 file changed, 18 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index fa9d695f3..91b41ef59 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -297,13 +297,17 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
     emoji_reactions =
       with %{data: %{"reactions" => emoji_reactions}} <- object do
-        Enum.map(emoji_reactions, fn [emoji, users] ->
-          %{
-            name: emoji,
-            count: length(users),
-            me: !!(opts[:for] && opts[:for].ap_id in users)
-          }
+        Enum.map(emoji_reactions, fn
+          [emoji, users] when is_list(users) ->
+            build_emoji_map(emoji, users, opts[:for])
+
+          {emoji, users} when is_list(users) ->
+            build_emoji_map(emoji, users, opts[:for])
+
+          _ ->
+            nil
         end)
+        |> Enum.reject(&is_nil/1)
       else
         _ -> []
       end
@@ -545,4 +549,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
   defp pinned?(%Activity{id: id}, %User{pinned_activities: pinned_activities}),
     do: id in pinned_activities
+
+  defp build_emoji_map(emoji, users, current_user) do
+    %{
+      name: emoji,
+      count: length(users),
+      me: !!(current_user && current_user.ap_id in users)
+    }
+  end
 end
-- 
cgit v1.2.3


From 6f5f7af607518b6f67df68bab9bf76142e9a622c Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Wed, 22 Jul 2020 19:06:00 +0300
Subject: [#1973] Fixed accounts rendering in GET /api/v1/pleroma/chats with
 truish :restrict_unauthenticated. Made
 `Pleroma.Web.MastodonAPI.AccountView.render("show.json", _)` demand :for or
 :force option in order to prevent incorrect rendering of empty map instead of
 expected user representation with truish :restrict_unauthenticated setting.

---
 lib/pleroma/web/activity_pub/utils.ex              |  9 ++++++---
 .../admin_api/controllers/admin_api_controller.ex  |  6 +++++-
 lib/pleroma/web/admin_api/views/account_view.ex    |  2 +-
 lib/pleroma/web/chat_channel.ex                    |  6 ++++--
 .../mastodon_api/controllers/search_controller.ex  |  1 -
 lib/pleroma/web/mastodon_api/views/account_view.ex | 23 +++++++++++++++++++---
 .../web/mastodon_api/views/conversation_view.ex    |  2 +-
 .../web/pleroma_api/controllers/chat_controller.ex | 15 ++++++++------
 lib/pleroma/web/pleroma_api/views/chat_view.ex     | 17 +++++++++++++---
 .../web/pleroma_api/views/emoji_reaction_view.ex   |  2 +-
 10 files changed, 61 insertions(+), 22 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index dfae602df..11c64cffd 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -719,15 +719,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do
 
     case Activity.get_by_ap_id_with_object(id) do
       %Activity{} = activity ->
+        activity_actor = User.get_by_ap_id(activity.object.data["actor"])
+
         %{
           "type" => "Note",
           "id" => activity.data["id"],
           "content" => activity.object.data["content"],
           "published" => activity.object.data["published"],
           "actor" =>
-            AccountView.render("show.json", %{
-              user: User.get_by_ap_id(activity.object.data["actor"])
-            })
+            AccountView.render(
+              "show.json",
+              %{user: activity_actor, force: true}
+            )
         }
 
       _ ->
diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
index e5f14269a..225ceb1fd 100644
--- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex
@@ -345,7 +345,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     with {:ok, users, count} <- Search.user(Map.merge(search_params, filters)) do
       json(
         conn,
-        AccountView.render("index.json", users: users, count: count, page_size: page_size)
+        AccountView.render("index.json",
+          users: users,
+          count: count,
+          page_size: page_size
+        )
       )
     end
   end
diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index e1e929632..4ae030b84 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -105,7 +105,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
   end
 
   def merge_account_views(%User{} = user) do
-    MastodonAPI.AccountView.render("show.json", %{user: user})
+    MastodonAPI.AccountView.render("show.json", %{user: user, force: true})
     |> Map.merge(AdminAPI.AccountView.render("show.json", %{user: user}))
   end
 
diff --git a/lib/pleroma/web/chat_channel.ex b/lib/pleroma/web/chat_channel.ex
index bce27897f..08d0e80f9 100644
--- a/lib/pleroma/web/chat_channel.ex
+++ b/lib/pleroma/web/chat_channel.ex
@@ -4,8 +4,10 @@
 
 defmodule Pleroma.Web.ChatChannel do
   use Phoenix.Channel
+
   alias Pleroma.User
   alias Pleroma.Web.ChatChannel.ChatChannelState
+  alias Pleroma.Web.MastodonAPI.AccountView
 
   def join("chat:public", _message, socket) do
     send(self(), :after_join)
@@ -22,9 +24,9 @@ defmodule Pleroma.Web.ChatChannel do
 
     if String.length(text) in 1..Pleroma.Config.get([:instance, :chat_limit]) do
       author = User.get_cached_by_nickname(user_name)
-      author = Pleroma.Web.MastodonAPI.AccountView.render("show.json", user: author)
+      author_json = AccountView.render("show.json", user: author, force: true)
 
-      message = ChatChannelState.add_message(%{text: text, author: author})
+      message = ChatChannelState.add_message(%{text: text, author: author_json})
 
       broadcast!(socket, "new_msg", message)
     end
diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
index 29affa7d5..5a983db39 100644
--- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex
@@ -93,7 +93,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do
     AccountView.render("index.json",
       users: accounts,
       for: options[:for_user],
-      as: :user,
       embed_relationships: options[:embed_relationships]
     )
   end
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index bc9745044..b929d5a03 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -27,21 +27,38 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
           UserRelationship.view_relationships_option(reading_user, users)
       end
 
-    opts = Map.put(opts, :relationships, relationships_opt)
+    opts =
+      opts
+      |> Map.merge(%{relationships: relationships_opt, as: :user})
+      |> Map.delete(:users)
 
     users
     |> render_many(AccountView, "show.json", opts)
     |> Enum.filter(&Enum.any?/1)
   end
 
-  def render("show.json", %{user: user} = opts) do
-    if User.visible_for(user, opts[:for]) == :visible do
+  @doc """
+  Renders specified user account.
+    :force option skips visibility check and renders any user (local or remote)
+      regardless of [:pleroma, :restrict_unauthenticated] setting.
+    :for option specifies the requester and can be a User record or nil.
+  """
+  def render("show.json", %{user: _user, force: true} = opts) do
+    do_render("show.json", opts)
+  end
+
+  def render("show.json", %{user: user, for: for_user_or_nil} = opts) do
+    if User.visible_for(user, for_user_or_nil) == :visible do
       do_render("show.json", opts)
     else
       %{}
     end
   end
 
+  def render("show.json", _) do
+    raise "In order to prevent account accessibility issues, :force or :for option is required."
+  end
+
   def render("mention.json", %{user: user}) do
     %{
       id: to_string(user.id),
diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
index 06f0c1728..a91994915 100644
--- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex
@@ -38,7 +38,7 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do
 
     %{
       id: participation.id |> to_string(),
-      accounts: render(AccountView, "index.json", users: users, as: :user),
+      accounts: render(AccountView, "index.json", users: users, for: user),
       unread: !participation.read,
       last_status:
         render(StatusView, "show.json",
diff --git a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
index c8ef3d915..e8a1746d4 100644
--- a/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/chat_controller.ex
@@ -89,11 +89,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
          cm_ref <- MessageReference.for_chat_and_object(chat, message) do
       conn
       |> put_view(MessageReferenceView)
-      |> render("show.json", for: user, chat_message_reference: cm_ref)
+      |> render("show.json", chat_message_reference: cm_ref)
     end
   end
 
-  def mark_message_as_read(%{assigns: %{user: %{id: user_id} = user}} = conn, %{
+  def mark_message_as_read(%{assigns: %{user: %{id: user_id}}} = conn, %{
         id: chat_id,
         message_id: message_id
       }) do
@@ -104,12 +104,15 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
          {:ok, cm_ref} <- MessageReference.mark_as_read(cm_ref) do
       conn
       |> put_view(MessageReferenceView)
-      |> render("show.json", for: user, chat_message_reference: cm_ref)
+      |> render("show.json", chat_message_reference: cm_ref)
     end
   end
 
   def mark_as_read(
-        %{body_params: %{last_read_id: last_read_id}, assigns: %{user: %{id: user_id}}} = conn,
+        %{
+          body_params: %{last_read_id: last_read_id},
+          assigns: %{user: %{id: user_id}}
+        } = conn,
         %{id: id}
       ) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id),
@@ -121,7 +124,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
     end
   end
 
-  def messages(%{assigns: %{user: %{id: user_id} = user}} = conn, %{id: id} = params) do
+  def messages(%{assigns: %{user: %{id: user_id}}} = conn, %{id: id} = params) do
     with %Chat{} = chat <- Repo.get_by(Chat, id: id, user_id: user_id) do
       cm_refs =
         chat
@@ -130,7 +133,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatController do
 
       conn
       |> put_view(MessageReferenceView)
-      |> render("index.json", for: user, chat_message_references: cm_refs)
+      |> render("index.json", chat_message_references: cm_refs)
     else
       _ ->
         conn
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 1c996da11..2ae7c8122 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -15,10 +15,11 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
   def render("show.json", %{chat: %Chat{} = chat} = opts) do
     recipient = User.get_cached_by_ap_id(chat.recipient)
     last_message = opts[:last_message] || MessageReference.last_message_for_chat(chat)
+    account_view_opts = account_view_opts(opts, recipient)
 
     %{
       id: chat.id |> to_string(),
-      account: AccountView.render("show.json", Map.put(opts, :user, recipient)),
+      account: AccountView.render("show.json", account_view_opts),
       unread: MessageReference.unread_count_for_chat(chat),
       last_message:
         last_message &&
@@ -27,7 +28,17 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
     }
   end
 
-  def render("index.json", %{chats: chats}) do
-    render_many(chats, __MODULE__, "show.json")
+  def render("index.json", %{chats: chats} = opts) do
+    render_many(chats, __MODULE__, "show.json", Map.delete(opts, :chats))
+  end
+
+  defp account_view_opts(opts, recipient) do
+    account_view_opts = Map.put(opts, :user, recipient)
+
+    if Map.has_key?(account_view_opts, :for) do
+      account_view_opts
+    else
+      Map.put(account_view_opts, :force, true)
+    end
   end
 end
diff --git a/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex b/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex
index 84d2d303d..e0f98b50a 100644
--- a/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/emoji_reaction_view.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Web.PleromaAPI.EmojiReactionView do
     %{
       name: emoji,
       count: length(users),
-      accounts: render(AccountView, "index.json", users: users, for: user, as: :user),
+      accounts: render(AccountView, "index.json", users: users, for: user),
       me: !!(user && user.ap_id in user_ap_ids)
     }
   end
-- 
cgit v1.2.3


From 9ea51a6de516b37341a9566d11d0110c2d87c1b6 Mon Sep 17 00:00:00 2001
From: Ivan Tashkinov <ivantashkinov@gmail.com>
Date: Thu, 23 Jul 2020 15:08:30 +0300
Subject: [#2791] AccountView: renamed `:force` option to
 `:skip_visibility_check`.

---
 lib/pleroma/web/activity_pub/utils.ex              | 2 +-
 lib/pleroma/web/admin_api/views/account_view.ex    | 2 +-
 lib/pleroma/web/chat_channel.ex                    | 2 +-
 lib/pleroma/web/mastodon_api/views/account_view.ex | 8 +++++---
 lib/pleroma/web/pleroma_api/views/chat_view.ex     | 2 +-
 5 files changed, 9 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index 11c64cffd..713b0ca1f 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -729,7 +729,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do
           "actor" =>
             AccountView.render(
               "show.json",
-              %{user: activity_actor, force: true}
+              %{user: activity_actor, skip_visibility_check: true}
             )
         }
 
diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index 4ae030b84..88fbb5315 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -105,7 +105,7 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
   end
 
   def merge_account_views(%User{} = user) do
-    MastodonAPI.AccountView.render("show.json", %{user: user, force: true})
+    MastodonAPI.AccountView.render("show.json", %{user: user, skip_visibility_check: true})
     |> Map.merge(AdminAPI.AccountView.render("show.json", %{user: user}))
   end
 
diff --git a/lib/pleroma/web/chat_channel.ex b/lib/pleroma/web/chat_channel.ex
index 08d0e80f9..3b1469c19 100644
--- a/lib/pleroma/web/chat_channel.ex
+++ b/lib/pleroma/web/chat_channel.ex
@@ -24,7 +24,7 @@ defmodule Pleroma.Web.ChatChannel do
 
     if String.length(text) in 1..Pleroma.Config.get([:instance, :chat_limit]) do
       author = User.get_cached_by_nickname(user_name)
-      author_json = AccountView.render("show.json", user: author, force: true)
+      author_json = AccountView.render("show.json", user: author, skip_visibility_check: true)
 
       message = ChatChannelState.add_message(%{text: text, author: author_json})
 
diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex
index b929d5a03..864c0417f 100644
--- a/lib/pleroma/web/mastodon_api/views/account_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/account_view.ex
@@ -39,11 +39,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
 
   @doc """
   Renders specified user account.
-    :force option skips visibility check and renders any user (local or remote)
+    :skip_visibility_check option skips visibility check and renders any user (local or remote)
       regardless of [:pleroma, :restrict_unauthenticated] setting.
     :for option specifies the requester and can be a User record or nil.
+      Only use `user: user, for: user` when `user` is the actual requester of own profile.
   """
-  def render("show.json", %{user: _user, force: true} = opts) do
+  def render("show.json", %{user: _user, skip_visibility_check: true} = opts) do
     do_render("show.json", opts)
   end
 
@@ -56,7 +57,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do
   end
 
   def render("show.json", _) do
-    raise "In order to prevent account accessibility issues, :force or :for option is required."
+    raise "In order to prevent account accessibility issues, " <>
+            ":skip_visibility_check or :for option is required."
   end
 
   def render("mention.json", %{user: user}) do
diff --git a/lib/pleroma/web/pleroma_api/views/chat_view.ex b/lib/pleroma/web/pleroma_api/views/chat_view.ex
index 2ae7c8122..04dc20d51 100644
--- a/lib/pleroma/web/pleroma_api/views/chat_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/chat_view.ex
@@ -38,7 +38,7 @@ defmodule Pleroma.Web.PleromaAPI.ChatView do
     if Map.has_key?(account_view_opts, :for) do
       account_view_opts
     else
-      Map.put(account_view_opts, :force, true)
+      Map.put(account_view_opts, :skip_visibility_check, true)
     end
   end
 end
-- 
cgit v1.2.3


From 4bfad0b483957acf755a043f33799742997da859 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Thu, 23 Jul 2020 12:59:40 -0500
Subject: Support blocking via query parameters as well and document the
 change.

---
 lib/pleroma/web/api_spec/operations/domain_block_operation.ex       | 3 +++
 lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex | 5 +++++
 2 files changed, 8 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex
index 8234394f9..1e0da8209 100644
--- a/lib/pleroma/web/api_spec/operations/domain_block_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/domain_block_operation.ex
@@ -31,6 +31,7 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do
     }
   end
 
+  # Supporting domain query parameter is deprecated in Mastodon API
   def create_operation do
     %Operation{
       tags: ["domain_blocks"],
@@ -45,11 +46,13 @@ defmodule Pleroma.Web.ApiSpec.DomainBlockOperation do
       """,
       operationId: "DomainBlockController.create",
       requestBody: domain_block_request(),
+      parameters: [Operation.parameter(:domain, :query, %Schema{type: :string}, "Domain name")],
       security: [%{"oAuth" => ["follow", "write:blocks"]}],
       responses: %{200 => empty_object_response()}
     }
   end
 
+  # Supporting domain query parameter is deprecated in Mastodon API
   def delete_operation do
     %Operation{
       tags: ["domain_blocks"],
diff --git a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex
index 117e89426..9c2d093cd 100644
--- a/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/domain_block_controller.ex
@@ -32,6 +32,11 @@ defmodule Pleroma.Web.MastodonAPI.DomainBlockController do
     json(conn, %{})
   end
 
+  def create(%{assigns: %{user: blocker}} = conn, %{domain: domain}) do
+    User.block_domain(blocker, domain)
+    json(conn, %{})
+  end
+
   @doc "DELETE /api/v1/domain_blocks"
   def delete(%{assigns: %{user: blocker}, body_params: %{domain: domain}} = conn, _params) do
     User.unblock_domain(blocker, domain)
-- 
cgit v1.2.3


From 9be66682369f1aa3c221d411073c20e10b5a3ac1 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 24 Jul 2020 12:05:42 -0500
Subject: Fix mix tasks that make HTTP calls by starting the Gun connection
 pool

---
 lib/mix/pleroma.ex | 23 +++++++++++++++++------
 1 file changed, 17 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index 9f0bf6ecb..c2b607fb3 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -24,8 +24,10 @@ defmodule Mix.Pleroma do
       Application.put_env(:logger, :console, level: :debug)
     end
 
+    adapter = Application.get_env(:tesla, :adapter)
+
     apps =
-      if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do
+      if adapter == Tesla.Adapter.Gun do
         [:gun | @apps]
       else
         [:hackney | @apps]
@@ -33,11 +35,13 @@ defmodule Mix.Pleroma do
 
     Enum.each(apps, &Application.ensure_all_started/1)
 
-    children = [
-      Pleroma.Repo,
-      {Pleroma.Config.TransferTask, false},
-      Pleroma.Web.Endpoint
-    ]
+    children =
+      [
+        Pleroma.Repo,
+        {Pleroma.Config.TransferTask, false},
+        Pleroma.Web.Endpoint
+      ] ++
+        http_children(adapter)
 
     cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, []))
 
@@ -115,4 +119,11 @@ defmodule Mix.Pleroma do
   def escape_sh_path(path) do
     ~S(') <> String.replace(path, ~S('), ~S(\')) <> ~S(')
   end
+
+  defp http_children(Tesla.Adapter.Gun) do
+    Pleroma.Gun.ConnectionPool.children() ++
+      [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}]
+  end
+
+  defp http_children(_), do: []
 end
-- 
cgit v1.2.3


From 65a1b048a8effa23eb99b1aeae3b97a7e7df3ef5 Mon Sep 17 00:00:00 2001
From: Mark Felder <feld@FreeBSD.org>
Date: Fri, 24 Jul 2020 12:06:56 -0500
Subject: Ensure Oban is available during mix tasks. Fixes: mix pleroma.user rm
 username

---
 lib/mix/pleroma.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex
index c2b607fb3..074492a46 100644
--- a/lib/mix/pleroma.ex
+++ b/lib/mix/pleroma.ex
@@ -39,7 +39,8 @@ defmodule Mix.Pleroma do
       [
         Pleroma.Repo,
         {Pleroma.Config.TransferTask, false},
-        Pleroma.Web.Endpoint
+        Pleroma.Web.Endpoint,
+        {Oban, Pleroma.Config.get(Oban)}
       ] ++
         http_children(adapter)
 
-- 
cgit v1.2.3


From b31844d6e01fc8bea4ecbe93b072846ca4309e88 Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Sun, 26 Jul 2020 13:54:56 +0000
Subject: OpenAPI: Replace actor_id by account_id to follow ChatMessage schema

---
 lib/pleroma/web/api_spec/operations/chat_operation.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index cf299bfc2..1a5b05899 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -300,11 +300,11 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
           "content" => "Check this out :firefox:",
           "id" => "13",
           "chat_id" => "1",
-          "actor_id" => "someflakeid",
+          "account_id" => "someflakeid",
           "unread" => false
         },
         %{
-          "actor_id" => "someflakeid",
+          "account_id" => "someflakeid",
           "content" => "Whats' up?",
           "id" => "12",
           "chat_id" => "1",
@@ -337,7 +337,7 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
 
   def mark_as_read do
     %Schema{
-      title: "MarkAsReadRequest",
+      title: "MarkAsReadRequest",Update chat_operation.ex
       description: "POST body for marking a number of chat messages as read",
       type: :object,
       required: [:last_read_id],
-- 
cgit v1.2.3


From 6107440ea0da3a9e59576a86a9dab50acd83936e Mon Sep 17 00:00:00 2001
From: Alibek Omarov <a1ba.omarov@gmail.com>
Date: Sun, 26 Jul 2020 13:59:46 +0000
Subject: OpenAPI: remove accidentally pasted buffer data

---
 lib/pleroma/web/api_spec/operations/chat_operation.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/api_spec/operations/chat_operation.ex b/lib/pleroma/web/api_spec/operations/chat_operation.ex
index 1a5b05899..b1a0d26ab 100644
--- a/lib/pleroma/web/api_spec/operations/chat_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/chat_operation.ex
@@ -337,7 +337,7 @@ defmodule Pleroma.Web.ApiSpec.ChatOperation do
 
   def mark_as_read do
     %Schema{
-      title: "MarkAsReadRequest",Update chat_operation.ex
+      title: "MarkAsReadRequest",
       description: "POST body for marking a number of chat messages as read",
       type: :object,
       required: [:last_read_id],
-- 
cgit v1.2.3