From 3189c44a0cf6821976819112e93e93ad811cba53 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 24 Mar 2020 15:21:40 +0400 Subject: Remove some TwitterAPI endpoints --- lib/pleroma/web/router.ex | 4 -- .../web/twitter_api/controllers/util_controller.ex | 83 ---------------------- 2 files changed, 87 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 3f36f6c1a..c3ea7b626 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -481,10 +481,6 @@ defmodule Pleroma.Web.Router do scope "/api", Pleroma.Web do pipe_through(:config) - get("/help/test", TwitterAPI.UtilController, :help_test) - post("/help/test", TwitterAPI.UtilController, :help_test) - get("/statusnet/config", TwitterAPI.UtilController, :config) - get("/statusnet/version", TwitterAPI.UtilController, :version) get("/pleroma/frontend_configurations", TwitterAPI.UtilController, :frontend_configurations) end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 537f9f778..bb08f5426 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -13,7 +13,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Notification alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User - alias Pleroma.Web alias Pleroma.Web.CommonAPI alias Pleroma.Web.WebFinger @@ -48,12 +47,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do plug(OAuthScopesPlug, %{scopes: ["write:notifications"]} when action == :notifications_read) - plug(Pleroma.Plugs.SetFormatPlug when action in [:config, :version]) - - def help_test(conn, _params) do - json(conn, "ok") - end - def remote_subscribe(conn, %{"nickname" => nick, "profile" => _}) do with %User{} = user <- User.get_cached_by_nickname(nick), avatar = User.avatar_url(user) do @@ -95,70 +88,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - def config(%{assigns: %{format: "xml"}} = conn, _params) do - instance = Pleroma.Config.get(:instance) - - response = """ - - - #{Keyword.get(instance, :name)} - #{Web.base_url()} - #{Keyword.get(instance, :limit)} - #{!Keyword.get(instance, :registrations_open)} - - - """ - - conn - |> put_resp_content_type("application/xml") - |> send_resp(200, response) - end - - def config(conn, _params) do - instance = Pleroma.Config.get(:instance) - - vapid_public_key = Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) - - uploadlimit = %{ - uploadlimit: to_string(Keyword.get(instance, :upload_limit)), - avatarlimit: to_string(Keyword.get(instance, :avatar_upload_limit)), - backgroundlimit: to_string(Keyword.get(instance, :background_upload_limit)), - bannerlimit: to_string(Keyword.get(instance, :banner_upload_limit)) - } - - data = %{ - name: Keyword.get(instance, :name), - description: Keyword.get(instance, :description), - server: Web.base_url(), - textlimit: to_string(Keyword.get(instance, :limit)), - uploadlimit: uploadlimit, - closed: bool_to_val(Keyword.get(instance, :registrations_open), "0", "1"), - private: bool_to_val(Keyword.get(instance, :public, true), "0", "1"), - vapidPublicKey: vapid_public_key, - accountActivationRequired: - bool_to_val(Keyword.get(instance, :account_activation_required, false)), - invitesEnabled: bool_to_val(Keyword.get(instance, :invites_enabled, false)), - safeDMMentionsEnabled: bool_to_val(Pleroma.Config.get([:instance, :safe_dm_mentions])) - } - - managed_config = Keyword.get(instance, :managed_config) - - data = - if managed_config do - pleroma_fe = Pleroma.Config.get([:frontend_configurations, :pleroma_fe]) - Map.put(data, "pleromafe", pleroma_fe) - else - data - end - - json(conn, %{site: data}) - end - - defp bool_to_val(true), do: "1" - defp bool_to_val(_), do: "0" - defp bool_to_val(true, val, _), do: val - defp bool_to_val(_, _, val), do: val - def frontend_configurations(conn, _params) do config = Pleroma.Config.get(:frontend_configurations, %{}) @@ -167,18 +96,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do json(conn, config) end - def version(%{assigns: %{format: "xml"}} = conn, _params) do - version = Pleroma.Application.named_version() - - conn - |> put_resp_content_type("application/xml") - |> send_resp(200, "#{version}") - end - - def version(conn, _params) do - json(conn, Pleroma.Application.named_version()) - end - def emoji(conn, _params) do emoji = Enum.reduce(Emoji.get_all(), %{}, fn {code, %Emoji{file: file, tags: tags}}, acc -> -- cgit v1.2.3 From 1c3f3a12edcdd4f11433e9ed5422b381afd3c5c4 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 26 Mar 2020 16:20:20 +0400 Subject: Add `characterLimit` and `vapidPublicKey` to nodeinfo --- lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 30838b1eb..6947c82b9 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -106,6 +106,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do }, staffAccounts: staff_accounts, federation: federation_response, + characterLimit: Config.get([:instance, :limit]), pollLimits: Config.get([:instance, :poll_limits]), postFormats: Config.get([:instance, :allowed_post_formats]), uploadLimits: %{ @@ -125,7 +126,8 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do mailerEnabled: Config.get([Pleroma.Emails.Mailer, :enabled], false), features: features, restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), - skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) + skipThreadContainment: Config.get([:instance, :skip_thread_containment], false), + vapidPublicKey: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) } } end -- cgit v1.2.3 From 94a6590e3cb9d5c340bfd589880c19717160706f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 26 Mar 2020 17:59:45 +0400 Subject: Partially restore `/api/statusnet/config.json` --- lib/pleroma/web/router.ex | 3 +++ lib/pleroma/web/twitter_api/controllers/util_controller.ex | 12 ++++++++++++ 2 files changed, 15 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c3ea7b626..322b074c2 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -482,6 +482,9 @@ defmodule Pleroma.Web.Router do pipe_through(:config) get("/pleroma/frontend_configurations", TwitterAPI.UtilController, :frontend_configurations) + + # Deprecated + get("/statusnet/config", TwitterAPI.UtilController, :config) end scope "/api", Pleroma.Web do diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index bb08f5426..2fc60da5a 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -88,6 +88,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end + # Deprecated in favor of `/nodeinfo` + # https://git.pleroma.social/pleroma/pleroma/-/merge_requests/2327 + # https://git.pleroma.social/pleroma/pleroma-fe/-/merge_requests/1084 + def config(conn, _params) do + json(conn, %{ + site: %{ + textlimit: to_string(Config.get([:instance, :limit])), + vapidPublicKey: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } + }) + end + def frontend_configurations(conn, _params) do config = Pleroma.Config.get(:frontend_configurations, %{}) -- cgit v1.2.3 From 9cf4c4fa73e68f03791c5cc70505b710be39b677 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 23 Apr 2020 14:12:42 +0400 Subject: Remove vapidPublicKey from Nodeinfo --- lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 6947c82b9..c90d4c009 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -126,8 +126,7 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do mailerEnabled: Config.get([Pleroma.Emails.Mailer, :enabled], false), features: features, restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), - skipThreadContainment: Config.get([:instance, :skip_thread_containment], false), - vapidPublicKey: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) } } end -- cgit v1.2.3 From 9eea80002673eb1359a2d4369c65a89c42c2f707 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 May 2020 10:16:09 -0500 Subject: Refactor notification settings --- lib/pleroma/notification.ex | 29 +++++++++-------------------- lib/pleroma/user/notification_setting.ex | 14 ++++++-------- lib/pleroma/web/api_spec/schemas/account.ex | 14 ++++++-------- 3 files changed, 21 insertions(+), 36 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 7eca55ac9..ca556f0bb 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -459,10 +459,9 @@ defmodule Pleroma.Notification do def skip?(%Activity{} = activity, %User{} = user) do [ :self, - :followers, - :follows, - :non_followers, - :non_follows, + :from_followers, + :from_following, + :from_strangers, :recently_followed ] |> Enum.find(&skip?(&1, activity, user)) @@ -476,9 +475,9 @@ defmodule Pleroma.Notification do end def skip?( - :followers, + :from_followers, %Activity{} = activity, - %User{notification_settings: %{followers: false}} = user + %User{notification_settings: %{from_followers: false}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) @@ -486,9 +485,9 @@ defmodule Pleroma.Notification do end def skip?( - :non_followers, + :from_strangers, %Activity{} = activity, - %User{notification_settings: %{non_followers: false}} = user + %User{notification_settings: %{from_strangers: false}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) @@ -496,25 +495,15 @@ defmodule Pleroma.Notification do end def skip?( - :follows, + :from_following, %Activity{} = activity, - %User{notification_settings: %{follows: false}} = user + %User{notification_settings: %{from_following: false}} = user ) do actor = activity.data["actor"] followed = User.get_cached_by_ap_id(actor) User.following?(user, followed) end - def skip?( - :non_follows, - %Activity{} = activity, - %User{notification_settings: %{non_follows: false}} = user - ) do - actor = activity.data["actor"] - followed = User.get_cached_by_ap_id(actor) - !User.following?(user, followed) - end - # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do actor = activity.data["actor"] diff --git a/lib/pleroma/user/notification_setting.ex b/lib/pleroma/user/notification_setting.ex index 4bd55e139..e47ac4cab 100644 --- a/lib/pleroma/user/notification_setting.ex +++ b/lib/pleroma/user/notification_setting.ex @@ -10,20 +10,18 @@ defmodule Pleroma.User.NotificationSetting do @primary_key false embedded_schema do - field(:followers, :boolean, default: true) - field(:follows, :boolean, default: true) - field(:non_follows, :boolean, default: true) - field(:non_followers, :boolean, default: true) + field(:from_followers, :boolean, default: true) + field(:from_following, :boolean, default: true) + field(:from_strangers, :boolean, default: true) field(:privacy_option, :boolean, default: false) end def changeset(schema, params) do schema |> cast(prepare_attrs(params), [ - :followers, - :follows, - :non_follows, - :non_followers, + :from_followers, + :from_following, + :from_strangers, :privacy_option ]) end diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index d54e2158d..ed90ef3db 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -57,10 +57,9 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do notification_settings: %Schema{ type: :object, properties: %{ - followers: %Schema{type: :boolean}, - follows: %Schema{type: :boolean}, - non_followers: %Schema{type: :boolean}, - non_follows: %Schema{type: :boolean}, + from_followers: %Schema{type: :boolean}, + from_following: %Schema{type: :boolean}, + from_strangers: %Schema{type: :boolean}, privacy_option: %Schema{type: :boolean} } }, @@ -123,10 +122,9 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do "unread_conversation_count" => 0, "tags" => [], "notification_settings" => %{ - "followers" => true, - "follows" => true, - "non_followers" => true, - "non_follows" => true, + "from_followers" => true, + "from_following" => true, + "from_strangers" => true, "privacy_option" => false }, "relationship" => %{ -- cgit v1.2.3 From 954acdda2072cac343409b3d17d831b86ac6a18c Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 29 May 2020 15:16:44 -0500 Subject: Add `account_activation_required` to /api/v1/instance --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 6a630eafa..bb7bd2a9f 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -36,6 +36,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_image: Keyword.get(instance, :background_image), pleroma: %{ metadata: %{ + account_activation_required: Keyword.get(instance, :account_activation_required), features: features(), federation: federation() }, -- cgit v1.2.3 From 29ae5bb77166d9d7f8108a965b0c3d147b747e80 Mon Sep 17 00:00:00 2001 From: stwf Date: Tue, 12 May 2020 11:08:00 -0400 Subject: preload data into index.html --- lib/pleroma/web/fallback_redirect_controller.ex | 82 ++++++++++----- lib/pleroma/web/nodeinfo/nodeinfo.ex | 130 ++++++++++++++++++++++++ lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 114 +++------------------ lib/pleroma/web/preload.ex | 30 ++++++ lib/pleroma/web/preload/instance.ex | 49 +++++++++ lib/pleroma/web/preload/provider.ex | 7 ++ lib/pleroma/web/preload/timelines.ex | 42 ++++++++ lib/pleroma/web/preload/user.ex | 25 +++++ lib/pleroma/web/router.ex | 2 +- 9 files changed, 353 insertions(+), 128 deletions(-) create mode 100644 lib/pleroma/web/nodeinfo/nodeinfo.ex create mode 100644 lib/pleroma/web/preload.ex create mode 100644 lib/pleroma/web/preload/instance.ex create mode 100644 lib/pleroma/web/preload/provider.ex create mode 100644 lib/pleroma/web/preload/timelines.ex create mode 100644 lib/pleroma/web/preload/user.ex (limited to 'lib') diff --git a/lib/pleroma/web/fallback_redirect_controller.ex b/lib/pleroma/web/fallback_redirect_controller.ex index 0d9d578fc..932fb8d7e 100644 --- a/lib/pleroma/web/fallback_redirect_controller.ex +++ b/lib/pleroma/web/fallback_redirect_controller.ex @@ -4,11 +4,10 @@ defmodule Fallback.RedirectController do use Pleroma.Web, :controller - require Logger - alias Pleroma.User alias Pleroma.Web.Metadata + alias Pleroma.Web.Preload def api_not_implemented(conn, _params) do conn @@ -16,16 +15,7 @@ defmodule Fallback.RedirectController do |> json(%{error: "Not implemented"}) end - def redirector(conn, _params, code \\ 200) - - # redirect to admin section - # /pleroma/admin -> /pleroma/admin/ - # - def redirector(conn, %{"path" => ["pleroma", "admin"]} = _, _code) do - redirect(conn, to: "/pleroma/admin/") - end - - def redirector(conn, _params, code) do + def redirector(conn, _params, code \\ 200) do conn |> put_resp_content_type("text/html") |> send_file(code, index_file_path()) @@ -43,28 +33,34 @@ defmodule Fallback.RedirectController do def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = - try do - Metadata.build_tags(params) - rescue - e -> - Logger.error( - "Metadata rendering for #{conn.request_path} failed.\n" <> - Exception.format(:error, e, __STACKTRACE__) - ) - - "" - end + tags = build_tags(conn, params) + preloads = preload_data(conn, params) - response = String.replace(index_content, "", tags) + response = + index_content + |> String.replace("", tags) + |> String.replace("", preloads) conn |> put_resp_content_type("text/html") |> send_resp(200, response) end - def index_file_path do - Pleroma.Plugs.InstanceStatic.file_path("index.html") + def redirector_with_preload(conn, %{"path" => ["pleroma", "admin"]}) do + redirect(conn, to: "/pleroma/admin/") + end + + def redirector_with_preload(conn, params) do + {:ok, index_content} = File.read(index_file_path()) + preloads = preload_data(conn, params) + + response = + index_content + |> String.replace("", preloads) + + conn + |> put_resp_content_type("text/html") + |> send_resp(200, response) end def registration_page(conn, params) do @@ -76,4 +72,36 @@ defmodule Fallback.RedirectController do |> put_status(204) |> text("") end + + defp index_file_path do + Pleroma.Plugs.InstanceStatic.file_path("index.html") + end + + defp build_tags(conn, params) do + try do + Metadata.build_tags(params) + rescue + e -> + Logger.error( + "Metadata rendering for #{conn.request_path} failed.\n" <> + Exception.format(:error, e, __STACKTRACE__) + ) + + "" + end + end + + defp preload_data(conn, params) do + try do + Preload.build_tags(conn, params) + rescue + e -> + Logger.error( + "Preloading for #{conn.request_path} failed.\n" <> + Exception.format(:error, e, __STACKTRACE__) + ) + + "" + end + end end diff --git a/lib/pleroma/web/nodeinfo/nodeinfo.ex b/lib/pleroma/web/nodeinfo/nodeinfo.ex new file mode 100644 index 000000000..d26b7c938 --- /dev/null +++ b/lib/pleroma/web/nodeinfo/nodeinfo.ex @@ -0,0 +1,130 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Nodeinfo.Nodeinfo do + alias Pleroma.Config + alias Pleroma.Stats + alias Pleroma.User + alias Pleroma.Web.ActivityPub.MRF + alias Pleroma.Web.Federator.Publisher + + # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field + # under software. + def get_nodeinfo("2.0") do + stats = Stats.get_stats() + + quarantined = Config.get([:instance, :quarantined_instances], []) + + staff_accounts = + User.all_superusers() + |> Enum.map(fn u -> u.ap_id end) + + federation_response = + if Config.get([:instance, :mrf_transparency]) do + {:ok, data} = MRF.describe() + + data + |> Map.merge(%{quarantined_instances: quarantined}) + else + %{} + end + |> Map.put(:enabled, Config.get([:instance, :federating])) + + features = + [ + "pleroma_api", + "mastodon_api", + "mastodon_api_streaming", + "polls", + "pleroma_explicit_addressing", + "shareable_emoji_packs", + "multifetch", + "pleroma:api/v1/notifications:include_types_filter", + if Config.get([:media_proxy, :enabled]) do + "media_proxy" + end, + if Config.get([:gopher, :enabled]) do + "gopher" + end, + if Config.get([:chat, :enabled]) do + "chat" + end, + if Config.get([:instance, :allow_relay]) do + "relay" + end, + if Config.get([:instance, :safe_dm_mentions]) do + "safe_dm_mentions" + end, + "pleroma_emoji_reactions" + ] + |> Enum.filter(& &1) + + %{ + version: "2.0", + software: %{ + name: Pleroma.Application.name() |> String.downcase(), + version: Pleroma.Application.version() + }, + protocols: Publisher.gather_nodeinfo_protocol_names(), + services: %{ + inbound: [], + outbound: [] + }, + openRegistrations: Config.get([:instance, :registrations_open]), + usage: %{ + users: %{ + total: Map.get(stats, :user_count, 0) + }, + localPosts: Map.get(stats, :status_count, 0) + }, + metadata: %{ + nodeName: Config.get([:instance, :name]), + nodeDescription: Config.get([:instance, :description]), + private: !Config.get([:instance, :public], true), + suggestions: %{ + enabled: false + }, + staffAccounts: staff_accounts, + federation: federation_response, + pollLimits: Config.get([:instance, :poll_limits]), + postFormats: Config.get([:instance, :allowed_post_formats]), + uploadLimits: %{ + general: Config.get([:instance, :upload_limit]), + avatar: Config.get([:instance, :avatar_upload_limit]), + banner: Config.get([:instance, :banner_upload_limit]), + background: Config.get([:instance, :background_upload_limit]) + }, + fieldsLimits: %{ + maxFields: Config.get([:instance, :max_account_fields]), + maxRemoteFields: Config.get([:instance, :max_remote_account_fields]), + nameLength: Config.get([:instance, :account_field_name_length]), + valueLength: Config.get([:instance, :account_field_value_length]) + }, + accountActivationRequired: Config.get([:instance, :account_activation_required], false), + invitesEnabled: Config.get([:instance, :invites_enabled], false), + mailerEnabled: Config.get([Pleroma.Emails.Mailer, :enabled], false), + features: features, + restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), + skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) + } + } + end + + def get_nodeinfo("2.1") do + raw_response = get_nodeinfo("2.0") + + updated_software = + raw_response + |> Map.get(:software) + |> Map.put(:repository, Pleroma.Application.repository()) + + raw_response + |> Map.put(:software, updated_software) + |> Map.put(:version, "2.1") + end + + def get_nodeinfo(_version) do + {:error, :missing} + end +end diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 721b599d4..8c7a9e565 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -5,12 +5,8 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do use Pleroma.Web, :controller - alias Pleroma.Config - alias Pleroma.Stats - alias Pleroma.User alias Pleroma.Web - alias Pleroma.Web.Federator.Publisher - alias Pleroma.Web.MastodonAPI.InstanceView + alias Pleroma.Web.Nodeinfo.Nodeinfo def schemas(conn, _params) do response = %{ @@ -29,102 +25,20 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do json(conn, response) end - # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field - # under software. - def raw_nodeinfo do - stats = Stats.get_stats() - - staff_accounts = - User.all_superusers() - |> Enum.map(fn u -> u.ap_id end) - - features = InstanceView.features() - federation = InstanceView.federation() - - %{ - version: "2.0", - software: %{ - name: Pleroma.Application.name() |> String.downcase(), - version: Pleroma.Application.version() - }, - protocols: Publisher.gather_nodeinfo_protocol_names(), - services: %{ - inbound: [], - outbound: [] - }, - openRegistrations: Config.get([:instance, :registrations_open]), - usage: %{ - users: %{ - total: Map.get(stats, :user_count, 0) - }, - localPosts: Map.get(stats, :status_count, 0) - }, - metadata: %{ - nodeName: Config.get([:instance, :name]), - nodeDescription: Config.get([:instance, :description]), - private: !Config.get([:instance, :public], true), - suggestions: %{ - enabled: false - }, - staffAccounts: staff_accounts, - federation: federation, - pollLimits: Config.get([:instance, :poll_limits]), - postFormats: Config.get([:instance, :allowed_post_formats]), - uploadLimits: %{ - general: Config.get([:instance, :upload_limit]), - avatar: Config.get([:instance, :avatar_upload_limit]), - banner: Config.get([:instance, :banner_upload_limit]), - background: Config.get([:instance, :background_upload_limit]) - }, - fieldsLimits: %{ - maxFields: Config.get([:instance, :max_account_fields]), - maxRemoteFields: Config.get([:instance, :max_remote_account_fields]), - nameLength: Config.get([:instance, :account_field_name_length]), - valueLength: Config.get([:instance, :account_field_value_length]) - }, - accountActivationRequired: Config.get([:instance, :account_activation_required], false), - invitesEnabled: Config.get([:instance, :invites_enabled], false), - mailerEnabled: Config.get([Pleroma.Emails.Mailer, :enabled], false), - features: features, - restrictedNicknames: Config.get([Pleroma.User, :restricted_nicknames]), - skipThreadContainment: Config.get([:instance, :skip_thread_containment], false) - } - } - end - # Schema definition: https://github.com/jhass/nodeinfo/blob/master/schemas/2.0/schema.json # and https://github.com/jhass/nodeinfo/blob/master/schemas/2.1/schema.json - def nodeinfo(conn, %{"version" => "2.0"}) do - conn - |> put_resp_header( - "content-type", - "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8" - ) - |> json(raw_nodeinfo()) - end - - def nodeinfo(conn, %{"version" => "2.1"}) do - raw_response = raw_nodeinfo() - - updated_software = - raw_response - |> Map.get(:software) - |> Map.put(:repository, Pleroma.Application.repository()) - - response = - raw_response - |> Map.put(:software, updated_software) - |> Map.put(:version, "2.1") - - conn - |> put_resp_header( - "content-type", - "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.1#; charset=utf-8" - ) - |> json(response) - end - - def nodeinfo(conn, _) do - render_error(conn, :not_found, "Nodeinfo schema version not handled") + def nodeinfo(conn, %{"version" => version}) do + case Nodeinfo.get_nodeinfo(version) do + {:error, :missing} -> + render_error(conn, :not_found, "Nodeinfo schema version not handled") + + node_info -> + conn + |> put_resp_header( + "content-type", + "application/json; profile=http://nodeinfo.diaspora.software/ns/schema/2.0#; charset=utf-8" + ) + |> json(node_info) + end end end diff --git a/lib/pleroma/web/preload.ex b/lib/pleroma/web/preload.ex new file mode 100644 index 000000000..c2211c597 --- /dev/null +++ b/lib/pleroma/web/preload.ex @@ -0,0 +1,30 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload do + alias Phoenix.HTML + require Logger + + def build_tags(_conn, params) do + preload_data = + Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), %{}, fn parser, acc -> + Map.merge(acc, parser.generate_terms(params)) + end) + + rendered_html = + preload_data + |> Jason.encode!() + |> build_script_tag() + |> HTML.safe_to_string() + + rendered_html + end + + def build_script_tag(content) do + HTML.Tag.content_tag(:script, HTML.raw(content), + id: "initial-results", + type: "application/json" + ) + end +end diff --git a/lib/pleroma/web/preload/instance.ex b/lib/pleroma/web/preload/instance.ex new file mode 100644 index 000000000..0b6fd3313 --- /dev/null +++ b/lib/pleroma/web/preload/instance.ex @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.Instance do + alias Pleroma.Web.MastodonAPI.InstanceView + alias Pleroma.Web.Nodeinfo.Nodeinfo + alias Pleroma.Web.Preload.Providers.Provider + + @behaviour Provider + @instance_url :"/api/v1/instance" + @panel_url :"/instance/panel.html" + @nodeinfo_url :"/nodeinfo/2.0" + + @impl Provider + def generate_terms(_params) do + %{} + |> build_info_tag() + |> build_panel_tag() + |> build_nodeinfo_tag() + end + + defp build_info_tag(acc) do + info_data = InstanceView.render("show.json", %{}) + + Map.put(acc, @instance_url, info_data) + end + + defp build_panel_tag(acc) do + instance_path = Path.join(:code.priv_dir(:pleroma), "static/instance/panel.html") + + if File.exists?(instance_path) do + panel_data = File.read!(instance_path) + Map.put(acc, @panel_url, panel_data) + else + acc + end + end + + defp build_nodeinfo_tag(acc) do + case Nodeinfo.get_nodeinfo("2.0") do + {:error, _} -> + acc + + nodeinfo_data -> + Map.put(acc, @nodeinfo_url, nodeinfo_data) + end + end +end diff --git a/lib/pleroma/web/preload/provider.ex b/lib/pleroma/web/preload/provider.ex new file mode 100644 index 000000000..7ef595a34 --- /dev/null +++ b/lib/pleroma/web/preload/provider.ex @@ -0,0 +1,7 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.Provider do + @callback generate_terms(map()) :: map() +end diff --git a/lib/pleroma/web/preload/timelines.ex b/lib/pleroma/web/preload/timelines.ex new file mode 100644 index 000000000..dbd7db407 --- /dev/null +++ b/lib/pleroma/web/preload/timelines.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.Timelines do + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.Preload.Providers.Provider + + @behaviour Provider + @public_url :"/api/v1/timelines/public" + + @impl Provider + def generate_terms(_params) do + build_public_tag(%{}) + end + + def build_public_tag(acc) do + if Pleroma.Config.get([:restrict_unauthenticated, :timelines, :federated], true) do + acc + else + Map.put(acc, @public_url, public_timeline(nil)) + end + end + + defp public_timeline(user) do + activities = + create_timeline_params(user) + |> Map.put("local_only", false) + |> ActivityPub.fetch_public_activities() + + StatusView.render("index.json", activities: activities, for: user, as: :activity) + end + + defp create_timeline_params(user) do + %{} + |> Map.put("type", ["Create", "Announce"]) + |> Map.put("blocking_user", user) + |> Map.put("muting_user", user) + |> Map.put("user", user) + end +end diff --git a/lib/pleroma/web/preload/user.ex b/lib/pleroma/web/preload/user.ex new file mode 100644 index 000000000..3a244845b --- /dev/null +++ b/lib/pleroma/web/preload/user.ex @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.User do + alias Pleroma.Web.MastodonAPI.AccountView + alias Pleroma.Web.Preload.Providers.Provider + + @behaviour Provider + @account_url :"/api/v1/accounts" + + @impl Provider + def generate_terms(%{user: user}) do + build_accounts_tag(%{}, user) + end + + def generate_terms(_params), do: %{} + + def build_accounts_tag(acc, nil), do: acc + + def build_accounts_tag(acc, user) do + account_data = AccountView.render("show.json", %{user: user, for: user}) + Map.put(acc, @account_url, account_data) + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 80ea28364..3b55afede 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -718,7 +718,7 @@ defmodule Pleroma.Web.Router do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) get("/api*path", RedirectController, :api_not_implemented) - get("/*path", RedirectController, :redirector) + get("/*path", RedirectController, :redirector_with_preload) options("/*path", RedirectController, :empty) end -- cgit v1.2.3 From dbcc1b105ee1a2552595d189d8ac9d8484ffb601 Mon Sep 17 00:00:00 2001 From: stwf Date: Tue, 2 Jun 2020 10:18:06 -0400 Subject: encode data properly --- lib/pleroma/web/fallback_redirect_controller.ex | 7 +++--- lib/pleroma/web/preload.ex | 2 ++ lib/pleroma/web/preload/timelines.ex | 31 +++++++++++-------------- 3 files changed, 20 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/fallback_redirect_controller.ex b/lib/pleroma/web/fallback_redirect_controller.ex index 932fb8d7e..431ad5485 100644 --- a/lib/pleroma/web/fallback_redirect_controller.ex +++ b/lib/pleroma/web/fallback_redirect_controller.ex @@ -4,7 +4,9 @@ defmodule Fallback.RedirectController do use Pleroma.Web, :controller + require Logger + alias Pleroma.User alias Pleroma.Web.Metadata alias Pleroma.Web.Preload @@ -38,8 +40,7 @@ defmodule Fallback.RedirectController do response = index_content - |> String.replace("", tags) - |> String.replace("", preloads) + |> String.replace("", tags <> preloads) conn |> put_resp_content_type("text/html") @@ -56,7 +57,7 @@ defmodule Fallback.RedirectController do response = index_content - |> String.replace("", preloads) + |> String.replace("", preloads) conn |> put_resp_content_type("text/html") diff --git a/lib/pleroma/web/preload.ex b/lib/pleroma/web/preload.ex index c2211c597..f13932b89 100644 --- a/lib/pleroma/web/preload.ex +++ b/lib/pleroma/web/preload.ex @@ -22,6 +22,8 @@ defmodule Pleroma.Web.Preload do end def build_script_tag(content) do + content = Base.encode64(content) + HTML.Tag.content_tag(:script, HTML.raw(content), id: "initial-results", type: "application/json" diff --git a/lib/pleroma/web/preload/timelines.ex b/lib/pleroma/web/preload/timelines.ex index dbd7db407..2bb57567b 100644 --- a/lib/pleroma/web/preload/timelines.ex +++ b/lib/pleroma/web/preload/timelines.ex @@ -11,32 +11,29 @@ defmodule Pleroma.Web.Preload.Providers.Timelines do @public_url :"/api/v1/timelines/public" @impl Provider - def generate_terms(_params) do - build_public_tag(%{}) + def generate_terms(params) do + build_public_tag(%{}, params) end - def build_public_tag(acc) do + def build_public_tag(acc, params) do if Pleroma.Config.get([:restrict_unauthenticated, :timelines, :federated], true) do acc else - Map.put(acc, @public_url, public_timeline(nil)) + Map.put(acc, @public_url, public_timeline(params)) end end - defp public_timeline(user) do - activities = - create_timeline_params(user) - |> Map.put("local_only", false) - |> ActivityPub.fetch_public_activities() + defp public_timeline(%{"path" => ["main", "all"]}), do: get_public_timeline(false) - StatusView.render("index.json", activities: activities, for: user, as: :activity) - end + defp public_timeline(_params), do: get_public_timeline(true) + + defp get_public_timeline(local_only) do + activities = + ActivityPub.fetch_public_activities(%{ + "type" => ["Create"], + "local_only" => local_only + }) - defp create_timeline_params(user) do - %{} - |> Map.put("type", ["Create", "Announce"]) - |> Map.put("blocking_user", user) - |> Map.put("muting_user", user) - |> Map.put("user", user) + StatusView.render("index.json", activities: activities, for: nil, as: :activity) end end -- cgit v1.2.3 From 3b8180d7d1f52a9eae1913a59b9c970f6600e674 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 3 Jun 2020 13:40:48 -0400 Subject: add status_net/config --- lib/pleroma/web/preload.ex | 10 ++++++--- lib/pleroma/web/preload/status_net.ex | 24 ++++++++++++++++++++++ .../web/twitter_api/controllers/util_controller.ex | 13 ++---------- lib/pleroma/web/twitter_api/views/util_view.ex | 14 +++++++++++++ 4 files changed, 47 insertions(+), 14 deletions(-) create mode 100644 lib/pleroma/web/preload/status_net.ex (limited to 'lib') diff --git a/lib/pleroma/web/preload.ex b/lib/pleroma/web/preload.ex index f13932b89..90e454468 100644 --- a/lib/pleroma/web/preload.ex +++ b/lib/pleroma/web/preload.ex @@ -9,7 +9,13 @@ defmodule Pleroma.Web.Preload do def build_tags(_conn, params) do preload_data = Enum.reduce(Pleroma.Config.get([__MODULE__, :providers], []), %{}, fn parser, acc -> - Map.merge(acc, parser.generate_terms(params)) + terms = + params + |> parser.generate_terms() + |> Enum.map(fn {k, v} -> {k, Base.encode64(Jason.encode!(v))} end) + |> Enum.into(%{}) + + Map.merge(acc, terms) end) rendered_html = @@ -22,8 +28,6 @@ defmodule Pleroma.Web.Preload do end def build_script_tag(content) do - content = Base.encode64(content) - HTML.Tag.content_tag(:script, HTML.raw(content), id: "initial-results", type: "application/json" diff --git a/lib/pleroma/web/preload/status_net.ex b/lib/pleroma/web/preload/status_net.ex new file mode 100644 index 000000000..7e592d60d --- /dev/null +++ b/lib/pleroma/web/preload/status_net.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Preload.Providers.StatusNet do + alias Pleroma.Web.TwitterAPI.UtilView + alias Pleroma.Web.Preload.Providers.Provider + + @behaviour Provider + @config_url :"/api/statusnet/config.json" + + @impl Provider + def generate_terms(_params) do + %{} + |> build_config_tag() + end + + defp build_config_tag(acc) do + instance = Pleroma.Config.get(:instance) + info_data = UtilView.status_net_config(instance) + + Map.put(acc, @config_url, info_data) + end +end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index fd2aee175..aaca182ec 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.User alias Pleroma.Web alias Pleroma.Web.CommonAPI + alias Pleroma.Web.TwitterAPI.UtilView alias Pleroma.Web.WebFinger plug(Pleroma.Web.FederatingPlug when action == :remote_subscribe) @@ -90,17 +91,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def config(%{assigns: %{format: "xml"}} = conn, _params) do instance = Pleroma.Config.get(:instance) - - response = """ - - - #{Keyword.get(instance, :name)} - #{Web.base_url()} - #{Keyword.get(instance, :limit)} - #{!Keyword.get(instance, :registrations_open)} - - - """ + response = UtilView.status_net_config(instance) conn |> put_resp_content_type("application/xml") diff --git a/lib/pleroma/web/twitter_api/views/util_view.ex b/lib/pleroma/web/twitter_api/views/util_view.ex index 52054e020..d3bdb4f62 100644 --- a/lib/pleroma/web/twitter_api/views/util_view.ex +++ b/lib/pleroma/web/twitter_api/views/util_view.ex @@ -5,4 +5,18 @@ defmodule Pleroma.Web.TwitterAPI.UtilView do use Pleroma.Web, :view import Phoenix.HTML.Form + alias Pleroma.Web + + def status_net_config(instance) do + """ + + + #{Keyword.get(instance, :name)} + #{Web.base_url()} + #{Keyword.get(instance, :limit)} + #{!Keyword.get(instance, :registrations_open)} + + + """ + end end -- cgit v1.2.3 From 5677b21e824aa7f3b5124068ef65041e04002579 Mon Sep 17 00:00:00 2001 From: stwf Date: Wed, 3 Jun 2020 16:32:38 -0400 Subject: clean up --- lib/pleroma/web/preload/status_net.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/preload/status_net.ex b/lib/pleroma/web/preload/status_net.ex index 7e592d60d..367442d5c 100644 --- a/lib/pleroma/web/preload/status_net.ex +++ b/lib/pleroma/web/preload/status_net.ex @@ -3,8 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Preload.Providers.StatusNet do - alias Pleroma.Web.TwitterAPI.UtilView alias Pleroma.Web.Preload.Providers.Provider + alias Pleroma.Web.TwitterAPI.UtilView @behaviour Provider @config_url :"/api/statusnet/config.json" -- cgit v1.2.3 From faba1a6e337715af557e2e222e62de6fd35c9e8a Mon Sep 17 00:00:00 2001 From: stwf Date: Mon, 15 Jun 2020 12:25:03 -0400 Subject: fix tests --- lib/pleroma/web/preload/timelines.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/preload/timelines.ex b/lib/pleroma/web/preload/timelines.ex index 2bb57567b..e531b8960 100644 --- a/lib/pleroma/web/preload/timelines.ex +++ b/lib/pleroma/web/preload/timelines.ex @@ -30,8 +30,8 @@ defmodule Pleroma.Web.Preload.Providers.Timelines do defp get_public_timeline(local_only) do activities = ActivityPub.fetch_public_activities(%{ - "type" => ["Create"], - "local_only" => local_only + type: ["Create"], + local_only: local_only }) StatusView.render("index.json", activities: activities, for: nil, as: :activity) -- cgit v1.2.3 From a8d967762ec5436ca9b478fbbedfec39b5d9e35e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 23 Jun 2020 15:09:01 +0300 Subject: migrate to oban 2.0-rc1 --- lib/pleroma/application.ex | 14 ++++++++- lib/pleroma/workers/attachments_cleanup_worker.ex | 11 ++++--- lib/pleroma/workers/background_worker.ex | 34 +++++++++++----------- .../workers/cron/clear_oauth_token_worker.ex | 2 +- lib/pleroma/workers/cron/digest_emails_worker.ex | 2 +- .../workers/cron/new_users_digest_worker.ex | 2 +- .../cron/purge_expired_activities_worker.ex | 2 +- lib/pleroma/workers/cron/stats_worker.ex | 2 +- lib/pleroma/workers/mailer_worker.ex | 2 +- lib/pleroma/workers/publisher_worker.ex | 6 ++-- lib/pleroma/workers/receiver_worker.ex | 2 +- lib/pleroma/workers/remote_fetcher_worker.ex | 8 +---- lib/pleroma/workers/scheduled_activity_worker.ex | 2 +- lib/pleroma/workers/transmogrifier_worker.ex | 2 +- lib/pleroma/workers/web_pusher_worker.ex | 2 +- lib/pleroma/workers/worker_helper.ex | 4 ++- 16 files changed, 52 insertions(+), 45 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9615af122..fb2731f97 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -80,7 +80,7 @@ defmodule Pleroma.Application do [ Pleroma.Stats, Pleroma.JobQueueMonitor, - {Oban, Config.get(Oban)} + {Oban, oban_config()} ] ++ task_children(@env) ++ streamer_child(@env) ++ @@ -138,6 +138,18 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint.Instrumenter.setup() end + defp oban_config do + config = Config.get(Oban) + + if Code.ensure_loaded?(IEx) and IEx.started?() do + config + |> Keyword.put(:crontab, false) + |> Keyword.put(:queues, false) + else + config + end + end + defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 8deeabda0..58226b395 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -11,13 +11,12 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" @impl Oban.Worker - def perform( - %{ + def perform(%Job{ + args: %{ "op" => "cleanup_attachments", "object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}} - }, - _job - ) do + } + }) do attachments |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end) |> fetch_objects @@ -28,7 +27,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do {:ok, :success} end - def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip} defp do_clean({object_ids, attachment_urls}) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 57c3a9c3a..cec5a7462 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -11,59 +11,59 @@ defmodule Pleroma.Workers.BackgroundWorker do @impl Oban.Worker - def perform(%{"op" => "deactivate_user", "user_id" => user_id, "status" => status}, _job) do + def perform(%Job{args: %{"op" => "deactivate_user", "user_id" => user_id, "status" => status}}) do user = User.get_cached_by_id(user_id) User.perform(:deactivate_async, user, status) end - def perform(%{"op" => "delete_user", "user_id" => user_id}, _job) do + def perform(%Job{args: %{"op" => "delete_user", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) User.perform(:delete, user) end - def perform(%{"op" => "force_password_reset", "user_id" => user_id}, _job) do + def perform(%Job{args: %{"op" => "force_password_reset", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) User.perform(:force_password_reset, user) end - def perform( - %{ + def perform(%Job{ + args: %{ "op" => "blocks_import", "blocker_id" => blocker_id, "blocked_identifiers" => blocked_identifiers - }, - _job - ) do + } + }) do blocker = User.get_cached_by_id(blocker_id) {:ok, User.perform(:blocks_import, blocker, blocked_identifiers)} end - def perform( - %{ + def perform(%Job{ + args: %{ "op" => "follow_import", "follower_id" => follower_id, "followed_identifiers" => followed_identifiers - }, - _job - ) do + } + }) do follower = User.get_cached_by_id(follower_id) {:ok, User.perform(:follow_import, follower, followed_identifiers)} end - def perform(%{"op" => "media_proxy_preload", "message" => message}, _job) do + def perform(%Job{args: %{"op" => "media_proxy_preload", "message" => message}}) do MediaProxyWarmingPolicy.perform(:preload, message) end - def perform(%{"op" => "media_proxy_prefetch", "url" => url}, _job) do + def perform(%Job{args: %{"op" => "media_proxy_prefetch", "url" => url}}) do MediaProxyWarmingPolicy.perform(:prefetch, url) end - def perform(%{"op" => "fetch_data_for_activity", "activity_id" => activity_id}, _job) do + def perform(%Job{args: %{"op" => "fetch_data_for_activity", "activity_id" => activity_id}}) do activity = Activity.get_by_id(activity_id) Pleroma.Web.RichMedia.Helpers.perform(:fetch, activity) end - def perform(%{"op" => "move_following", "origin_id" => origin_id, "target_id" => target_id}, _) do + def perform(%Job{ + args: %{"op" => "move_following", "origin_id" => origin_id, "target_id" => target_id} + }) do origin = User.get_cached_by_id(origin_id) target = User.get_cached_by_id(target_id) diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex index a4c3b9516..d41be4e87 100644 --- a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex +++ b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do alias Pleroma.Web.OAuth.Token @impl Oban.Worker - def perform(_opts, _job) do + def perform(_job) do if Config.get([:oauth2, :clean_expired_tokens], false) do Token.delete_expired_tokens() else diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 7f09ff3cf..ee646229f 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do require Logger @impl Oban.Worker - def perform(_opts, _job) do + def perform(_job) do config = Config.get([:email_notifications, :digest]) if config[:active] do diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 5c816b3fe..abc8a5e95 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do use Pleroma.Workers.WorkerHelper, queue: "new_users_digest" @impl Oban.Worker - def perform(_args, _job) do + def perform(_job) do if Pleroma.Config.get([Pleroma.Emails.NewUsersDigestEmail, :enabled]) do today = NaiveDateTime.utc_now() |> Timex.beginning_of_day() diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex index 84b3b84de..e926c5dc8 100644 --- a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex +++ b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do @interval :timer.minutes(1) @impl Oban.Worker - def perform(_opts, _job) do + def perform(_job) do if Config.get([ActivityExpiration, :enabled]) do Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1) else diff --git a/lib/pleroma/workers/cron/stats_worker.ex b/lib/pleroma/workers/cron/stats_worker.ex index e9b8d59c4..e54bd9a7f 100644 --- a/lib/pleroma/workers/cron/stats_worker.ex +++ b/lib/pleroma/workers/cron/stats_worker.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Workers.Cron.StatsWorker do use Oban.Worker, queue: "background" @impl Oban.Worker - def perform(_opts, _job) do + def perform(_job) do Pleroma.Stats.do_collect() end end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 6955338a5..32273cfa5 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.MailerWorker do use Pleroma.Workers.WorkerHelper, queue: "mailer" @impl Oban.Worker - def perform(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}, _job) do + def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do encoded_email |> Base.decode64!() |> :erlang.binary_to_term() diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index daf79efc0..e739c3cd0 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -8,17 +8,17 @@ defmodule Pleroma.Workers.PublisherWorker do use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" - def backoff(attempt) when is_integer(attempt) do + def backoff(%Job{attempt: attempt}) when is_integer(attempt) do Pleroma.Workers.WorkerHelper.sidekiq_backoff(attempt, 5) end @impl Oban.Worker - def perform(%{"op" => "publish", "activity_id" => activity_id}, _job) do + def perform(%Job{args: %{"op" => "publish", "activity_id" => activity_id}}) do activity = Activity.get_by_id(activity_id) Federator.perform(:publish, activity) end - def perform(%{"op" => "publish_one", "module" => module_name, "params" => params}, _job) do + def perform(%Job{args: %{"op" => "publish_one", "module" => module_name, "params" => params}}) do params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end) Federator.perform(:publish_one, String.to_atom(module_name), params) end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index f7a7124f3..1b97af1a8 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Workers.ReceiverWorker do use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" @impl Oban.Worker - def perform(%{"op" => "incoming_ap_doc", "params" => params}, _job) do + def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do Federator.perform(:incoming_ap_doc, params) end end diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index ec6534f21..27e2e3386 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -8,13 +8,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" @impl Oban.Worker - def perform( - %{ - "op" => "fetch_remote", - "id" => id - } = args, - _job - ) do + def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do {:ok, _object} = Fetcher.fetch_object_from_id(id, depth: args["depth"]) end end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 97d1efbfb..dd9986fe4 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do require Logger @impl Oban.Worker - def perform(%{"activity_id" => activity_id}, _job) do + def perform(%Job{args: %{"activity_id" => activity_id}}) do if Config.get([ScheduledActivity, :enabled]) do case Pleroma.Repo.get(ScheduledActivity, activity_id) do %ScheduledActivity{} = scheduled_activity -> diff --git a/lib/pleroma/workers/transmogrifier_worker.ex b/lib/pleroma/workers/transmogrifier_worker.ex index 11239ca5e..15f36375c 100644 --- a/lib/pleroma/workers/transmogrifier_worker.ex +++ b/lib/pleroma/workers/transmogrifier_worker.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Workers.TransmogrifierWorker do use Pleroma.Workers.WorkerHelper, queue: "transmogrifier" @impl Oban.Worker - def perform(%{"op" => "user_upgrade", "user_id" => user_id}, _job) do + def perform(%Job{args: %{"op" => "user_upgrade", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) end diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index 58ad25e39..0cfdc6a6f 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.WebPusherWorker do use Pleroma.Workers.WorkerHelper, queue: "web_push" @impl Oban.Worker - def perform(%{"op" => "web_push", "notification_id" => notification_id}, _job) do + def perform(%Job{args: %{"op" => "web_push", "notification_id" => notification_id}}) do notification = Notification |> Repo.get(notification_id) diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex index d1f90c35b..7d1289be2 100644 --- a/lib/pleroma/workers/worker_helper.ex +++ b/lib/pleroma/workers/worker_helper.ex @@ -32,6 +32,8 @@ defmodule Pleroma.Workers.WorkerHelper do queue: unquote(queue), max_attempts: 1 + alias Oban.Job + def enqueue(op, params, worker_args \\ []) do params = Map.merge(%{"op" => op}, params) queue_atom = String.to_atom(unquote(queue)) @@ -39,7 +41,7 @@ defmodule Pleroma.Workers.WorkerHelper do unquote(caller_module) |> apply(:new, [params, worker_args]) - |> Pleroma.Repo.insert() + |> Oban.insert() end end end -- cgit v1.2.3 From 35f6770436837e2e500971a54d51984bd059adfd Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 24 Jun 2020 13:29:08 +0200 Subject: StatusView: Add pleroma.parent_visible --- lib/pleroma/web/activity_pub/visibility.ex | 6 ++++-- lib/pleroma/web/mastodon_api/views/status_view.ex | 5 +++-- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 453a6842e..343f41caa 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -47,6 +47,10 @@ defmodule Pleroma.Web.ActivityPub.Visibility do @spec visible_for_user?(Activity.t(), User.t() | nil) :: boolean() def visible_for_user?(%{actor: ap_id}, %User{ap_id: ap_id}), do: true + def visible_for_user?(nil, _), do: false + + def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false + def visible_for_user?(%{data: %{"listMessage" => list_ap_id}} = activity, %User{} = user) do user.ap_id in activity.data["to"] || list_ap_id @@ -54,8 +58,6 @@ defmodule Pleroma.Web.ActivityPub.Visibility do |> Pleroma.List.member?(user) end - def visible_for_user?(%{data: %{"listMessage" => _}}, nil), do: false - def visible_for_user?(%{local: local} = activity, nil) do cfg_key = if local, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2c49bedb3..6ee17f4dd 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy - import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1] + import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2] # TODO: Add cached version. defp get_replied_to_activities([]), do: %{} @@ -364,7 +364,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do expires_at: expires_at, direct_conversation_id: direct_conversation_id, thread_muted: thread_muted?, - emoji_reactions: emoji_reactions + emoji_reactions: emoji_reactions, + parent_visible: visible_for_user?(reply_to, opts[:for]) } } end -- cgit v1.2.3 From 4c5fb831b3b59309a475a141eb73cc440533d0ff Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 24 Jun 2020 14:33:00 +0200 Subject: Status schema: Add parent_visible. --- lib/pleroma/web/api_spec/schemas/status.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 8b87cb25b..28cde963e 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -184,6 +184,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do thread_muted: %Schema{ type: :boolean, description: "`true` if the thread the post belongs to is muted" + }, + parent_visible: %Schema{ + type: :boolean, + description: "`true` if the parent post is visible to the user" } } }, -- cgit v1.2.3 From cc837f9d157f9d43a015a8908f5e2ee178442041 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Wed, 24 Jun 2020 21:21:33 +0300 Subject: fixed config/descpiption.exs --- lib/pleroma/application.ex | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index fb2731f97..9615af122 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -80,7 +80,7 @@ defmodule Pleroma.Application do [ Pleroma.Stats, Pleroma.JobQueueMonitor, - {Oban, oban_config()} + {Oban, Config.get(Oban)} ] ++ task_children(@env) ++ streamer_child(@env) ++ @@ -138,18 +138,6 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint.Instrumenter.setup() end - defp oban_config do - config = Config.get(Oban) - - if Code.ensure_loaded?(IEx) and IEx.started?() do - config - |> Keyword.put(:crontab, false) - |> Keyword.put(:queues, false) - else - config - end - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), -- cgit v1.2.3 From 89e5b2046bd15b3fead7a6194a2b9cecd2fedbd3 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 25 Jun 2020 11:13:35 +0200 Subject: ObjectValidator: Basic `Block` support. --- lib/pleroma/web/activity_pub/builder.ex | 12 +++++++ lib/pleroma/web/activity_pub/object_validator.ex | 11 ++++++ .../object_validators/block_validator.ex | 42 ++++++++++++++++++++++ 3 files changed, 65 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/object_validators/block_validator.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 135a5c431..cabc28de9 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -138,6 +138,18 @@ defmodule Pleroma.Web.ActivityPub.Builder do }, []} end + @spec block(User.t(), User.t()) :: {:ok, map(), keyword()} + def block(blocker, blocked) do + {:ok, + %{ + "id" => Utils.generate_activity_id(), + "type" => "Block", + "actor" => blocker.ap_id, + "object" => blocked.ap_id, + "to" => [blocked.ap_id] + }, []} + end + @spec announce(User.t(), Object.t(), keyword()) :: {:ok, map(), keyword()} def announce(actor, object, options \\ []) do public? = Keyword.get(options, :public, false) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 2c657b467..737c0fd64 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator @@ -24,6 +25,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} def validate(object, meta) + def validate(%{"type" => "Block"} = block_activity, meta) do + with {:ok, block_activity} <- + block_activity + |> BlockValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + block_activity = stringify_keys(block_activity) + {:ok, block_activity, meta} + end + end + def validate(%{"type" => "Update"} = update_activity, meta) do with {:ok, update_activity} <- update_activity diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex new file mode 100644 index 000000000..1dde77198 --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:type, :string) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:object, ObjectValidators.ObjectID) + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end + + def validate_data(cng) do + cng + |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_inclusion(:type, ["Block"]) + |> validate_actor_presence() + |> validate_actor_presence(field_name: :object) + end + + def cast_and_validate(data) do + data + |> cast_data + |> validate_data + end +end -- cgit v1.2.3 From e38293c8f1adca40447ba39f4919b2b08bf0329a Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 25 Jun 2020 11:33:54 +0200 Subject: Transmogrifier: Switch to pipeline for Blocks. --- lib/pleroma/web/activity_pub/side_effects.ex | 16 ++++++++++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 22 ++++------------------ 2 files changed, 20 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index de143b8f0..48350d2b3 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -20,6 +20,22 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do def handle(object, meta \\ []) + # Tasks this handles: + # - Unfollow and block + def handle( + %{data: %{"type" => "Block", "object" => blocked_user, "actor" => blocking_user}} = + object, + meta + ) do + with %User{} = blocker <- User.get_cached_by_ap_id(blocking_user), + %User{} = blocked <- User.get_cached_by_ap_id(blocked_user) do + User.unfollow(blocker, blocked) + User.block(blocker, blocked) + end + + {:ok, object, meta} + end + # Tasks this handles: # - Update the user # diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 4e318e89c..278fbbeab 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -673,7 +673,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming(%{"type" => type} = data, _options) - when type in ["Like", "EmojiReact", "Announce"] do + when type in ~w{Like EmojiReact Announce} do with :ok <- ObjectValidator.fetch_actor_and_object(data), {:ok, activity, _meta} <- Pipeline.common_pipeline(data, local: false) do @@ -684,9 +684,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def handle_incoming( - %{"type" => "Update"} = data, + %{"type" => type} = data, _options - ) do + ) + when type in ~w{Update Block} do with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} @@ -765,21 +766,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming( - %{"type" => "Block", "object" => blocked, "actor" => blocker, "id" => id} = _data, - _options - ) do - with %User{local: true} = blocked = User.get_cached_by_ap_id(blocked), - {:ok, %User{} = blocker} = User.get_or_fetch_by_ap_id(blocker), - {:ok, activity} <- ActivityPub.block(blocker, blocked, id, false) do - User.unfollow(blocker, blocked) - User.block(blocker, blocked) - {:ok, activity} - else - _e -> :error - end - end - def handle_incoming( %{ "type" => "Move", -- cgit v1.2.3 From 8cfb58a8c0a2ee0c69eb727cc810e8571289f813 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 25 Jun 2020 11:44:04 +0200 Subject: AccountController: Extract blocking to CommonAPI. --- lib/pleroma/web/common_api/common_api.ex | 7 +++++++ lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 3 +-- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 04e081a8e..fd7149079 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -25,6 +25,13 @@ defmodule Pleroma.Web.CommonAPI do require Pleroma.Constants require Logger + def block(blocker, blocked) do + with {:ok, block_data, _} <- Builder.block(blocker, blocked), + {:ok, block, _} <- Pipeline.common_pipeline(block_data, local: true) do + {:ok, block} + end + end + def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]), :ok <- validate_chat_content_length(content, !!maybe_attachment), diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 7a88a847c..b5008d69b 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -385,8 +385,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do @doc "POST /api/v1/accounts/:id/block" def block(%{assigns: %{user: blocker, account: blocked}} = conn, _params) do - with {:ok, _user_block} <- User.block(blocker, blocked), - {:ok, _activity} <- ActivityPub.block(blocker, blocked) do + with {:ok, _activity} <- CommonAPI.block(blocker, blocked) do render(conn, "relationship.json", user: blocker, target: blocked) else {:error, message} -> json_response(conn, :forbidden, %{error: message}) -- cgit v1.2.3 From 44bb7cfccdf2c25ae641b4cffa8e5c7fdedc3f54 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 25 Jun 2020 11:51:33 +0200 Subject: ActivityPub: Remove `block`. --- lib/pleroma/user.ex | 3 +-- lib/pleroma/web/activity_pub/activity_pub.ex | 27 --------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 1d70a37ef..c3e2a89ad 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1527,8 +1527,7 @@ defmodule Pleroma.User do blocked_identifiers, fn blocked_identifier -> with {:ok, %User{} = blocked} <- get_or_fetch(blocked_identifier), - {:ok, _user_block} <- block(blocker, blocked), - {:ok, _} <- ActivityPub.block(blocker, blocked) do + {:ok, _block} <- CommonAPI.block(blocker, blocked) do blocked else err -> diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7cd3eab39..05bd824f5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -366,33 +366,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - @spec block(User.t(), User.t(), String.t() | nil, boolean()) :: - {:ok, Activity.t()} | {:error, any()} - def block(blocker, blocked, activity_id \\ nil, local \\ true) do - with {:ok, result} <- - Repo.transaction(fn -> do_block(blocker, blocked, activity_id, local) end) do - result - end - end - - defp do_block(blocker, blocked, activity_id, local) do - unfollow_blocked = Config.get([:activitypub, :unfollow_blocked]) - - if unfollow_blocked and fetch_latest_follow(blocker, blocked) do - unfollow(blocker, blocked, nil, local) - end - - block_data = make_block_data(blocker, blocked, activity_id) - - with {:ok, activity} <- insert(block_data, local), - _ <- notify_and_stream(activity), - :ok <- maybe_federate(activity) do - {:ok, activity} - else - {:error, error} -> Repo.rollback(error) - end - end - @spec flag(map()) :: {:ok, Activity.t()} | {:error, any()} def flag( %{ -- cgit v1.2.3 From 84f9ca19568777861ff9520cbef09a0259efd536 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 25 Jun 2020 12:03:14 +0200 Subject: Blocking: Don't federate if the options is set. --- lib/pleroma/web/activity_pub/object_validator.ex | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 737c0fd64..bb6324460 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -31,6 +31,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do |> BlockValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do block_activity = stringify_keys(block_activity) + outgoing_blocks = Pleroma.Config.get([:activitypub, :outgoing_blocks]) + + meta = + if !outgoing_blocks do + Keyword.put(meta, :do_not_federate, true) + else + meta + end + {:ok, block_activity, meta} end end -- cgit v1.2.3 From 04abee782b8745b21d0f9e58b27a805db6a94aa7 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 25 Jun 2020 12:40:39 +0200 Subject: AntiSpamLinkPolicy: Exempt local users. --- lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 9e7800997..a7e187b5e 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -27,11 +27,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do @impl true def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do - with {:ok, %User{} = u} <- User.get_or_fetch_by_ap_id(actor), + with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), {:contains_links, true} <- {:contains_links, contains_links?(object)}, {:old_user, true} <- {:old_user, old_user?(u)} do {:ok, message} else + {:ok, %User{local: true}} -> + {:ok, message} + {:contains_links, false} -> {:ok, message} -- cgit v1.2.3 From d6c958b4c22ee7658ee8b7b11fc6ddede1082cca Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jun 2020 05:33:59 +0200 Subject: nodeinfo: Fix MRF transparency --- lib/pleroma/web/nodeinfo/nodeinfo.ex | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/nodeinfo/nodeinfo.ex b/lib/pleroma/web/nodeinfo/nodeinfo.ex index d26b7c938..f7ab6d86a 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo.ex @@ -6,30 +6,19 @@ defmodule Pleroma.Web.Nodeinfo.Nodeinfo do alias Pleroma.Config alias Pleroma.Stats alias Pleroma.User - alias Pleroma.Web.ActivityPub.MRF alias Pleroma.Web.Federator.Publisher + alias Pleroma.Web.MastodonAPI.InstanceView # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field # under software. def get_nodeinfo("2.0") do stats = Stats.get_stats() - quarantined = Config.get([:instance, :quarantined_instances], []) - staff_accounts = User.all_superusers() |> Enum.map(fn u -> u.ap_id end) - federation_response = - if Config.get([:instance, :mrf_transparency]) do - {:ok, data} = MRF.describe() - - data - |> Map.merge(%{quarantined_instances: quarantined}) - else - %{} - end - |> Map.put(:enabled, Config.get([:instance, :federating])) + federation = InstanceView.federation() features = [ @@ -86,7 +75,7 @@ defmodule Pleroma.Web.Nodeinfo.Nodeinfo do enabled: false }, staffAccounts: staff_accounts, - federation: federation_response, + federation: federation, pollLimits: Config.get([:instance, :poll_limits]), postFormats: Config.get([:instance, :allowed_post_formats]), uploadLimits: %{ -- cgit v1.2.3 From 27c33f216ad250b60d44fe0662c3be3c4cee987e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jun 2020 05:39:35 +0200 Subject: activity_draft: Add source field --- lib/pleroma/web/common_api/activity_draft.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 9bcb9f587..f849b2e01 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -186,6 +186,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do draft.poll ) |> Map.put("emoji", emoji) + |> Map.put("source", draft.status) %__MODULE__{draft | object: object} end -- cgit v1.2.3 From c3383d4fab6181d9f605a6058805333611534398 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 26 Jun 2020 11:58:40 +0200 Subject: BlockValidator: Restore old behavior for incoming blocks. --- .../web/activity_pub/object_validators/block_validator.ex | 13 +++++++++++++ lib/pleroma/web/activity_pub/side_effects.ex | 1 - 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex index 1dde77198..1989585b7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.User import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -32,6 +33,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do |> validate_inclusion(:type, ["Block"]) |> validate_actor_presence() |> validate_actor_presence(field_name: :object) + |> validate_block_acceptance() end def cast_and_validate(data) do @@ -39,4 +41,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do |> cast_data |> validate_data end + + def validate_block_acceptance(cng) do + actor = get_field(cng, :actor) |> User.get_cached_by_ap_id() + + if actor.local || Pleroma.Config.get([:activitypub, :unfollow_blocked], true) do + cng + else + cng + |> add_error(:actor, "Not accepting remote blocks") + end + end end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 48350d2b3..5cc2eb378 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -29,7 +29,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do ) do with %User{} = blocker <- User.get_cached_by_ap_id(blocking_user), %User{} = blocked <- User.get_cached_by_ap_id(blocked_user) do - User.unfollow(blocker, blocked) User.block(blocker, blocked) end -- cgit v1.2.3 From 15a8b703185c685fc3d25a381fcb9dee522c78bf Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 26 Jun 2020 12:06:00 +0200 Subject: User: Don't unfollow on block when the relevant setting is set. --- lib/pleroma/user.ex | 3 ++- .../web/activity_pub/object_validators/block_validator.ex | 13 ------------- 2 files changed, 2 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c3e2a89ad..9d5c61e79 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1309,7 +1309,8 @@ defmodule Pleroma.User do unsubscribe(blocked, blocker) - if following?(blocked, blocker), do: unfollow(blocked, blocker) + unfollowing_blocked = Config.get([:activitypub, :unfollow_blocked], true) + if unfollowing_blocked && following?(blocked, blocker), do: unfollow(blocked, blocker) {:ok, blocker} = update_follower_count(blocker) {:ok, blocker, _} = Participation.mark_all_as_read(blocker, blocked) diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex index 1989585b7..1dde77198 100644 --- a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.User import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -33,7 +32,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do |> validate_inclusion(:type, ["Block"]) |> validate_actor_presence() |> validate_actor_presence(field_name: :object) - |> validate_block_acceptance() end def cast_and_validate(data) do @@ -41,15 +39,4 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do |> cast_data |> validate_data end - - def validate_block_acceptance(cng) do - actor = get_field(cng, :actor) |> User.get_cached_by_ap_id() - - if actor.local || Pleroma.Config.get([:activitypub, :unfollow_blocked], true) do - cng - else - cng - |> add_error(:actor, "Not accepting remote blocks") - end - end end -- cgit v1.2.3 From 7ed229641667f52dd82eb7c388ea28e79e09e507 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 26 Jun 2020 13:04:15 +0200 Subject: Nodeinfo: Add chat information back in. --- lib/pleroma/web/nodeinfo/nodeinfo.ex | 30 +----------------------------- 1 file changed, 1 insertion(+), 29 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/nodeinfo/nodeinfo.ex b/lib/pleroma/web/nodeinfo/nodeinfo.ex index f7ab6d86a..47fa46376 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo.ex @@ -19,35 +19,7 @@ defmodule Pleroma.Web.Nodeinfo.Nodeinfo do |> Enum.map(fn u -> u.ap_id end) federation = InstanceView.federation() - - features = - [ - "pleroma_api", - "mastodon_api", - "mastodon_api_streaming", - "polls", - "pleroma_explicit_addressing", - "shareable_emoji_packs", - "multifetch", - "pleroma:api/v1/notifications:include_types_filter", - if Config.get([:media_proxy, :enabled]) do - "media_proxy" - end, - if Config.get([:gopher, :enabled]) do - "gopher" - end, - if Config.get([:chat, :enabled]) do - "chat" - end, - if Config.get([:instance, :allow_relay]) do - "relay" - end, - if Config.get([:instance, :safe_dm_mentions]) do - "safe_dm_mentions" - end, - "pleroma_emoji_reactions" - ] - |> Enum.filter(& &1) + features = InstanceView.features() %{ version: "2.0", -- cgit v1.2.3 From 4a7a34ae8c2ad12b2b9903c1d70bfe85d10af49e Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 26 Jun 2020 14:47:38 +0200 Subject: Preloading: Return correct data for statusnet stuff. --- lib/pleroma/web/preload/status_net.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/preload/status_net.ex b/lib/pleroma/web/preload/status_net.ex index 367442d5c..810ad512b 100644 --- a/lib/pleroma/web/preload/status_net.ex +++ b/lib/pleroma/web/preload/status_net.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.Preload.Providers.StatusNet do alias Pleroma.Web.Preload.Providers.Provider - alias Pleroma.Web.TwitterAPI.UtilView + alias Pleroma.Web.TwitterAPI.UtilController @behaviour Provider @config_url :"/api/statusnet/config.json" @@ -16,9 +16,10 @@ defmodule Pleroma.Web.Preload.Providers.StatusNet do end defp build_config_tag(acc) do - instance = Pleroma.Config.get(:instance) - info_data = UtilView.status_net_config(instance) + resp = + Plug.Test.conn(:get, @config_url |> to_string()) + |> UtilController.config(nil) - Map.put(acc, @config_url, info_data) + Map.put(acc, @config_url, resp.resp_body) end end -- cgit v1.2.3 From a2002ebb6393d53030d5fc565bae90f3fedd48a8 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 26 Jun 2020 14:48:23 +0200 Subject: Preloading: Fix nodeinfo url. --- lib/pleroma/web/preload/instance.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/preload/instance.ex b/lib/pleroma/web/preload/instance.ex index 0b6fd3313..3b95fe403 100644 --- a/lib/pleroma/web/preload/instance.ex +++ b/lib/pleroma/web/preload/instance.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.Preload.Providers.Instance do @behaviour Provider @instance_url :"/api/v1/instance" @panel_url :"/instance/panel.html" - @nodeinfo_url :"/nodeinfo/2.0" + @nodeinfo_url :"/nodeinfo/2.0.json" @impl Provider def generate_terms(_params) do -- cgit v1.2.3 From f378e93bf4ca4bc9547f242e76e6258e25852972 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 26 Jun 2020 16:15:27 +0200 Subject: AccountController: Return scope in proper format. --- lib/pleroma/web/api_spec/operations/account_operation.ex | 4 ++-- lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 9bde8fc0d..d94dae374 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -446,13 +446,13 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do properties: %{ token_type: %Schema{type: :string}, access_token: %Schema{type: :string}, - scope: %Schema{type: :array, items: %Schema{type: :string}}, + scope: %Schema{type: :string}, created_at: %Schema{type: :integer, format: :"date-time"} }, example: %{ "access_token" => "i9hAVVzGld86Pl5JtLtizKoXVvtTlSCJvwaugCxvZzk", "created_at" => 1_585_918_714, - "scope" => ["read", "write", "follow", "push"], + "scope" => "read write follow push", "token_type" => "Bearer" } } diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 7a88a847c..a87dddddf 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -104,7 +104,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do json(conn, %{ token_type: "Bearer", access_token: token.token, - scope: app.scopes, + scope: app.scopes |> Enum.join(" "), created_at: Token.Utils.format_created_at(token) }) else -- cgit v1.2.3 From a5bbfa21a1fabe97bfff1cc80348d2944319f3ad Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 26 Jun 2020 16:27:39 +0200 Subject: StaticFE: Prioritize json in requests. --- lib/pleroma/plugs/static_fe_plug.ex | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex index 156e6788e..7c69b2dac 100644 --- a/lib/pleroma/plugs/static_fe_plug.ex +++ b/lib/pleroma/plugs/static_fe_plug.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Plugs.StaticFEPlug do def init(options), do: options def call(conn, _) do - if enabled?() and accepts_html?(conn) do + if enabled?() and requires_html?(conn) do conn |> StaticFEController.call(:show) |> halt() @@ -20,10 +20,13 @@ defmodule Pleroma.Plugs.StaticFEPlug do defp enabled?, do: Pleroma.Config.get([:static_fe, :enabled], false) - defp accepts_html?(conn) do + defp requires_html?(conn) do case get_req_header(conn, "accept") do - [accept | _] -> String.contains?(accept, "text/html") - _ -> false + [accept | _] -> + !String.contains?(accept, "json") && String.contains?(accept, "text/html") + + _ -> + false end end end -- cgit v1.2.3 From fd5e797379155e5a85deb88dc79f8fbca483948e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 Jun 2020 11:24:28 -0500 Subject: Simplify notification filtering settings further --- lib/pleroma/notification.ex | 28 +++------------------------- lib/pleroma/user/notification_setting.ex | 8 ++------ lib/pleroma/web/api_spec/schemas/account.ex | 8 ++------ 3 files changed, 7 insertions(+), 37 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 9d09cf082..8a28a1821 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -550,9 +550,7 @@ defmodule Pleroma.Notification do [ :self, :invisible, - :from_followers, - :from_following, - :from_strangers, + :block_from_strangers, :recently_followed ] |> Enum.find(&skip?(&1, activity, user)) @@ -572,35 +570,15 @@ defmodule Pleroma.Notification do end def skip?( - :from_followers, + :block_from_strangers, %Activity{} = activity, - %User{notification_settings: %{from_followers: false}} = user - ) do - actor = activity.data["actor"] - follower = User.get_cached_by_ap_id(actor) - User.following?(follower, user) - end - - def skip?( - :from_strangers, - %Activity{} = activity, - %User{notification_settings: %{from_strangers: false}} = user + %User{notification_settings: %{block_from_strangers: true}} = user ) do actor = activity.data["actor"] follower = User.get_cached_by_ap_id(actor) !User.following?(follower, user) end - def skip?( - :from_following, - %Activity{} = activity, - %User{notification_settings: %{from_following: false}} = user - ) do - actor = activity.data["actor"] - followed = User.get_cached_by_ap_id(actor) - User.following?(user, followed) - end - # To do: consider defining recency in hours and checking FollowingRelationship with a single SQL def skip?(:recently_followed, %Activity{data: %{"type" => "Follow"}} = activity, %User{} = user) do actor = activity.data["actor"] diff --git a/lib/pleroma/user/notification_setting.ex b/lib/pleroma/user/notification_setting.ex index e47ac4cab..ffe9860de 100644 --- a/lib/pleroma/user/notification_setting.ex +++ b/lib/pleroma/user/notification_setting.ex @@ -10,18 +10,14 @@ defmodule Pleroma.User.NotificationSetting do @primary_key false embedded_schema do - field(:from_followers, :boolean, default: true) - field(:from_following, :boolean, default: true) - field(:from_strangers, :boolean, default: true) + field(:block_from_strangers, :boolean, default: false) field(:privacy_option, :boolean, default: false) end def changeset(schema, params) do schema |> cast(prepare_attrs(params), [ - :from_followers, - :from_following, - :from_strangers, + :block_from_strangers, :privacy_option ]) end diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index ed90ef3db..91bb1ba88 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -57,9 +57,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do notification_settings: %Schema{ type: :object, properties: %{ - from_followers: %Schema{type: :boolean}, - from_following: %Schema{type: :boolean}, - from_strangers: %Schema{type: :boolean}, + block_from_strangers: %Schema{type: :boolean}, privacy_option: %Schema{type: :boolean} } }, @@ -122,9 +120,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do "unread_conversation_count" => 0, "tags" => [], "notification_settings" => %{ - "from_followers" => true, - "from_following" => true, - "from_strangers" => true, + "block_from_strangers" => false, "privacy_option" => false }, "relationship" => %{ -- cgit v1.2.3 From 69848d5c97c9e5d4c14fb5613eb174cb81d5026d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 Jun 2020 12:45:46 -0500 Subject: Rename notification "privacy_option" setting --- lib/mix/tasks/pleroma/notification_settings.ex | 18 +++++++++--------- lib/pleroma/user/notification_setting.ex | 4 ++-- lib/pleroma/web/api_spec/schemas/account.ex | 4 ++-- lib/pleroma/web/push/impl.ex | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/notification_settings.ex b/lib/mix/tasks/pleroma/notification_settings.ex index 7d65f0587..00f5ba7bf 100644 --- a/lib/mix/tasks/pleroma/notification_settings.ex +++ b/lib/mix/tasks/pleroma/notification_settings.ex @@ -3,8 +3,8 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do @moduledoc """ Example: - > mix pleroma.notification_settings --privacy-option=false --nickname-users="parallel588" # set false only for parallel588 user - > mix pleroma.notification_settings --privacy-option=true # set true for all users + > mix pleroma.notification_settings --hide-notification-contents=false --nickname-users="parallel588" # set false only for parallel588 user + > mix pleroma.notification_settings --hide-notification-contents=true # set true for all users """ @@ -19,16 +19,16 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do OptionParser.parse( args, strict: [ - privacy_option: :boolean, + hide_notification_contents: :boolean, email_users: :string, nickname_users: :string ] ) - privacy_option = Keyword.get(options, :privacy_option) + hide_notification_contents = Keyword.get(options, :hide_notification_contents) - if not is_nil(privacy_option) do - privacy_option + if not is_nil(hide_notification_contents) do + hide_notification_contents |> build_query(options) |> Pleroma.Repo.update_all([]) end @@ -36,15 +36,15 @@ defmodule Mix.Tasks.Pleroma.NotificationSettings do shell_info("Done") end - defp build_query(privacy_option, options) do + defp build_query(hide_notification_contents, options) do query = from(u in Pleroma.User, update: [ set: [ notification_settings: fragment( - "jsonb_set(notification_settings, '{privacy_option}', ?)", - ^privacy_option + "jsonb_set(notification_settings, '{hide_notification_contents}', ?)", + ^hide_notification_contents ) ] ] diff --git a/lib/pleroma/user/notification_setting.ex b/lib/pleroma/user/notification_setting.ex index ffe9860de..7d9e8a000 100644 --- a/lib/pleroma/user/notification_setting.ex +++ b/lib/pleroma/user/notification_setting.ex @@ -11,14 +11,14 @@ defmodule Pleroma.User.NotificationSetting do embedded_schema do field(:block_from_strangers, :boolean, default: false) - field(:privacy_option, :boolean, default: false) + field(:hide_notification_contents, :boolean, default: false) end def changeset(schema, params) do schema |> cast(prepare_attrs(params), [ :block_from_strangers, - :privacy_option + :hide_notification_contents ]) end diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 91bb1ba88..71d402b18 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do type: :object, properties: %{ block_from_strangers: %Schema{type: :boolean}, - privacy_option: %Schema{type: :boolean} + hide_notification_contents: %Schema{type: :boolean} } }, relationship: AccountRelationship, @@ -121,7 +121,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do "tags" => [], "notification_settings" => %{ "block_from_strangers" => false, - "privacy_option" => false + "hide_notification_contents" => false }, "relationship" => %{ "blocked_by" => false, diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index cdb827e76..16368485e 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -104,7 +104,7 @@ defmodule Pleroma.Web.Push.Impl do def build_content( %{ - user: %{notification_settings: %{privacy_option: true}} + user: %{notification_settings: %{hide_notification_contents: true}} } = notification, _actor, _object, -- cgit v1.2.3 From 244655e884130df6dccabc0d2d78d33857809a36 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jun 2020 07:16:24 +0200 Subject: MastoAPI: Show source field when deleting --- lib/pleroma/web/api_spec/operations/status_operation.ex | 2 +- lib/pleroma/web/api_spec/schemas/status.ex | 5 +++++ .../web/mastodon_api/controllers/status_controller.ex | 15 +++++++++++---- lib/pleroma/web/mastodon_api/views/status_view.ex | 1 + 4 files changed, 18 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 0b7fad793..5bd4619d5 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -84,7 +84,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do operationId: "StatusController.delete", parameters: [id_param()], responses: %{ - 200 => empty_object_response(), + 200 => status_response(), 403 => Operation.response("Forbidden", "application/json", ApiError), 404 => Operation.response("Not Found", "application/json", ApiError) } diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 8b87cb25b..a38b5b40f 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -62,6 +62,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do } }, content: %Schema{type: :string, format: :html, description: "HTML-encoded status content"}, + text: %Schema{ + type: :string, + description: "Original unformatted content in plain text", + nullable: true + }, created_at: %Schema{ type: :string, format: "date-time", diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 468b44b67..3f4c53437 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -200,11 +200,18 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do @doc "DELETE /api/v1/statuses/:id" def delete(%{assigns: %{user: user}} = conn, %{id: id}) do - with {:ok, %Activity{}} <- CommonAPI.delete(id, user) do - json(conn, %{}) + with %Activity{} = activity <- Activity.get_by_id_with_object(id), + render <- + try_render(conn, "show.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_source: true + ), + {:ok, %Activity{}} <- CommonAPI.delete(id, user) do + render else - {:error, :not_found} = e -> e - _e -> render_error(conn, :forbidden, "Can't delete this post") + _e -> {:error, :not_found} end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2c49bedb3..4df47f584 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -333,6 +333,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblog: nil, card: card, content: content_html, + text: opts[:with_source] && object.data["source"], created_at: created_at, reblogs_count: announcement_count, replies_count: object.data["repliesCount"] || 0, -- cgit v1.2.3 From 1566543bec70e6497df77ed83bf4d3cc39c116eb Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jun 2020 20:10:47 +0200 Subject: object/fetcher: Pass full Transmogrifier error --- lib/pleroma/object/fetcher.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 263ded5dd..3e2949ee2 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -83,8 +83,8 @@ defmodule Pleroma.Object.Fetcher do {:transmogrifier, {:error, {:reject, nil}}} -> {:reject, nil} - {:transmogrifier, _} -> - {:error, "Transmogrifier failure."} + {:transmogrifier, _} = e -> + {:error, e} {:object, data, nil} -> reinject_object(%Object{}, data) -- cgit v1.2.3 From 9f51b03eed85d4a3ea24e1d449fcb4969f299096 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 29 Jun 2020 03:31:33 +0200 Subject: ApiSpec.Schemas.Account: import description from differences_in_mastoapi_responses --- lib/pleroma/web/api_spec/schemas/account.ex | 83 +++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index d54e2158d..84f18f1b6 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -40,20 +40,53 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do pleroma: %Schema{ type: :object, properties: %{ - allow_following_move: %Schema{type: :boolean}, - background_image: %Schema{type: :string, nullable: true}, + allow_following_move: %Schema{ + type: :boolean, + description: "whether the user allows automatically follow moved following accounts" + }, + background_image: %Schema{type: :string, nullable: true, format: :uri}, chat_token: %Schema{type: :string}, - confirmation_pending: %Schema{type: :boolean}, + confirmation_pending: %Schema{ + type: :boolean, + description: + "whether the user account is waiting on email confirmation to be activated" + }, hide_favorites: %Schema{type: :boolean}, - hide_followers_count: %Schema{type: :boolean}, - hide_followers: %Schema{type: :boolean}, - hide_follows_count: %Schema{type: :boolean}, - hide_follows: %Schema{type: :boolean}, - is_admin: %Schema{type: :boolean}, - is_moderator: %Schema{type: :boolean}, + hide_followers_count: %Schema{ + type: :boolean, + description: "whether the user has follower stat hiding enabled" + }, + hide_followers: %Schema{ + type: :boolean, + description: "whether the user has follower hiding enabled" + }, + hide_follows_count: %Schema{ + type: :boolean, + description: "whether the user has follow stat hiding enabled" + }, + hide_follows: %Schema{ + type: :boolean, + description: "whether the user has follow hiding enabled" + }, + is_admin: %Schema{ + type: :boolean, + description: "whether the user is an admin of the local instance" + }, + is_moderator: %Schema{ + type: :boolean, + description: "whether the user is a moderator of the local instance" + }, skip_thread_containment: %Schema{type: :boolean}, - tags: %Schema{type: :array, items: %Schema{type: :string}}, - unread_conversation_count: %Schema{type: :integer}, + tags: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: + "List of tags being used for things like extra roles or moderation(ie. marking all media as nsfw all)." + }, + unread_conversation_count: %Schema{ + type: :integer, + description: "The count of unread conversations. Only returned to the account owner." + }, notification_settings: %Schema{ type: :object, properties: %{ @@ -66,7 +99,9 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do }, relationship: AccountRelationship, settings_store: %Schema{ - type: :object + type: :object, + description: + "A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials`" } } }, @@ -74,16 +109,32 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do type: :object, properties: %{ fields: %Schema{type: :array, items: AccountField}, - note: %Schema{type: :string}, + note: %Schema{ + type: :string, + description: + "Plaintext version of the bio without formatting applied by the backend, used for editing the bio." + }, privacy: VisibilityScope, sensitive: %Schema{type: :boolean}, pleroma: %Schema{ type: :object, properties: %{ actor_type: ActorType, - discoverable: %Schema{type: :boolean}, - no_rich_text: %Schema{type: :boolean}, - show_role: %Schema{type: :boolean} + discoverable: %Schema{ + type: :boolean, + description: + "whether the user allows discovery of the account in search results and other services." + }, + no_rich_text: %Schema{ + type: :boolean, + description: + "whether the HTML tags for rich-text formatting are stripped from all statuses requested from the API." + }, + show_role: %Schema{ + type: :boolean, + description: + "whether the user wants their role (e.g admin, moderator) to be shown" + } } } } -- cgit v1.2.3 From a19f8778afddb7f504b08cedde752e37da52dc96 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 29 Jun 2020 11:06:20 +0200 Subject: User preloader: Put user info at correct key --- lib/pleroma/web/preload/user.ex | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/preload/user.ex b/lib/pleroma/web/preload/user.ex index 3a244845b..7fef0a4ac 100644 --- a/lib/pleroma/web/preload/user.ex +++ b/lib/pleroma/web/preload/user.ex @@ -3,11 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Preload.Providers.User do + alias Pleroma.User alias Pleroma.Web.MastodonAPI.AccountView alias Pleroma.Web.Preload.Providers.Provider @behaviour Provider - @account_url :"/api/v1/accounts" + @account_url_base :"/api/v1/accounts" @impl Provider def generate_terms(%{user: user}) do @@ -16,10 +17,10 @@ defmodule Pleroma.Web.Preload.Providers.User do def generate_terms(_params), do: %{} - def build_accounts_tag(acc, nil), do: acc - - def build_accounts_tag(acc, user) do + def build_accounts_tag(acc, %User{} = user) do account_data = AccountView.render("show.json", %{user: user, for: user}) - Map.put(acc, @account_url, account_data) + Map.put(acc, :"#{@account_url_base}/#{user.id}", account_data) end + + def build_accounts_tag(acc, _), do: acc end -- cgit v1.2.3 From 8630a6c7f52a68ab32025b1c80a6398599908c68 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 29 Jun 2020 11:41:00 +0200 Subject: Preloaders: Use strings as keys. --- lib/pleroma/web/preload/instance.ex | 6 +++--- lib/pleroma/web/preload/status_net.ex | 2 +- lib/pleroma/web/preload/timelines.ex | 2 +- lib/pleroma/web/preload/user.ex | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/preload/instance.ex b/lib/pleroma/web/preload/instance.ex index 3b95fe403..b34d7cf37 100644 --- a/lib/pleroma/web/preload/instance.ex +++ b/lib/pleroma/web/preload/instance.ex @@ -8,9 +8,9 @@ defmodule Pleroma.Web.Preload.Providers.Instance do alias Pleroma.Web.Preload.Providers.Provider @behaviour Provider - @instance_url :"/api/v1/instance" - @panel_url :"/instance/panel.html" - @nodeinfo_url :"/nodeinfo/2.0.json" + @instance_url "/api/v1/instance" + @panel_url "/instance/panel.html" + @nodeinfo_url "/nodeinfo/2.0.json" @impl Provider def generate_terms(_params) do diff --git a/lib/pleroma/web/preload/status_net.ex b/lib/pleroma/web/preload/status_net.ex index 810ad512b..9b62f87a2 100644 --- a/lib/pleroma/web/preload/status_net.ex +++ b/lib/pleroma/web/preload/status_net.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.Preload.Providers.StatusNet do alias Pleroma.Web.TwitterAPI.UtilController @behaviour Provider - @config_url :"/api/statusnet/config.json" + @config_url "/api/statusnet/config.json" @impl Provider def generate_terms(_params) do diff --git a/lib/pleroma/web/preload/timelines.ex b/lib/pleroma/web/preload/timelines.ex index e531b8960..57de04051 100644 --- a/lib/pleroma/web/preload/timelines.ex +++ b/lib/pleroma/web/preload/timelines.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Web.Preload.Providers.Timelines do alias Pleroma.Web.Preload.Providers.Provider @behaviour Provider - @public_url :"/api/v1/timelines/public" + @public_url "/api/v1/timelines/public" @impl Provider def generate_terms(params) do diff --git a/lib/pleroma/web/preload/user.ex b/lib/pleroma/web/preload/user.ex index 7fef0a4ac..b3d2e9b8d 100644 --- a/lib/pleroma/web/preload/user.ex +++ b/lib/pleroma/web/preload/user.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Web.Preload.Providers.User do alias Pleroma.Web.Preload.Providers.Provider @behaviour Provider - @account_url_base :"/api/v1/accounts" + @account_url_base "/api/v1/accounts" @impl Provider def generate_terms(%{user: user}) do @@ -19,7 +19,7 @@ defmodule Pleroma.Web.Preload.Providers.User do def build_accounts_tag(acc, %User{} = user) do account_data = AccountView.render("show.json", %{user: user, for: user}) - Map.put(acc, :"#{@account_url_base}/#{user.id}", account_data) + Map.put(acc, "#{@account_url_base}/#{user.id}", account_data) end def build_accounts_tag(acc, _), do: acc -- cgit v1.2.3 From bb168ed94a6b4d02879472e30149a494d7b7ebb5 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 29 Jun 2020 13:39:09 +0200 Subject: OAuth: Extract view-type functions to a view. --- lib/pleroma/web/oauth/mfa_controller.ex | 3 ++- lib/pleroma/web/oauth/mfa_view.ex | 9 +++++++ lib/pleroma/web/oauth/oauth_controller.ex | 18 +++++++------- lib/pleroma/web/oauth/oauth_view.ex | 22 +++++++++++++++++ lib/pleroma/web/oauth/token/response.ex | 39 ------------------------------- 5 files changed, 41 insertions(+), 50 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/oauth/mfa_controller.ex b/lib/pleroma/web/oauth/mfa_controller.ex index 53e19f82e..f102c93e7 100644 --- a/lib/pleroma/web/oauth/mfa_controller.ex +++ b/lib/pleroma/web/oauth/mfa_controller.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.OAuth.MFAController do alias Pleroma.Web.Auth.TOTPAuthenticator alias Pleroma.Web.OAuth.MFAView, as: View alias Pleroma.Web.OAuth.OAuthController + alias Pleroma.Web.OAuth.OAuthView alias Pleroma.Web.OAuth.Token plug(:fetch_session when action in [:show, :verify]) @@ -74,7 +75,7 @@ defmodule Pleroma.Web.OAuth.MFAController do {:ok, %{user: user, authorization: auth}} <- MFA.Token.validate(mfa_token), {:ok, _} <- validates_challenge(user, params), {:ok, token} <- Token.exchange_token(app, auth) do - json(conn, Token.Response.build(user, token)) + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else _error -> conn diff --git a/lib/pleroma/web/oauth/mfa_view.ex b/lib/pleroma/web/oauth/mfa_view.ex index 41d5578dc..5d87db268 100644 --- a/lib/pleroma/web/oauth/mfa_view.ex +++ b/lib/pleroma/web/oauth/mfa_view.ex @@ -5,4 +5,13 @@ defmodule Pleroma.Web.OAuth.MFAView do use Pleroma.Web, :view import Phoenix.HTML.Form + alias Pleroma.MFA + + def render("mfa_response.json", %{token: token, user: user}) do + %{ + error: "mfa_required", + mfa_token: token.token, + supported_challenge_types: MFA.supported_methods(user) + } + end end diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index c557778ca..3da104933 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do use Pleroma.Web, :controller alias Pleroma.Helpers.UriHelper - alias Pleroma.Maps alias Pleroma.MFA + alias Pleroma.Maps alias Pleroma.Plugs.RateLimiter alias Pleroma.Registration alias Pleroma.Repo @@ -17,6 +17,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.MFAController + alias Pleroma.Web.OAuth.OAuthView + alias Pleroma.Web.OAuth.MFAView alias Pleroma.Web.OAuth.Scopes alias Pleroma.Web.OAuth.Token alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken @@ -233,9 +235,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do with {:ok, app} <- Token.Utils.fetch_app(conn), {:ok, %{user: user} = token} <- Token.get_by_refresh_token(app, token), {:ok, token} <- RefreshToken.grant(token) do - response_attrs = %{created_at: Token.Utils.format_created_at(token)} - - json(conn, Token.Response.build(user, token, response_attrs)) + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else _error -> render_invalid_credentials_error(conn) end @@ -247,9 +247,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do {:ok, auth} <- Authorization.get_by_token(app, fixed_token), %User{} = user <- User.get_cached_by_id(auth.user_id), {:ok, token} <- Token.exchange_token(app, auth) do - response_attrs = %{created_at: Token.Utils.format_created_at(token)} - - json(conn, Token.Response.build(user, token, response_attrs)) + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else error -> handle_token_exchange_error(conn, error) @@ -267,7 +265,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do {:ok, auth} <- Authorization.create_authorization(app, user, scopes), {:mfa_required, _, _, false} <- {:mfa_required, user, auth, MFA.require?(user)}, {:ok, token} <- Token.exchange_token(app, auth) do - json(conn, Token.Response.build(user, token)) + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else error -> handle_token_exchange_error(conn, error) @@ -290,7 +288,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do with {:ok, app} <- Token.Utils.fetch_app(conn), {:ok, auth} <- Authorization.create_authorization(app, %User{}), {:ok, token} <- Token.exchange_token(app, auth) do - json(conn, Token.Response.build_for_client_credentials(token)) + json(conn, OAuthView.render("token.json", %{token: token})) else _error -> handle_token_exchange_error(conn, :invalid_credentails) @@ -548,7 +546,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do defp build_and_response_mfa_token(user, auth) do with {:ok, token} <- MFA.Token.create_token(user, auth) do - Token.Response.build_for_mfa_token(user, token) + MFAView.render("mfa_response.json", %{token: token, user: user}) end end diff --git a/lib/pleroma/web/oauth/oauth_view.ex b/lib/pleroma/web/oauth/oauth_view.ex index 94ddaf913..f55247ebd 100644 --- a/lib/pleroma/web/oauth/oauth_view.ex +++ b/lib/pleroma/web/oauth/oauth_view.ex @@ -5,4 +5,26 @@ defmodule Pleroma.Web.OAuth.OAuthView do use Pleroma.Web, :view import Phoenix.HTML.Form + + alias Pleroma.Web.OAuth.Token.Utils + + def render("token.json", %{token: token} = opts) do + response = %{ + token_type: "Bearer", + access_token: token.token, + refresh_token: token.refresh_token, + expires_in: expires_in(), + scope: Enum.join(token.scopes, " "), + created_at: Utils.format_created_at(token) + } + + if user = opts[:user] do + response + |> Map.put(:me, user.ap_id) + else + response + end + end + + defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600) end diff --git a/lib/pleroma/web/oauth/token/response.ex b/lib/pleroma/web/oauth/token/response.ex index 0e72c31e9..a12a6865c 100644 --- a/lib/pleroma/web/oauth/token/response.ex +++ b/lib/pleroma/web/oauth/token/response.ex @@ -3,43 +3,4 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.OAuth.Token.Response do - @moduledoc false - - alias Pleroma.MFA - alias Pleroma.User - alias Pleroma.Web.OAuth.Token.Utils - - @doc false - def build(%User{} = user, token, opts \\ %{}) do - %{ - token_type: "Bearer", - access_token: token.token, - refresh_token: token.refresh_token, - expires_in: expires_in(), - scope: Enum.join(token.scopes, " "), - me: user.ap_id - } - |> Map.merge(opts) - end - - def build_for_client_credentials(token) do - %{ - token_type: "Bearer", - access_token: token.token, - refresh_token: token.refresh_token, - created_at: Utils.format_created_at(token), - expires_in: expires_in(), - scope: Enum.join(token.scopes, " ") - } - end - - def build_for_mfa_token(user, mfa_token) do - %{ - error: "mfa_required", - mfa_token: mfa_token.token, - supported_challenge_types: MFA.supported_methods(user) - } - end - - defp expires_in, do: Pleroma.Config.get([:oauth2, :token_expires_in], 600) end -- cgit v1.2.3 From e374872fe7d10aa659723ee31003f3e9188edfdd Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 29 Jun 2020 13:49:48 +0200 Subject: AccountOperation: Correctly describe create response. --- lib/pleroma/web/api_spec/operations/account_operation.ex | 11 +++++++++-- .../web/mastodon_api/controllers/account_controller.ex | 8 ++------ 2 files changed, 11 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index d94dae374..f3ffa1ad4 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -438,6 +438,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end + # TODO: This is actually a token respone, but there's no oauth operation file yet. defp create_response do %Schema{ title: "AccountCreateResponse", @@ -446,14 +447,20 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do properties: %{ token_type: %Schema{type: :string}, access_token: %Schema{type: :string}, + refresh_token: %Schema{type: :string}, scope: %Schema{type: :string}, - created_at: %Schema{type: :integer, format: :"date-time"} + created_at: %Schema{type: :integer, format: :"date-time"}, + me: %Schema{type: :string}, + expires_in: %Schema{type: :integer} }, example: %{ + "token_type" => "Bearer", "access_token" => "i9hAVVzGld86Pl5JtLtizKoXVvtTlSCJvwaugCxvZzk", + "refresh_token" => "i9hAVVzGld86Pl5JtLtizKoXVvtTlSCJvwaugCxvZzz", "created_at" => 1_585_918_714, + "expires_in" => 600, "scope" => "read write follow push", - "token_type" => "Bearer" + "me" => "https://gensokyo.2hu/users/raymoo" } } end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index a87dddddf..a143675ec 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -28,6 +28,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do alias Pleroma.Web.MastodonAPI.MastodonAPIController alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.OAuth.Token + alias Pleroma.Web.OAuth.OAuthView alias Pleroma.Web.TwitterAPI.TwitterAPI plug(Pleroma.Web.ApiSpec.CastAndValidate) @@ -101,12 +102,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do :ok <- TwitterAPI.validate_captcha(app, params), {:ok, user} <- TwitterAPI.register_user(params, need_confirmation: true), {:ok, token} <- Token.create_token(app, user, %{scopes: app.scopes}) do - json(conn, %{ - token_type: "Bearer", - access_token: token.token, - scope: app.scopes |> Enum.join(" "), - created_at: Token.Utils.format_created_at(token) - }) + json(conn, OAuthView.render("token.json", %{user: user, token: token})) else {:error, error} -> json_response(conn, :bad_request, %{error: error}) end -- cgit v1.2.3 From f308196b7528fab92b3cfba12ea71c464e2f9ab0 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 29 Jun 2020 13:52:50 +0200 Subject: Token Response: Remove empty file. --- lib/pleroma/web/oauth/token/response.ex | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 lib/pleroma/web/oauth/token/response.ex (limited to 'lib') diff --git a/lib/pleroma/web/oauth/token/response.ex b/lib/pleroma/web/oauth/token/response.ex deleted file mode 100644 index a12a6865c..000000000 --- a/lib/pleroma/web/oauth/token/response.ex +++ /dev/null @@ -1,6 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.OAuth.Token.Response do -end -- cgit v1.2.3 From 59540131c189afb10faf98d1bfeccf8f94985a90 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 29 Jun 2020 14:09:03 +0200 Subject: Credo fixes. --- lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 2 +- lib/pleroma/web/oauth/oauth_controller.ex | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index a143675ec..2942ed336 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -27,8 +27,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do alias Pleroma.Web.MastodonAPI.MastodonAPI alias Pleroma.Web.MastodonAPI.MastodonAPIController alias Pleroma.Web.MastodonAPI.StatusView - alias Pleroma.Web.OAuth.Token alias Pleroma.Web.OAuth.OAuthView + alias Pleroma.Web.OAuth.Token alias Pleroma.Web.TwitterAPI.TwitterAPI plug(Pleroma.Web.ApiSpec.CastAndValidate) diff --git a/lib/pleroma/web/oauth/oauth_controller.ex b/lib/pleroma/web/oauth/oauth_controller.ex index 3da104933..7683589cf 100644 --- a/lib/pleroma/web/oauth/oauth_controller.ex +++ b/lib/pleroma/web/oauth/oauth_controller.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do use Pleroma.Web, :controller alias Pleroma.Helpers.UriHelper - alias Pleroma.MFA alias Pleroma.Maps + alias Pleroma.MFA alias Pleroma.Plugs.RateLimiter alias Pleroma.Registration alias Pleroma.Repo @@ -17,8 +17,8 @@ defmodule Pleroma.Web.OAuth.OAuthController do alias Pleroma.Web.OAuth.App alias Pleroma.Web.OAuth.Authorization alias Pleroma.Web.OAuth.MFAController - alias Pleroma.Web.OAuth.OAuthView alias Pleroma.Web.OAuth.MFAView + alias Pleroma.Web.OAuth.OAuthView alias Pleroma.Web.OAuth.Scopes alias Pleroma.Web.OAuth.Token alias Pleroma.Web.OAuth.Token.Strategy.RefreshToken -- cgit v1.2.3 From 6512ef6879a5f857f02479da1bad7242e916d918 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 29 Jun 2020 15:25:57 +0300 Subject: excluding attachment links from RichMedia --- lib/pleroma/html.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index d78c5f202..dc1b9b840 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -109,7 +109,7 @@ defmodule Pleroma.HTML do result = content |> Floki.parse_fragment!() - |> Floki.filter_out("a.mention,a.hashtag,a[rel~=\"tag\"]") + |> Floki.filter_out("a.mention,a.hashtag,a.attachment,a[rel~=\"tag\"]") |> Floki.attribute("a", "href") |> Enum.at(0) -- cgit v1.2.3 From 8693e01799308295011a39c8fab71f8a49d3a9bd Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jun 2020 16:29:51 +0400 Subject: Fix warning --- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 4ec523a4e..76f4bb8f4 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -14,7 +14,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.User alias Pleroma.Web.CommonAPI - alias Pleroma.Web.TwitterAPI.UtilView alias Pleroma.Web.WebFinger plug(Pleroma.Web.FederatingPlug when action == :remote_subscribe) -- cgit v1.2.3 From 67d92ac7b7b977debac8f8e580db1f0e1ef3ed52 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jun 2020 17:00:37 +0400 Subject: Remove `/statusnet/config` --- lib/pleroma/web/router.ex | 3 --- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 12 ------------ 2 files changed, 15 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 930bf7314..9eee74e6c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -517,9 +517,6 @@ defmodule Pleroma.Web.Router do pipe_through(:config) get("/pleroma/frontend_configurations", TwitterAPI.UtilController, :frontend_configurations) - - # Deprecated - get("/statusnet/config", TwitterAPI.UtilController, :config) end scope "/api", Pleroma.Web do diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 76f4bb8f4..8314e75b4 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -81,18 +81,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end end - # Deprecated in favor of `/nodeinfo` - # https://git.pleroma.social/pleroma/pleroma/-/merge_requests/2327 - # https://git.pleroma.social/pleroma/pleroma-fe/-/merge_requests/1084 - def config(conn, _params) do - json(conn, %{ - site: %{ - textlimit: to_string(Config.get([:instance, :limit])), - vapidPublicKey: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) - } - }) - end - def frontend_configurations(conn, _params) do config = Pleroma.Config.get(:frontend_configurations, %{}) -- cgit v1.2.3 From 979f02ec947443835f480d13bd1dbcf521743a71 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Mon, 29 Jun 2020 17:33:00 +0400 Subject: Fix CastAndValidate plug --- lib/pleroma/web/api_spec/cast_and_validate.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index bd9026237..fbfc27d6f 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do |> List.first() _ -> - nil + "application/json" end private_data = Map.put(private_data, :operation_id, operation_id) -- cgit v1.2.3 From 90083a754dc0bfe0c8a04fbaa3e78f68a848035e Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 29 Jun 2020 17:48:18 +0200 Subject: Notifications: Never return `nil` in the notification list. --- lib/pleroma/notification.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 9ee9606be..58dcf880a 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -367,6 +367,7 @@ defmodule Pleroma.Notification do do_send = do_send && user in enabled_receivers create_notification(activity, user, do_send) end) + |> Enum.filter(& &1) {:ok, notifications} end -- cgit v1.2.3 From 09c5991f82e91878a940f5957ac993e1fca72545 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 29 Jun 2020 16:04:14 +0000 Subject: Apply suggestion to lib/pleroma/notification.ex --- lib/pleroma/notification.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 58dcf880a..2ef1a80c5 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -367,7 +367,7 @@ defmodule Pleroma.Notification do do_send = do_send && user in enabled_receivers create_notification(activity, user, do_send) end) - |> Enum.filter(& &1) + |> Enum.reject(&is_nil/1) {:ok, notifications} end -- cgit v1.2.3 From 2382a2a1511e1042d960946aacfde7a49fac9dd0 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 30 Jun 2020 11:35:54 +0200 Subject: Preload: Load the correct instance panel --- lib/pleroma/web/preload/instance.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/preload/instance.ex b/lib/pleroma/web/preload/instance.ex index 0b6fd3313..5c6e33e47 100644 --- a/lib/pleroma/web/preload/instance.ex +++ b/lib/pleroma/web/preload/instance.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.Preload.Providers.Instance do alias Pleroma.Web.MastodonAPI.InstanceView alias Pleroma.Web.Nodeinfo.Nodeinfo alias Pleroma.Web.Preload.Providers.Provider + alias Pleroma.Plugs.InstanceStatic @behaviour Provider @instance_url :"/api/v1/instance" @@ -27,7 +28,7 @@ defmodule Pleroma.Web.Preload.Providers.Instance do end defp build_panel_tag(acc) do - instance_path = Path.join(:code.priv_dir(:pleroma), "static/instance/panel.html") + instance_path = InstanceStatic.file_path(@panel_url |> to_string()) if File.exists?(instance_path) do panel_data = File.read!(instance_path) -- cgit v1.2.3 From 8b7055e25e76565cd3376c0b5dda5e54d24881f0 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 30 Jun 2020 11:55:58 +0200 Subject: Credo fixes --- lib/pleroma/web/preload/instance.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/preload/instance.ex b/lib/pleroma/web/preload/instance.ex index 3d16f290b..50d1f3382 100644 --- a/lib/pleroma/web/preload/instance.ex +++ b/lib/pleroma/web/preload/instance.ex @@ -3,10 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Preload.Providers.Instance do + alias Pleroma.Plugs.InstanceStatic alias Pleroma.Web.MastodonAPI.InstanceView alias Pleroma.Web.Nodeinfo.Nodeinfo alias Pleroma.Web.Preload.Providers.Provider - alias Pleroma.Plugs.InstanceStatic @behaviour Provider @instance_url "/api/v1/instance" -- cgit v1.2.3 From d69af7f74290a67c9201782b7d4bafa29b6e0bd8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 30 Jun 2020 11:50:53 -0500 Subject: Rename user.settings column This is used exclusively by MastoFE/GlitchFE now --- lib/pleroma/user.ex | 6 +++--- lib/pleroma/web/views/masto_fe_view.ex | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9d5c61e79..8a54546d6 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -115,7 +115,7 @@ defmodule Pleroma.User do field(:is_moderator, :boolean, default: false) field(:is_admin, :boolean, default: false) field(:show_role, :boolean, default: true) - field(:settings, :map, default: nil) + field(:mastofe_settings, :map, default: nil) field(:uri, ObjectValidators.Uri, default: nil) field(:hide_followers_count, :boolean, default: false) field(:hide_follows_count, :boolean, default: false) @@ -2118,8 +2118,8 @@ defmodule Pleroma.User do def mastodon_settings_update(user, settings) do user - |> cast(%{settings: settings}, [:settings]) - |> validate_required([:settings]) + |> cast(%{mastofe_settings: settings}, [:mastofe_settings]) + |> validate_required([:mastofe_settings]) |> update_and_set_cache() end diff --git a/lib/pleroma/web/views/masto_fe_view.ex b/lib/pleroma/web/views/masto_fe_view.ex index c3096006e..f739dacb6 100644 --- a/lib/pleroma/web/views/masto_fe_view.ex +++ b/lib/pleroma/web/views/masto_fe_view.ex @@ -86,7 +86,7 @@ defmodule Pleroma.Web.MastoFEView do "video\/mp4" ] }, - settings: user.settings || @default_settings, + settings: user.mastofe_settings || @default_settings, push_subscription: nil, accounts: %{user.id => render(AccountView, "show.json", user: user, for: user)}, custom_emojis: render(CustomEmojiView, "index.json", custom_emojis: custom_emojis), -- cgit v1.2.3 From 3d2989278c2f97fb5247d0b58b99b77f400f3185 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 30 Jun 2020 21:26:39 +0300 Subject: [#1892] Excluded bot actors (applications, services) from search results. --- lib/pleroma/user/search.ex | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index cec59c372..0293c6ae7 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -52,6 +52,7 @@ defmodule Pleroma.User.Search do |> base_query(following) |> filter_blocked_user(for_user) |> filter_invisible_users() + |> filter_bots() |> filter_blocked_domains(for_user) |> fts_search(query_string) |> trigram_rank(query_string) @@ -109,6 +110,10 @@ defmodule Pleroma.User.Search do from(q in query, where: q.invisible == false) end + defp filter_bots(query) do + from(q in query, where: q.actor_type not in ["Application", "Service"]) + end + defp filter_blocked_user(query, %User{} = blocker) do query |> join(:left, [u], b in Pleroma.UserRelationship, -- cgit v1.2.3 From 5a8e0208b1cfb1995353b83338f20dc5cca195e1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 30 Jun 2020 15:25:10 -0500 Subject: Add fields limits to instance metadata, add tests --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 118678962..347480d49 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -38,7 +38,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do metadata: %{ account_activation_required: Keyword.get(instance, :account_activation_required), features: features(), - federation: federation() + federation: federation(), + fields_limits: fields_limits() }, vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) } @@ -89,4 +90,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end |> Map.put(:enabled, Config.get([:instance, :federating])) end + + def fields_limits do + %{ + maxFields: Config.get([:instance, :max_account_fields]), + maxRemoteFields: Config.get([:instance, :max_remote_account_fields]), + nameLength: Config.get([:instance, :account_field_name_length]), + valueLength: Config.get([:instance, :account_field_value_length]) + } + end end -- cgit v1.2.3 From ce92e6e5ce24a68bedd744c01cc1a99f01c4fa91 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 1 Jul 2020 11:48:51 +0200 Subject: Fetcher: Work when we can't get the OP. --- lib/pleroma/web/activity_pub/transmogrifier.ex | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 278fbbeab..bc6fc4bd8 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -446,12 +446,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do when objtype in ["Article", "Event", "Note", "Video", "Page", "Question", "Answer", "Audio"] do actor = Containment.get_actor(data) - data = - Map.put(data, "actor", actor) - |> fix_addressing - with nil <- Activity.get_create_by_object_ap_id(object["id"]), - {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(data["actor"]) do + {:ok, %User{} = user} <- User.get_or_fetch_by_ap_id(actor), + data <- Map.put(data, "actor", actor) |> fix_addressing() do object = fix_object(object, options) params = %{ -- cgit v1.2.3 From fedfe8f7d6f78d77e9cbaf70fa8a9e8df38463f7 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 1 Jul 2020 12:26:07 +0200 Subject: ActivityPub: Handle clashing nicknames for the same ap id If we get a new user (identified by ap_id) that would have the same nickname as an existing user, give the existing user a nickname that is prepended with the user id, as this will never clash. This can happen when a user switches server software and that soft- ware generates ap ids in a different way. --- lib/pleroma/web/activity_pub/activity_pub.ex | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 05bd824f5..94117202c 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1371,6 +1371,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end + def maybe_handle_clashing_nickname(nickname) do + with %User{} = old_user <- User.get_by_nickname(nickname) do + Logger.info("Found an old user for #{nickname}, ap id is #{old_user.ap_id}, renaming.") + + old_user + |> User.remote_user_changeset(%{nickname: "#{old_user.id}.#{old_user.nickname}"}) + |> User.update_and_set_cache() + end + end + def make_user_from_ap_id(ap_id) do user = User.get_cached_by_ap_id(ap_id) @@ -1383,6 +1393,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> User.remote_user_changeset(data) |> User.update_and_set_cache() else + maybe_handle_clashing_nickname(data[:nickname]) + data |> User.remote_user_changeset() |> Repo.insert() -- cgit v1.2.3 From 61fe94d698a6f73e7a3f6224ed4be93b30ba0e54 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 2 Jul 2020 09:33:50 +0200 Subject: SideEffects: Refactor. --- lib/pleroma/web/activity_pub/side_effects.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 5cc2eb378..c84af68f4 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Activity.Ir.Topics alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Push @@ -97,7 +98,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if !User.is_internal_user?(user) do Notification.create_notifications(object) - ActivityPub.stream_out(object) + + object + |> Topics.get_activity_topics() + |> Streamer.stream(object) end {:ok, object, meta} -- cgit v1.2.3 From 311b7c19d0a34654b785116fe22823132d5a9284 Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 2 Jul 2020 09:50:26 +0200 Subject: Streamer: Align announce streaming with polling. --- lib/pleroma/web/streamer/streamer.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index d1d2c9b9c..73ee3e1e1 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -116,6 +116,7 @@ defmodule Pleroma.Web.Streamer do true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(item.actor not in &1)), true <- item.data["type"] != "Announce" || item.actor not in reblog_muted_ap_ids, + true <- !(item.data["type"] == "Announce" && parent.data["actor"] == user.ap_id), true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(parent.data["actor"] not in &1)), true <- MapSet.disjoint?(recipients, recipient_blocks), %{host: item_host} <- URI.parse(item.actor), -- cgit v1.2.3 From bad08f34caf0f037b5d6e724628a222fe65b751b Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 2 Jul 2020 09:57:31 +0200 Subject: Credo fixes. --- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index c84af68f4..61feeae4d 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do collection, and so on. """ alias Pleroma.Activity + alias Pleroma.Activity.Ir.Topics alias Pleroma.Chat alias Pleroma.Chat.MessageReference alias Pleroma.Notification @@ -13,7 +14,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Activity.Ir.Topics alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Push -- cgit v1.2.3 From d44ec2bf4ced0278b8cc5a0b5fa36fcfe38df38b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 2 Jul 2020 12:55:08 -0500 Subject: Remove camelCase from the keys --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 347480d49..89e48fba5 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -93,10 +93,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def fields_limits do %{ - maxFields: Config.get([:instance, :max_account_fields]), - maxRemoteFields: Config.get([:instance, :max_remote_account_fields]), - nameLength: Config.get([:instance, :account_field_name_length]), - valueLength: Config.get([:instance, :account_field_value_length]) + max_fields: Config.get([:instance, :max_account_fields]), + max_remote_fields: Config.get([:instance, :max_remote_account_fields]), + name_length: Config.get([:instance, :account_field_name_length]), + value_length: Config.get([:instance, :account_field_value_length]) } end end -- cgit v1.2.3 From 90764670dc83c39c28cd7851f08f77f1e8bcf25a Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 3 Jul 2020 11:02:15 +0300 Subject: [#1892] Excluded internal users (applications) from user search results, reinstated service actors in search results. --- lib/pleroma/user/search.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 0293c6ae7..42ff1de78 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -52,7 +52,7 @@ defmodule Pleroma.User.Search do |> base_query(following) |> filter_blocked_user(for_user) |> filter_invisible_users() - |> filter_bots() + |> filter_internal_users() |> filter_blocked_domains(for_user) |> fts_search(query_string) |> trigram_rank(query_string) @@ -110,8 +110,8 @@ defmodule Pleroma.User.Search do from(q in query, where: q.invisible == false) end - defp filter_bots(query) do - from(q in query, where: q.actor_type not in ["Application", "Service"]) + defp filter_internal_users(query) do + from(q in query, where: q.actor_type != "Application") end defp filter_blocked_user(query, %User{} = blocker) do -- cgit v1.2.3 From 59b426ebefd1881181888a5b0e6abe8338b65d3f Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 11:25:12 +0200 Subject: Notification Backfill: Explicitly select the needed fields. Prevents a crashing migration when we change user fields. --- lib/pleroma/migration_helper/notification_backfill.ex | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex index b3770307a..d260e62ca 100644 --- a/lib/pleroma/migration_helper/notification_backfill.ex +++ b/lib/pleroma/migration_helper/notification_backfill.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.MigrationHelper.NotificationBackfill do - alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User @@ -25,18 +24,27 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do |> type_from_activity() notification - |> Notification.changeset(%{type: type}) + |> Ecto.Changeset.change(%{type: type}) |> Repo.update() end) end + defp get_by_ap_id(ap_id) do + q = + from(u in User, + select: u.id + ) + + Repo.get_by(q, ap_id: ap_id) + end + # This is copied over from Notifications to keep this stable. defp type_from_activity(%{data: %{"type" => type}} = activity) do case type do "Follow" -> accepted_function = fn activity -> - with %User{} = follower <- User.get_by_ap_id(activity.data["actor"]), - %User{} = followed <- User.get_by_ap_id(activity.data["object"]) do + with %User{} = follower <- get_by_ap_id(activity.data["actor"]), + %User{} = followed <- get_by_ap_id(activity.data["object"]) do Pleroma.FollowingRelationship.following?(follower, followed) end end -- cgit v1.2.3 From 98bfdba108d4213eea82dc4d63edb8bb834118fb Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 12:47:05 +0200 Subject: User: On registration, set `accepts_chat_messages` to true. --- lib/pleroma/user.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 8a54546d6..79e094a79 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -138,6 +138,7 @@ defmodule Pleroma.User do field(:also_known_as, {:array, :string}, default: []) field(:inbox, :string) field(:shared_inbox, :string) + field(:accepts_chat_messages, :boolean, default: false) embeds_one( :notification_settings, @@ -623,6 +624,7 @@ defmodule Pleroma.User do def register_changeset(struct, params \\ %{}, opts \\ []) do bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) + params = Map.put_new(params, :accepts_chat_messages, true) need_confirmation? = if is_nil(opts[:need_confirmation]) do @@ -641,7 +643,8 @@ defmodule Pleroma.User do :nickname, :password, :password_confirmation, - :emoji + :emoji, + :accepts_chat_messages ]) |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) -- cgit v1.2.3 From 3250228be9719b0afa24c97b64f56d2275c4fe67 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 13:07:33 +0200 Subject: AccountView: Add 'accepts_chat_messages' to view. --- lib/pleroma/web/mastodon_api/views/account_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index a6e64b4ab..6a643bfcc 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -245,7 +245,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do hide_favorites: user.hide_favorites, relationship: relationship, skip_thread_containment: user.skip_thread_containment, - background_image: image_url(user.background) |> MediaProxy.url() + background_image: image_url(user.background) |> MediaProxy.url(), + accepts_chat_messages: user.accepts_chat_messages } } |> maybe_put_role(user, opts[:for]) -- cgit v1.2.3 From 37fdb05058d17abde11fd3e55ce896464c7d22e4 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 13:12:23 +0200 Subject: User, Migration: Change `accepts_chat_messages` to be nullable This is to model the ambiguous state of most users. --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 79e094a79..7a684b192 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -138,7 +138,7 @@ defmodule Pleroma.User do field(:also_known_as, {:array, :string}, default: []) field(:inbox, :string) field(:shared_inbox, :string) - field(:accepts_chat_messages, :boolean, default: false) + field(:accepts_chat_messages, :boolean, default: nil) embeds_one( :notification_settings, -- cgit v1.2.3 From 26a7cc3f003d79d6026d67a3a8370516b13c2c90 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 13:38:59 +0200 Subject: UserView: Add acceptsChatMessages field --- lib/pleroma/web/activity_pub/views/user_view.ex | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 4a02b09a1..d062d6230 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -81,6 +81,15 @@ defmodule Pleroma.Web.ActivityPub.UserView do fields = Enum.map(user.fields, &Map.put(&1, "type", "PropertyValue")) + chat_message_acceptance = + if is_boolean(user.accepts_chat_messages) do + %{ + "acceptsChatMessages" => user.accepts_chat_messages + } + else + %{} + end + %{ "id" => user.ap_id, "type" => user.actor_type, @@ -103,6 +112,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "tag" => emoji_tags, "discoverable" => user.discoverable } + |> Map.merge(chat_message_acceptance) |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) |> Map.merge(Utils.make_json_ld_header()) -- cgit v1.2.3 From 5c0bf4c4721f03bd854d4466e77aa08e260c9299 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 13:58:34 +0200 Subject: ActivityPub: Ingest information about chat acceptance. --- lib/pleroma/user.ex | 3 ++- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 7a684b192..a4130c89f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -437,7 +437,8 @@ defmodule Pleroma.User do :discoverable, :invisible, :actor_type, - :also_known_as + :also_known_as, + :accepts_chat_messages ] ) |> validate_required([:name, :ap_id]) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 94117202c..86428b861 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1224,6 +1224,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) locked = data["manuallyApprovesFollowers"] || false + accepts_chat_messages = data["acceptsChatMessages"] data = Transmogrifier.maybe_fix_user_object(data) discoverable = data["discoverable"] || false invisible = data["invisible"] || false @@ -1262,7 +1263,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do also_known_as: Map.get(data, "alsoKnownAs", []), public_key: public_key, inbox: data["inbox"], - shared_inbox: shared_inbox + shared_inbox: shared_inbox, + accepts_chat_messages: accepts_chat_messages } # nickname can be nil because of virtual actors -- cgit v1.2.3 From 3ca9af1f9fef081830820b5bea90f789e460b83a Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 14:31:04 +0200 Subject: Account Schema: Add `accepts_chat_messages` --- lib/pleroma/web/api_spec/schemas/account.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 84f18f1b6..3a84a1593 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -102,7 +102,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do type: :object, description: "A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials`" - } + }, + accepts_chat_messages: %Schema{type: :boolean, nullable: true} } }, source: %Schema{ @@ -169,6 +170,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do "is_admin" => false, "is_moderator" => false, "skip_thread_containment" => false, + "accepts_chat_messages" => true, "chat_token" => "SFMyNTY.g3QAAAACZAAEZGF0YW0AAAASOXRLaTNlc2JHN09RZ1oyOTIwZAAGc2lnbmVkbgYARNplS3EB.Mb_Iaqew2bN1I1o79B_iP7encmVCpTKC4OtHZRxdjKc", "unread_conversation_count" => 0, -- cgit v1.2.3 From 4a7b89e37217af4d98746bb934b8264d7a8de51d Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 15:13:27 +0200 Subject: ChatMessageValidator: Additional validation. --- .../web/activity_pub/object_validators/chat_message_validator.ex | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index c481d79e0..91b475393 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -93,12 +93,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do - If both users are in our system - If at least one of the users in this ChatMessage is a local user - If the recipient is not blocking the actor + - If the recipient is explicitly not accepting chat messages """ def validate_local_concern(cng) do with actor_ap <- get_field(cng, :actor), {_, %User{} = actor} <- {:find_actor, User.get_cached_by_ap_id(actor_ap)}, {_, %User{} = recipient} <- {:find_recipient, User.get_cached_by_ap_id(get_field(cng, :to) |> hd())}, + {_, false} <- {:not_accepting_chats?, recipient.accepts_chat_messages == false}, {_, false} <- {:blocking_actor?, User.blocks?(recipient, actor)}, {_, true} <- {:local?, Enum.any?([actor, recipient], & &1.local)} do cng @@ -107,6 +109,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do cng |> add_error(:actor, "actor is blocked by recipient") + {:not_accepting_chats?, true} -> + cng + |> add_error(:to, "recipient does not accept chat messages") + {:local?, false} -> cng |> add_error(:actor, "actor and recipient are both remote") -- cgit v1.2.3 From e3b5559780f798945eea59170afa9ef41bbf59b3 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 15:54:25 +0200 Subject: AccountController: Make setting accepts_chat_messages possible. --- lib/pleroma/user.ex | 3 ++- lib/pleroma/web/api_spec/operations/account_operation.ex | 9 +++++++-- lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 3 ++- 3 files changed, 11 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a4130c89f..712bc3047 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -483,7 +483,8 @@ defmodule Pleroma.User do :pleroma_settings_store, :discoverable, :actor_type, - :also_known_as + :also_known_as, + :accepts_chat_messages ] ) |> unique_constraint(:nickname) diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 9bde8fc0d..3c05fa55f 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -61,7 +61,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do description: "Update the user's display and preferences.", operationId: "AccountController.update_credentials", security: [%{"oAuth" => ["write:accounts"]}], - requestBody: request_body("Parameters", update_creadentials_request(), required: true), + requestBody: request_body("Parameters", update_credentials_request(), required: true), responses: %{ 200 => Operation.response("Account", "application/json", Account), 403 => Operation.response("Error", "application/json", ApiError) @@ -458,7 +458,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end - defp update_creadentials_request do + defp update_credentials_request do %Schema{ title: "AccountUpdateCredentialsRequest", description: "POST body for creating an account", @@ -492,6 +492,11 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do nullable: true, description: "Whether manual approval of follow requests is required." }, + accepts_chat_messages: %Schema{ + allOf: [BooleanLike], + nullable: true, + description: "Whether the user accepts receiving chat messages." + }, fields_attributes: %Schema{ nullable: true, oneOf: [ diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index b5008d69b..7ff767db6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -160,7 +160,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do :show_role, :skip_thread_containment, :allow_following_move, - :discoverable + :discoverable, + :accepts_chat_messages ] |> Enum.reduce(%{}, fn key, acc -> Maps.put_if_present(acc, key, params[key], &{:ok, truthy_param?(&1)}) -- cgit v1.2.3 From 945e75c8e8f05fadd669c7aa084dd6ba7e9b5ab2 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 Jul 2020 16:36:55 +0200 Subject: SearchController: Trim query. --- lib/pleroma/web/mastodon_api/controllers/search_controller.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index e50980122..29affa7d5 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -44,6 +44,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do def search(conn, params), do: do_search(:v1, conn, params) defp do_search(version, %{assigns: %{user: user}} = conn, %{q: query} = params) do + query = String.trim(query) options = search_options(params, user) timeout = Keyword.get(Repo.config(), :timeout, 15_000) default_values = %{"statuses" => [], "accounts" => [], "hashtags" => []} -- cgit v1.2.3 From eaa59daa4c229bf47e30ac389563c82b11378e07 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 3 Jul 2020 17:06:20 -0500 Subject: Add Captcha endpoint to CSP headers when MediaProxy is enabled. Our CSP rules are lax when MediaProxy enabled, but lenient otherwise. This fixes broken captcha on instances not using MediaProxy. --- lib/pleroma/plugs/http_security_plug.ex | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 1420a9611..f7192ebfc 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -125,11 +125,19 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do if Config.get([Pleroma.Upload, :uploader]) == Pleroma.Uploaders.S3, do: URI.parse(Config.get([Pleroma.Uploaders.S3, :public_endpoint])).host + captcha_method = Config.get([Pleroma.Captcha, :method]) + + captcha_endpoint = + if Config.get([Pleroma.Captcha, :enabled]) && + captcha_method != "Pleroma.Captcha.Native", + do: Config.get([captcha_method, :endpoint]) + [] |> add_source(media_proxy_base_url) |> add_source(upload_base_url) |> add_source(s3_endpoint) |> add_source(media_proxy_whitelist) + |> add_source(captcha_endpoint) end defp add_source(iodata, nil), do: iodata -- cgit v1.2.3 From e9a28078ad969204faae600df3ddff8e75ed2f8a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 3 Jul 2020 17:18:22 -0500 Subject: Rename function and clarify that CSP is only strict with MediaProxy enabled --- lib/pleroma/plugs/http_security_plug.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index f7192ebfc..23a641faf 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -69,10 +69,11 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" + # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src} = if Config.get([:media_proxy, :enabled]) && !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do - sources = get_proxy_and_attachment_sources() + sources = build_csp_multimedia_source_list() {[img_src, sources], [media_src, sources]} else {[img_src, " https:"], [media_src, " https:"]} @@ -107,7 +108,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do |> :erlang.iolist_to_binary() end - defp get_proxy_and_attachment_sources do + defp build_csp_multimedia_source_list do media_proxy_whitelist = Enum.reduce(Config.get([:media_proxy, :whitelist]), [], fn host, acc -> add_source(acc, host) -- cgit v1.2.3 From cf566556147975d45958d2d87a5ce23831eb91df Mon Sep 17 00:00:00 2001 From: lain Date: Sat, 4 Jul 2020 17:11:37 +0200 Subject: Streamer: Don't filter out announce notifications. --- lib/pleroma/web/streamer/streamer.ex | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex index 73ee3e1e1..d1d70e556 100644 --- a/lib/pleroma/web/streamer/streamer.ex +++ b/lib/pleroma/web/streamer/streamer.ex @@ -104,7 +104,9 @@ defmodule Pleroma.Web.Streamer do :ok end - def filtered_by_user?(%User{} = user, %Activity{} = item) do + def filtered_by_user?(user, item, streamed_type \\ :activity) + + def filtered_by_user?(%User{} = user, %Activity{} = item, streamed_type) do %{block: blocked_ap_ids, mute: muted_ap_ids, reblog_mute: reblog_muted_ap_ids} = User.outgoing_relationships_ap_ids(user, [:block, :mute, :reblog_mute]) @@ -116,7 +118,9 @@ defmodule Pleroma.Web.Streamer do true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(item.actor not in &1)), true <- item.data["type"] != "Announce" || item.actor not in reblog_muted_ap_ids, - true <- !(item.data["type"] == "Announce" && parent.data["actor"] == user.ap_id), + true <- + !(streamed_type == :activity && item.data["type"] == "Announce" && + parent.data["actor"] == user.ap_id), true <- Enum.all?([blocked_ap_ids, muted_ap_ids], &(parent.data["actor"] not in &1)), true <- MapSet.disjoint?(recipients, recipient_blocks), %{host: item_host} <- URI.parse(item.actor), @@ -131,8 +135,8 @@ defmodule Pleroma.Web.Streamer do end end - def filtered_by_user?(%User{} = user, %Notification{activity: activity}) do - filtered_by_user?(user, activity) + def filtered_by_user?(%User{} = user, %Notification{activity: activity}, _) do + filtered_by_user?(user, activity, :notification) end defp do_stream("direct", item) do -- cgit v1.2.3 From af612bd006a2792e27f9b995c0c86e010cc77e6c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 5 Jul 2020 10:11:43 -0500 Subject: Ensure all CSP parameters for remote hosts have a scheme --- lib/pleroma/plugs/http_security_plug.ex | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 23a641faf..3bf0b8ce7 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -116,22 +116,22 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do media_proxy_base_url = if Config.get([:media_proxy, :base_url]), - do: URI.parse(Config.get([:media_proxy, :base_url])).host + do: build_csp_param(Config.get([:media_proxy, :base_url])) upload_base_url = if Config.get([Pleroma.Upload, :base_url]), - do: URI.parse(Config.get([Pleroma.Upload, :base_url])).host + do: build_csp_param(Config.get([Pleroma.Upload, :base_url])) s3_endpoint = if Config.get([Pleroma.Upload, :uploader]) == Pleroma.Uploaders.S3, - do: URI.parse(Config.get([Pleroma.Uploaders.S3, :public_endpoint])).host + do: build_csp_param(Config.get([Pleroma.Uploaders.S3, :public_endpoint])) captcha_method = Config.get([Pleroma.Captcha, :method]) captcha_endpoint = if Config.get([Pleroma.Captcha, :enabled]) && captcha_method != "Pleroma.Captcha.Native", - do: Config.get([captcha_method, :endpoint]) + do: build_csp_param(Config.get([captcha_method, :endpoint])) [] |> add_source(media_proxy_base_url) @@ -148,6 +148,14 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do defp add_csp_param(csp_iodata, param), do: [[param, ?;] | csp_iodata] + defp build_csp_param(url) when is_binary(url) do + %{host: host, scheme: scheme} = URI.parse(url) + + if scheme do + scheme <> "://" <> host + end + end + def warn_if_disabled do unless Config.get([:http_security, :enabled]) do Logger.warn(" -- cgit v1.2.3 From fc1f34b85125b24a8094aaa963acb46acacd8eee Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Mon, 6 Jul 2020 00:01:25 +0300 Subject: Delete activity before sending response to client --- .../web/mastodon_api/controllers/status_controller.ex | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 3f4c53437..12be530c9 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -201,15 +201,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do @doc "DELETE /api/v1/statuses/:id" def delete(%{assigns: %{user: user}} = conn, %{id: id}) do with %Activity{} = activity <- Activity.get_by_id_with_object(id), - render <- - try_render(conn, "show.json", - activity: activity, - for: user, - with_direct_conversation_id: true, - with_source: true - ), {:ok, %Activity{}} <- CommonAPI.delete(id, user) do - render + try_render(conn, "show.json", + activity: activity, + for: user, + with_direct_conversation_id: true, + with_source: true + ) else _e -> {:error, :not_found} end -- cgit v1.2.3 From 480dfafa831245976a5c21940adca6f2a73c1213 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 6 Jul 2020 08:48:20 +0300 Subject: don't save tesla settings into db --- lib/pleroma/config/loader.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/config/loader.ex b/lib/pleroma/config/loader.ex index 0f3ecf1ed..64e7de6df 100644 --- a/lib/pleroma/config/loader.ex +++ b/lib/pleroma/config/loader.ex @@ -12,6 +12,11 @@ defmodule Pleroma.Config.Loader do :swarm ] + @reject_groups [ + :postgrex, + :tesla + ] + if Code.ensure_loaded?(Config.Reader) do @reader Config.Reader @@ -47,7 +52,8 @@ defmodule Pleroma.Config.Loader do @spec filter_group(atom(), keyword()) :: keyword() def filter_group(group, configs) do Enum.reject(configs[group], fn {key, _v} -> - key in @reject_keys or (group == :phoenix and key == :serve_endpoints) or group == :postgrex + key in @reject_keys or group in @reject_groups or + (group == :phoenix and key == :serve_endpoints) end) end end -- cgit v1.2.3 From 4a8c26654eb7ca7ce049dd4c485c16672b5837a6 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Sat, 16 Nov 2019 22:54:13 +0100 Subject: Restrict statuses that contain user's irreversible filters --- lib/pleroma/filter.ex | 42 ++++++++++++++++++++-- lib/pleroma/web/activity_pub/activity_pub.ex | 22 ++++++++++++ .../mastodon_api/controllers/filter_controller.ex | 2 +- 3 files changed, 63 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 4d61b3650..91884c6b3 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -34,10 +34,18 @@ defmodule Pleroma.Filter do Repo.one(query) end - def get_filters(%User{id: user_id} = _user) do + def get_active(query) do + from(f in query, where: is_nil(f.expires_at) or f.expires_at > ^NaiveDateTime.utc_now()) + end + + def get_irreversible(query) do + from(f in query, where: f.hide) + end + + def get_by_user(query, %User{id: user_id} = _user) do query = from( - f in Pleroma.Filter, + f in query, where: f.user_id == ^user_id, order_by: [desc: :id] ) @@ -95,4 +103,34 @@ defmodule Pleroma.Filter do |> validate_required([:phrase, :context]) |> Repo.update() end + + def compose_regex(user_or_filters, format \\ :postgres) + + def compose_regex(%User{} = user, format) do + __MODULE__ + |> get_active() + |> get_irreversible() + |> get_by_user(user) + |> compose_regex(format) + end + + def compose_regex([_ | _] = filters, format) do + phrases = + filters + |> Enum.map(& &1.phrase) + |> Enum.join("|") + + case format do + :postgres -> + "\\y(#{phrases})\\y" + + :re -> + ~r/\b#{phrases}\b/i + + _ -> + nil + end + end + + def compose_regex(_, _), do: nil end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 94117202c..31353c866 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do alias Pleroma.Constants alias Pleroma.Conversation alias Pleroma.Conversation.Participation + alias Pleroma.Filter alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -961,6 +962,26 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_instance(query, _), do: query + defp restrict_filtered(query, %{user: %User{} = user}) do + case Filter.compose_regex(user) do + nil -> + query + + regex -> + from([activity, object] in query, + where: + fragment("not(?->>'content' ~* ?)", object.data, ^regex) or + activity.actor == ^user.ap_id + ) + end + end + + defp restrict_filtered(query, %{blocking_user: %User{} = user}) do + restrict_filtered(query, %{user: user}) + end + + defp restrict_filtered(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1099,6 +1120,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_muted_reblogs(restrict_muted_reblogs_opts) |> restrict_instance(opts) |> restrict_announce_object_actor(opts) + |> restrict_filtered(opts) |> Activity.restrict_deactivated_users() |> exclude_poll_votes(opts) |> exclude_chat_messages(opts) diff --git a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex index abbf0ce02..db1ff3189 100644 --- a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterController do @doc "GET /api/v1/filters" def index(%{assigns: %{user: user}} = conn, _) do - filters = Filter.get_filters(user) + filters = Filter.get_by_user(Filter, user) render(conn, "index.json", filters: filters) end -- cgit v1.2.3 From 5af1bf443dfd21a6b0be9efc1f55a73e590f6ba3 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Fri, 22 Nov 2019 19:52:50 +0100 Subject: Skip notifications for statuses that contain an irreversible filtered word --- lib/pleroma/notification.ex | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 2ef1a80c5..3f749cace 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -130,6 +130,7 @@ defmodule Pleroma.Notification do |> preload([n, a, o], activity: {a, object: o}) |> exclude_notification_muted(user, exclude_notification_muted_opts) |> exclude_blocked(user, exclude_blocked_opts) + |> exclude_filtered(user) |> exclude_visibility(opts) end @@ -158,6 +159,20 @@ defmodule Pleroma.Notification do |> where([n, a, o, tm], is_nil(tm.user_id)) end + defp exclude_filtered(query, user) do + case Pleroma.Filter.compose_regex(user) do + nil -> + query + + regex -> + from([_n, a, o] in query, + where: + fragment("not(?->>'content' ~* ?)", o.data, ^regex) or + fragment("?->>'actor' = ?", o.data, ^user.ap_id) + ) + end + end + @valid_visibilities ~w[direct unlisted public private] defp exclude_visibility(query, %{exclude_visibilities: visibility}) @@ -555,7 +570,8 @@ defmodule Pleroma.Notification do :follows, :non_followers, :non_follows, - :recently_followed + :recently_followed, + :filtered ] |> Enum.find(&skip?(&1, activity, user)) end @@ -624,6 +640,24 @@ defmodule Pleroma.Notification do end) end + def skip?(:filtered, activity, user) do + object = Object.normalize(activity) + + cond do + is_nil(object) -> + false + + object.data["actor"] == user.ap_id -> + false + + not is_nil(regex = Pleroma.Filter.compose_regex(user, :re)) -> + Regex.match?(regex, object.data["content"]) + + true -> + false + end + end + def skip?(_, _, _), do: false def for_user_and_activity(user, activity) do -- cgit v1.2.3 From 8277b29790dfd283d94b995539dcb28e51131150 Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Mon, 25 Nov 2019 16:59:55 +0100 Subject: Restrict thread statuses that contain user's irreversible filters --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 31353c866..8abbef487 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -447,6 +447,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_set_thread_muted_field(opts) |> restrict_blocked(opts) |> restrict_recipients(recipients, opts[:user]) + |> restrict_filtered(opts) |> where( [activity], fragment( @@ -1112,6 +1113,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_favorited_by(opts) |> restrict_blocked(restrict_blocked_opts) |> restrict_muted(restrict_muted_opts) + |> restrict_filtered(opts) |> restrict_media(opts) |> restrict_visibility(opts) |> restrict_thread_visibility(opts, config) -- cgit v1.2.3 From 6558f31cda07b8472ed99823ed0f46deffa584cc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 7 Feb 2020 18:16:39 +0300 Subject: don't filter notifications for follow and move types --- lib/pleroma/notification.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 3f749cace..d439f51bc 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -640,6 +640,8 @@ defmodule Pleroma.Notification do end) end + def skip?(:filtered, %{data: %{"type" => type}}, _) when type in ["Follow", "Move"], do: false + def skip?(:filtered, activity, user) do object = Object.normalize(activity) -- cgit v1.2.3 From 771748db1fa01a71c52c20b890e1b80bfcf1e230 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 26 Feb 2020 13:59:07 +0000 Subject: Apply suggestion to lib/pleroma/filter.ex --- lib/pleroma/filter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 91884c6b3..98cb575a9 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -42,7 +42,7 @@ defmodule Pleroma.Filter do from(f in query, where: f.hide) end - def get_by_user(query, %User{id: user_id} = _user) do + def get_filters(query \\ __MODULE__, %User{id: user_id}) do query = from( f in query, -- cgit v1.2.3 From 20c27bef4083330a2415f1c0a04e4cad128b267a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 26 Feb 2020 17:50:56 +0300 Subject: renaming back and reject nil on create --- lib/pleroma/filter.ex | 2 +- lib/pleroma/notification.ex | 1 + lib/pleroma/web/mastodon_api/controllers/filter_controller.ex | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index 98cb575a9..5d6df9530 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -110,7 +110,7 @@ defmodule Pleroma.Filter do __MODULE__ |> get_active() |> get_irreversible() - |> get_by_user(user) + |> get_filters(user) |> compose_regex(format) end diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index d439f51bc..fcb2144ae 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -352,6 +352,7 @@ defmodule Pleroma.Notification do end end + @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []} def create_notifications(activity, options \\ []) def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do diff --git a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex index db1ff3189..abbf0ce02 100644 --- a/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/filter_controller.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MastodonAPI.FilterController do @doc "GET /api/v1/filters" def index(%{assigns: %{user: user}} = conn, _) do - filters = Filter.get_by_user(Filter, user) + filters = Filter.get_filters(user) render(conn, "index.json", filters: filters) end -- cgit v1.2.3 From af7720237b448341932a4a0b53d94b006114e915 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Jul 2020 11:08:13 +0200 Subject: Upload: Restrict description length --- lib/pleroma/upload.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 797555bff..0fa6b89dc 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -63,6 +63,10 @@ defmodule Pleroma.Upload do with {:ok, upload} <- prepare_upload(upload, opts), upload = %__MODULE__{upload | path: upload.path || "#{upload.id}/#{upload.name}"}, {:ok, upload} <- Pleroma.Upload.Filter.filter(opts.filters, upload), + description = Map.get(opts, :description) || upload.name, + {_, true} <- + {:description_limit, + String.length(description) <= Pleroma.Config.get([:instance, :description_limit])}, {:ok, url_spec} <- Pleroma.Uploaders.Uploader.put_file(opts.uploader, upload) do {:ok, %{ @@ -75,9 +79,12 @@ defmodule Pleroma.Upload do "href" => url_from_spec(upload, opts.base_url, url_spec) } ], - "name" => Map.get(opts, :description) || upload.name + "name" => description }} else + {:description_limit, _} -> + {:error, :description_too_long} + {:error, error} -> Logger.error( "#{__MODULE__} store (using #{inspect(opts.uploader)}) failed: #{inspect(error)}" -- cgit v1.2.3 From cc8b4e48d966211fdad43121850ac1ecfbb73c74 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Jul 2020 11:12:37 +0200 Subject: InstanceView: Add chat limit, description limit --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 89e48fba5..5deb0d7ed 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -34,6 +34,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_upload_limit: Keyword.get(instance, :background_upload_limit), banner_upload_limit: Keyword.get(instance, :banner_upload_limit), background_image: Keyword.get(instance, :background_image), + chat_limit: Keyword.get(instance, :chat_limit), + description_limit: Keyword.get(instance, :description_limit), pleroma: %{ metadata: %{ account_activation_required: Keyword.get(instance, :account_activation_required), -- cgit v1.2.3 From 208baf157ad0c8be470566d5d51d0214c229e6a5 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Jul 2020 11:38:40 +0200 Subject: ActivityPub: Add new 'capabilities' to user. --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 ++- lib/pleroma/web/activity_pub/views/user_view.ex | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 86428b861..17c9d8f21 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1224,7 +1224,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) locked = data["manuallyApprovesFollowers"] || false - accepts_chat_messages = data["acceptsChatMessages"] + capabilities = data["capabilities"] || %{} + accepts_chat_messages = capabilities["acceptsChatMessages"] data = Transmogrifier.maybe_fix_user_object(data) discoverable = data["discoverable"] || false invisible = data["invisible"] || false diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index d062d6230..3a4564912 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -81,7 +81,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do fields = Enum.map(user.fields, &Map.put(&1, "type", "PropertyValue")) - chat_message_acceptance = + capabilities = if is_boolean(user.accepts_chat_messages) do %{ "acceptsChatMessages" => user.accepts_chat_messages @@ -110,9 +110,9 @@ defmodule Pleroma.Web.ActivityPub.UserView do "endpoints" => endpoints, "attachment" => fields, "tag" => emoji_tags, - "discoverable" => user.discoverable + "discoverable" => user.discoverable, + "capabilities" => capabilities } - |> Map.merge(chat_message_acceptance) |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) |> Map.merge(Utils.make_json_ld_header()) -- cgit v1.2.3 From 158c26d7ddb3c77dc99a6298114929faf6a2915a Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Jul 2020 12:11:10 +0200 Subject: StaticFE Plug: Use phoenix helper to get the requested format. --- lib/pleroma/plugs/static_fe_plug.ex | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/static_fe_plug.ex b/lib/pleroma/plugs/static_fe_plug.ex index 7c69b2dac..143665c71 100644 --- a/lib/pleroma/plugs/static_fe_plug.ex +++ b/lib/pleroma/plugs/static_fe_plug.ex @@ -21,12 +21,6 @@ defmodule Pleroma.Plugs.StaticFEPlug do defp enabled?, do: Pleroma.Config.get([:static_fe, :enabled], false) defp requires_html?(conn) do - case get_req_header(conn, "accept") do - [accept | _] -> - !String.contains?(accept, "json") && String.contains?(accept, "text/html") - - _ -> - false - end + Phoenix.Controller.get_format(conn) == "html" end end -- cgit v1.2.3 From 28feba8af4f51871a3ac1ffd3826e798c2265d43 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Jul 2020 12:24:45 +0200 Subject: Preloaders: Remove status_net preloader --- lib/pleroma/web/preload/status_net.ex | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 lib/pleroma/web/preload/status_net.ex (limited to 'lib') diff --git a/lib/pleroma/web/preload/status_net.ex b/lib/pleroma/web/preload/status_net.ex deleted file mode 100644 index 9b62f87a2..000000000 --- a/lib/pleroma/web/preload/status_net.ex +++ /dev/null @@ -1,25 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Preload.Providers.StatusNet do - alias Pleroma.Web.Preload.Providers.Provider - alias Pleroma.Web.TwitterAPI.UtilController - - @behaviour Provider - @config_url "/api/statusnet/config.json" - - @impl Provider - def generate_terms(_params) do - %{} - |> build_config_tag() - end - - defp build_config_tag(acc) do - resp = - Plug.Test.conn(:get, @config_url |> to_string()) - |> UtilController.config(nil) - - Map.put(acc, @config_url, resp.resp_body) - end -end -- cgit v1.2.3 From a6a12b241fbacd3ff35cd901190e62d14aaac3c2 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 6 Jul 2020 15:57:19 +0200 Subject: FollowValidator: Add basic validation. --- lib/pleroma/web/activity_pub/builder.ex | 13 +++++++ lib/pleroma/web/activity_pub/object_validator.ex | 11 ++++++ .../object_validators/follow_validator.ex | 42 ++++++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/object_validators/follow_validator.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index cabc28de9..d5f3610ed 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -14,6 +14,19 @@ defmodule Pleroma.Web.ActivityPub.Builder do require Pleroma.Constants + @spec follow(User.t(), User.t()) :: {:ok, map(), keyword()} + def follow(follower, followed) do + data = %{ + "id" => Utils.generate_activity_id(), + "actor" => follower.ap_id, + "type" => "Follow", + "object" => followed.ap_id, + "to" => [followed.ap_id] + } + + {:ok, data, []} + end + @spec emoji_react(User.t(), Object.t(), String.t()) :: {:ok, map(), keyword()} def emoji_react(actor, object, emoji) do with {:ok, data, meta} <- object_action(actor, object) do diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index bb6324460..df926829c 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -18,6 +18,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.UpdateValidator @@ -25,6 +26,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} def validate(object, meta) + def validate(%{"type" => "Follow"} = object, meta) do + with {:ok, object} <- + object + |> FollowValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + def validate(%{"type" => "Block"} = block_activity, meta) do with {:ok, block_activity} <- block_activity diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex new file mode 100644 index 000000000..2035ad9ba --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex @@ -0,0 +1,42 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do + use Ecto.Schema + + alias Pleroma.EctoType.ActivityPub.ObjectValidators + + import Ecto.Changeset + import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + + @primary_key false + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:type, :string) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:object, ObjectValidators.ObjectID) + end + + def cast_data(data) do + %__MODULE__{} + |> cast(data, __schema__(:fields)) + end + + def validate_data(cng) do + cng + |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_inclusion(:type, ["Follow"]) + |> validate_actor_presence() + |> validate_actor_presence(field_name: :object) + end + + def cast_and_validate(data) do + data + |> cast_data + |> validate_data + end +end -- cgit v1.2.3 From 65843d92c4d3999f727d00500dcf8943cfe7bbc0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 6 Jul 2020 10:59:41 -0500 Subject: Simplify the logic --- lib/pleroma/plugs/http_security_plug.ex | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 3bf0b8ce7..13423ca3f 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -114,24 +114,15 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do add_source(acc, host) end) - media_proxy_base_url = - if Config.get([:media_proxy, :base_url]), - do: build_csp_param(Config.get([:media_proxy, :base_url])) + media_proxy_base_url = build_csp_param(Config.get([:media_proxy, :base_url])) - upload_base_url = - if Config.get([Pleroma.Upload, :base_url]), - do: build_csp_param(Config.get([Pleroma.Upload, :base_url])) + upload_base_url = build_csp_param(Config.get([Pleroma.Upload, :base_url])) - s3_endpoint = - if Config.get([Pleroma.Upload, :uploader]) == Pleroma.Uploaders.S3, - do: build_csp_param(Config.get([Pleroma.Uploaders.S3, :public_endpoint])) + s3_endpoint = build_csp_param(Config.get([Pleroma.Uploaders.S3, :public_endpoint])) captcha_method = Config.get([Pleroma.Captcha, :method]) - captcha_endpoint = - if Config.get([Pleroma.Captcha, :enabled]) && - captcha_method != "Pleroma.Captcha.Native", - do: build_csp_param(Config.get([captcha_method, :endpoint])) + captcha_endpoint = build_csp_param(Config.get([captcha_method, :endpoint])) [] |> add_source(media_proxy_base_url) @@ -148,6 +139,8 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do defp add_csp_param(csp_iodata, param), do: [[param, ?;] | csp_iodata] + defp build_csp_param(nil), do: nil + defp build_csp_param(url) when is_binary(url) do %{host: host, scheme: scheme} = URI.parse(url) -- cgit v1.2.3 From da4029391d217b1e2c151e69479de42e2221e02f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 6 Jul 2020 11:28:08 -0500 Subject: IO list, not concatenation --- lib/pleroma/plugs/http_security_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 13423ca3f..472a3ff42 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -145,7 +145,7 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do %{host: host, scheme: scheme} = URI.parse(url) if scheme do - scheme <> "://" <> host + [scheme, "://", host] end end -- cgit v1.2.3 From fbb9743a7058e8a7ace69804b79eb032e03da078 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 25 May 2020 13:13:42 +0200 Subject: Fix getting videos from peertube --- lib/pleroma/web/activity_pub/transmogrifier.ex | 34 +++++++++++++++++--------- 1 file changed, 23 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index bc6fc4bd8..117e930b3 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -233,18 +233,24 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do is_map(url) && is_binary(url["href"]) -> url["href"] is_binary(data["url"]) -> data["url"] is_binary(data["href"]) -> data["href"] + true -> nil end - attachment_url = - %{"href" => href} - |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("type", Map.get(url || %{}, "type")) + if href do + attachment_url = + %{"href" => href} + |> Maps.put_if_present("mediaType", media_type) + |> Maps.put_if_present("type", Map.get(url || %{}, "type")) - %{"url" => [attachment_url]} - |> Maps.put_if_present("mediaType", media_type) - |> Maps.put_if_present("type", data["type"]) - |> Maps.put_if_present("name", data["name"]) + %{"url" => [attachment_url]} + |> Maps.put_if_present("mediaType", media_type) + |> Maps.put_if_present("type", data["type"]) + |> Maps.put_if_present("name", data["name"]) + else + nil + end end) + |> Enum.filter(& &1) Map.put(object, "attachment", attachments) end @@ -263,12 +269,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_url(%{"type" => object_type, "url" => url} = object) when object_type in ["Video", "Audio"] and is_list(url) do - first_element = Enum.at(url, 0) + attachment = + Enum.find(url, fn x -> + media_type = x["mediaType"] || x["mimeType"] || "" + + is_map(x) and String.starts_with?(media_type, ["audio/", "video/"]) + end) - link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) + link_element = + Enum.find(url, fn x -> is_map(x) and (x["mediaType"] || x["mimeType"]) == "text/html" end) object - |> Map.put("attachment", [first_element]) + |> Map.put("attachment", [attachment]) |> Map.put("url", link_element["href"]) end -- cgit v1.2.3 From 59cf78e41236a527b21befaadd329e882a62b40a Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 7 Jul 2020 16:20:50 +0200 Subject: AccountController: Allow removal / reset of user images. --- lib/pleroma/user.ex | 13 +++++-------- .../web/mastodon_api/controllers/account_controller.ex | 13 ++++++++++--- 2 files changed, 15 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 8a54546d6..e98332744 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -89,7 +89,7 @@ defmodule Pleroma.User do field(:keys, :string) field(:public_key, :string) field(:ap_id, :string) - field(:avatar, :map) + field(:avatar, :map, default: %{}) field(:local, :boolean, default: true) field(:follower_address, :string) field(:following_address, :string) @@ -539,14 +539,11 @@ defmodule Pleroma.User do end defp put_change_if_present(changeset, map_field, value_function) do - if value = get_change(changeset, map_field) do - with {:ok, new_value} <- value_function.(value) do - put_change(changeset, map_field, new_value) - else - _ -> changeset - end + with {:ok, value} <- fetch_change(changeset, map_field), + {:ok, new_value} <- value_function.(value) do + put_change(changeset, map_field, new_value) else - changeset + _ -> changeset end end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index b5008d69b..d4532258c 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -148,6 +148,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do |> Enum.filter(fn {_, value} -> not is_nil(value) end) |> Enum.into(%{}) + # We use an empty string as a special value to reset + # avatars, banners, backgrounds + user_image_value = fn + "" -> {:ok, nil} + value -> {:ok, value} + end + user_params = [ :no_rich_text, @@ -168,9 +175,9 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do |> Maps.put_if_present(:name, params[:display_name]) |> Maps.put_if_present(:bio, params[:note]) |> Maps.put_if_present(:raw_bio, params[:note]) - |> Maps.put_if_present(:avatar, params[:avatar]) - |> Maps.put_if_present(:banner, params[:header]) - |> Maps.put_if_present(:background, params[:pleroma_background_image]) + |> Maps.put_if_present(:avatar, params[:avatar], user_image_value) + |> Maps.put_if_present(:banner, params[:header], user_image_value) + |> Maps.put_if_present(:background, params[:pleroma_background_image], user_image_value) |> Maps.put_if_present( :raw_fields, params[:fields_attributes], -- cgit v1.2.3 From c8dd973af5241547beb8c2207a0c13b933745cf6 Mon Sep 17 00:00:00 2001 From: lain Date: Tue, 7 Jul 2020 16:48:47 +0200 Subject: AccountController: Remove unused `update_?` routes. These were not documented and are also not used anymore. --- .../operations/pleroma_account_operation.ex | 91 ---------------------- .../pleroma_api/controllers/account_controller.ex | 62 --------------- lib/pleroma/web/router.ex | 4 - 3 files changed, 157 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex index 90922c064..97836b2eb 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex @@ -4,7 +4,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do alias OpenApiSpex.Operation - alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Schemas.AccountRelationship alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.FlakeID @@ -40,48 +39,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do } end - def update_avatar_operation do - %Operation{ - tags: ["Accounts"], - summary: "Set/clear user avatar image", - operationId: "PleromaAPI.AccountController.update_avatar", - requestBody: - request_body("Parameters", update_avatar_or_background_request(), required: true), - security: [%{"oAuth" => ["write:accounts"]}], - responses: %{ - 200 => update_response(), - 403 => Operation.response("Forbidden", "application/json", ApiError) - } - } - end - - def update_banner_operation do - %Operation{ - tags: ["Accounts"], - summary: "Set/clear user banner image", - operationId: "PleromaAPI.AccountController.update_banner", - requestBody: request_body("Parameters", update_banner_request(), required: true), - security: [%{"oAuth" => ["write:accounts"]}], - responses: %{ - 200 => update_response() - } - } - end - - def update_background_operation do - %Operation{ - tags: ["Accounts"], - summary: "Set/clear user background image", - operationId: "PleromaAPI.AccountController.update_background", - security: [%{"oAuth" => ["write:accounts"]}], - requestBody: - request_body("Parameters", update_avatar_or_background_request(), required: true), - responses: %{ - 200 => update_response() - } - } - end - def favourites_operation do %Operation{ tags: ["Accounts"], @@ -136,52 +93,4 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do required: true ) end - - defp update_avatar_or_background_request do - %Schema{ - title: "PleromaAccountUpdateAvatarOrBackgroundRequest", - type: :object, - properties: %{ - img: %Schema{ - nullable: true, - type: :string, - format: :binary, - description: "Image encoded using `multipart/form-data` or an empty string to clear" - } - } - } - end - - defp update_banner_request do - %Schema{ - title: "PleromaAccountUpdateBannerRequest", - type: :object, - properties: %{ - banner: %Schema{ - type: :string, - nullable: true, - format: :binary, - description: "Image encoded using `multipart/form-data` or an empty string to clear" - } - } - } - end - - defp update_response do - Operation.response("PleromaAccountUpdateResponse", "application/json", %Schema{ - type: :object, - properties: %{ - url: %Schema{ - type: :string, - format: :uri, - nullable: true, - description: "Image URL" - } - }, - example: %{ - "url" => - "https://cofe.party/media/9d0add56-bcb6-4c0f-8225-cbbd0b6dd773/13eadb6972c9ccd3f4ffa3b8196f0e0d38b4d2f27594457c52e52946c054cd9a.gif" - } - }) - end end diff --git a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex index f3554d919..563edded7 100644 --- a/lib/pleroma/web/pleroma_api/controllers/account_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/account_controller.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do import Pleroma.Web.ControllerHelper, only: [json_response: 3, add_link_headers: 2, assign_account_by_id: 2] - alias Ecto.Changeset alias Pleroma.Plugs.EnsurePublicOrAuthenticatedPlug alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Plugs.RateLimiter @@ -35,17 +34,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do %{scopes: ["follow", "write:follows"]} when action in [:subscribe, :unsubscribe] ) - plug( - OAuthScopesPlug, - %{scopes: ["write:accounts"]} - # Note: the following actions are not permission-secured in Mastodon: - when action in [ - :update_avatar, - :update_banner, - :update_background - ] - ) - plug( OAuthScopesPlug, %{scopes: ["read:favourites"], fallback: :proceed_unauthenticated} when action == :favourites @@ -68,56 +56,6 @@ defmodule Pleroma.Web.PleromaAPI.AccountController do end end - @doc "PATCH /api/v1/pleroma/accounts/update_avatar" - def update_avatar(%{assigns: %{user: user}, body_params: %{img: ""}} = conn, _) do - {:ok, _user} = - user - |> Changeset.change(%{avatar: nil}) - |> User.update_and_set_cache() - - json(conn, %{url: nil}) - end - - def update_avatar(%{assigns: %{user: user}, body_params: params} = conn, _params) do - {:ok, %{data: data}} = ActivityPub.upload(params, type: :avatar) - {:ok, _user} = user |> Changeset.change(%{avatar: data}) |> User.update_and_set_cache() - %{"url" => [%{"href" => href} | _]} = data - - json(conn, %{url: href}) - end - - @doc "PATCH /api/v1/pleroma/accounts/update_banner" - def update_banner(%{assigns: %{user: user}, body_params: %{banner: ""}} = conn, _) do - with {:ok, _user} <- User.update_banner(user, %{}) do - json(conn, %{url: nil}) - end - end - - def update_banner(%{assigns: %{user: user}, body_params: params} = conn, _) do - with {:ok, object} <- ActivityPub.upload(%{img: params[:banner]}, type: :banner), - {:ok, _user} <- User.update_banner(user, object.data) do - %{"url" => [%{"href" => href} | _]} = object.data - - json(conn, %{url: href}) - end - end - - @doc "PATCH /api/v1/pleroma/accounts/update_background" - def update_background(%{assigns: %{user: user}, body_params: %{img: ""}} = conn, _) do - with {:ok, _user} <- User.update_background(user, %{}) do - json(conn, %{url: nil}) - end - end - - def update_background(%{assigns: %{user: user}, body_params: params} = conn, _) do - with {:ok, object} <- ActivityPub.upload(params, type: :background), - {:ok, _user} <- User.update_background(user, object.data) do - %{"url" => [%{"href" => href} | _]} = object.data - - json(conn, %{url: href}) - end - end - @doc "GET /api/v1/pleroma/accounts/:id/favourites" def favourites(%{assigns: %{account: %{hide_favorites: true}}} = conn, _params) do render_error(conn, :forbidden, "Can't get favorites") diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9e457848e..74e940f8e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -328,10 +328,6 @@ defmodule Pleroma.Web.Router do delete("/statuses/:id/reactions/:emoji", EmojiReactionController, :delete) post("/notifications/read", NotificationController, :mark_as_read) - patch("/accounts/update_avatar", AccountController, :update_avatar) - patch("/accounts/update_banner", AccountController, :update_banner) - patch("/accounts/update_background", AccountController, :update_background) - get("/mascot", MascotController, :show) put("/mascot", MascotController, :update) -- cgit v1.2.3 From 18438a9bf0added295b119de2838fcea0f28b701 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 Jul 2020 11:21:05 -0500 Subject: Add "Bot" to User Agent to coerce Twitter into serving OGP tags. --- lib/pleroma/web/rich_media/parser.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index ef5ead2da..c8a767935 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -86,7 +86,10 @@ defmodule Pleroma.Web.RichMedia.Parser do end try do - {:ok, %Tesla.Env{body: html}} = Pleroma.HTTP.get(url, [], adapter: opts) + rich_media_agent = Pleroma.Application.user_agent() <> "; Bot" + + {:ok, %Tesla.Env{body: html}} = + Pleroma.HTTP.get(url, [{"user-agent", rich_media_agent}], adapter: opts) html |> parse_html() -- cgit v1.2.3 From 3e08e7715126ca1f3bfaf7dddf4806e76d9bd993 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 7 Jul 2020 20:37:11 +0300 Subject: [#1895] Made hashtag timeline respect `:restrict_unauthenticated` instance setting. --- .../controllers/timeline_controller.ex | 43 ++++++++++++++-------- 1 file changed, 27 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 4bdd46d7e..4bbb82c23 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -88,21 +88,23 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do ) end - # GET /api/v1/timelines/public - def public(%{assigns: %{user: user}} = conn, params) do - local_only = params[:local] - - cfg_key = + defp restrict_unauthenticated?(local_only) do + config_key = if local_only do :local else :federated end - restrict? = Pleroma.Config.get([:restrict_unauthenticated, :timelines, cfg_key]) + Pleroma.Config.get([:restrict_unauthenticated, :timelines, config_key]) + end + + # GET /api/v1/timelines/public + def public(%{assigns: %{user: user}} = conn, params) do + local_only = params[:local] - if restrict? and is_nil(user) do - render_error(conn, :unauthorized, "authorization required for timeline view") + if is_nil(user) and restrict_unauthenticated?(local_only) do + fail_on_bad_auth(conn) else activities = params @@ -123,6 +125,10 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do end end + defp fail_on_bad_auth(conn) do + render_error(conn, :unauthorized, "authorization required for timeline view") + end + defp hashtag_fetching(params, user, local_only) do tags = [params[:tag], params[:any]] @@ -157,15 +163,20 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do # GET /api/v1/timelines/tag/:tag def hashtag(%{assigns: %{user: user}} = conn, params) do local_only = params[:local] - activities = hashtag_fetching(params, user, local_only) - conn - |> add_link_headers(activities, %{"local" => local_only}) - |> render("index.json", - activities: activities, - for: user, - as: :activity - ) + if is_nil(user) and restrict_unauthenticated?(local_only) do + fail_on_bad_auth(conn) + else + activities = hashtag_fetching(params, user, local_only) + + conn + |> add_link_headers(activities, %{"local" => local_only}) + |> render("index.json", + activities: activities, + for: user, + as: :activity + ) + end end # GET /api/v1/timelines/list/:list_id -- cgit v1.2.3 From f6d09fafee83514889bbcf6531e0bc01e33b0b16 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 1 Mar 2020 09:48:32 +0100 Subject: Add support for remote favicons --- lib/pleroma/user.ex | 30 ++++++++++++++++++++++ lib/pleroma/web/mastodon_api/views/account_view.ex | 3 ++- 2 files changed, 32 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e98332744..25ea112a2 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2253,4 +2253,34 @@ defmodule Pleroma.User do |> Map.put(:bio, HTML.filter_tags(user.bio, filter)) |> Map.put(:fields, fields) end + + def get_cached_favicon(%User{} = user) do + key = "favicon:#{user.ap_id}" + Cachex.fetch!(:user_cache, key, fn _ -> get_favicon(user) end) + end + + def get_cached_favicon(_user) do + nil + end + + def get_favicon(user) do + try do + with url <- user.ap_id, + true <- is_binary(url), + {:ok, %Tesla.Env{body: html}} <- Pleroma.HTTP.get(url), + favicon_rel <- + html + |> Floki.parse_document!() + |> Floki.attribute("link[rel=icon]", "href") + |> List.first(), + favicon_url <- URI.merge(URI.parse(url), favicon_rel) |> to_string(), + true <- is_binary(favicon_url) do + favicon_url + else + _ -> nil + end + rescue + _ -> nil + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index a6e64b4ab..efe835e3c 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -245,7 +245,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do hide_favorites: user.hide_favorites, relationship: relationship, skip_thread_containment: user.skip_thread_containment, - background_image: image_url(user.background) |> MediaProxy.url() + background_image: image_url(user.background) |> MediaProxy.url(), + favicon: User.get_cached_favicon(user) |> MediaProxy.url() } } |> maybe_put_role(user, opts[:for]) -- cgit v1.2.3 From 6a679d80c9030afa8327377928f8ac2fcf1a4a0e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 2 Mar 2020 05:38:25 +0100 Subject: Move get_favicon to Pleroma.Instances, use / --- lib/pleroma/application.ex | 1 + lib/pleroma/instances.ex | 28 ++++++++++++++++++++ lib/pleroma/user.ex | 30 ---------------------- lib/pleroma/web/mastodon_api/views/account_view.ex | 11 +++++++- 4 files changed, 39 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9615af122..c7fc95f75 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -150,6 +150,7 @@ defmodule Pleroma.Application do build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) + build_cachex("instances", default_ttl: 25_000, ttl_interval: 1000, limit: 2500) ] end diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 557e8decf..c9b1ed4ce 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -37,4 +37,32 @@ defmodule Pleroma.Instances do url_or_host end end + + def get_cached_favicon(instance_url) when is_binary(instance_url) do + Cachex.fetch!(:instances_cache, instance_url, fn _ -> get_favicon(instance_url) end) + end + + def get_cached_favicon(_instance_url) do + nil + end + + def get_favicon(instance_url) when is_binary(instance_url) do + try do + with {:ok, %Tesla.Env{body: html}} <- + Pleroma.HTTP.get(instance_url, [{:Accept, "text/html"}]), + favicon_rel <- + html + |> Floki.parse_document!() + |> Floki.attribute("link[rel=icon]", "href") + |> List.first(), + favicon_url <- URI.merge(URI.parse(instance_url), favicon_rel) |> to_string(), + true <- is_binary(favicon_url) do + favicon_url + else + _ -> nil + end + rescue + _ -> nil + end + end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 25ea112a2..e98332744 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2253,34 +2253,4 @@ defmodule Pleroma.User do |> Map.put(:bio, HTML.filter_tags(user.bio, filter)) |> Map.put(:fields, fields) end - - def get_cached_favicon(%User{} = user) do - key = "favicon:#{user.ap_id}" - Cachex.fetch!(:user_cache, key, fn _ -> get_favicon(user) end) - end - - def get_cached_favicon(_user) do - nil - end - - def get_favicon(user) do - try do - with url <- user.ap_id, - true <- is_binary(url), - {:ok, %Tesla.Env{body: html}} <- Pleroma.HTTP.get(url), - favicon_rel <- - html - |> Floki.parse_document!() - |> Floki.attribute("link[rel=icon]", "href") - |> List.first(), - favicon_url <- URI.merge(URI.parse(url), favicon_rel) |> to_string(), - true <- is_binary(favicon_url) do - favicon_url - else - _ -> nil - end - rescue - _ -> nil - end - end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index efe835e3c..3ee50dfd0 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -204,6 +204,15 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do %{} end + favicon = + user + |> Map.get(:ap_id, "") + |> URI.parse() + |> URI.merge("/") + |> to_string() + |> Pleroma.Instances.get_cached_favicon() + |> MediaProxy.url() + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -246,7 +255,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do relationship: relationship, skip_thread_containment: user.skip_thread_containment, background_image: image_url(user.background) |> MediaProxy.url(), - favicon: User.get_cached_favicon(user) |> MediaProxy.url() + favicon: favicon } } |> maybe_put_role(user, opts[:for]) -- cgit v1.2.3 From 013e2c505786dff311bcc8bf23631d6a1a1636ef Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 7 Jul 2020 11:13:38 +0200 Subject: Use instances table instead of Cachex --- lib/pleroma/application.ex | 1 - lib/pleroma/instances.ex | 28 ----------- lib/pleroma/instances/instance.ex | 55 +++++++++++++++++++++- lib/pleroma/web/mastodon_api/views/account_view.ex | 3 +- 4 files changed, 55 insertions(+), 32 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c7fc95f75..9615af122 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -150,7 +150,6 @@ defmodule Pleroma.Application do build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) - build_cachex("instances", default_ttl: 25_000, ttl_interval: 1000, limit: 2500) ] end diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index c9b1ed4ce..557e8decf 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -37,32 +37,4 @@ defmodule Pleroma.Instances do url_or_host end end - - def get_cached_favicon(instance_url) when is_binary(instance_url) do - Cachex.fetch!(:instances_cache, instance_url, fn _ -> get_favicon(instance_url) end) - end - - def get_cached_favicon(_instance_url) do - nil - end - - def get_favicon(instance_url) when is_binary(instance_url) do - try do - with {:ok, %Tesla.Env{body: html}} <- - Pleroma.HTTP.get(instance_url, [{:Accept, "text/html"}]), - favicon_rel <- - html - |> Floki.parse_document!() - |> Floki.attribute("link[rel=icon]", "href") - |> List.first(), - favicon_url <- URI.merge(URI.parse(instance_url), favicon_rel) |> to_string(), - true <- is_binary(favicon_url) do - favicon_url - else - _ -> nil - end - rescue - _ -> nil - end - end end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 74458c09a..b97e229e5 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Instances.Instance do schema "instances" do field(:host, :string) field(:unreachable_since, :naive_datetime_usec) + field(:favicon, :string) + field(:favicon_updated_at, :naive_datetime) timestamps() end @@ -25,7 +27,7 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since]) + |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) |> validate_required([:host]) |> unique_constraint(:host) end @@ -120,4 +122,55 @@ defmodule Pleroma.Instances.Instance do end defp parse_datetime(datetime), do: datetime + + def get_or_update_favicon(%URI{host: host} = instance_uri) do + existing_record = Repo.get_by(Instance, %{host: host}) + now = NaiveDateTime.utc_now() + + if existing_record && existing_record.favicon && + NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do + existing_record.favicon + else + favicon = scrape_favicon(instance_uri) + + cond do + is_binary(favicon) && existing_record -> + existing_record + |> changeset(%{favicon: favicon, favicon_updated_at: now}) + |> Repo.update() + + favicon + + is_binary(favicon) -> + %Instance{} + |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) + |> Repo.insert() + + favicon + + true -> + nil + end + end + end + + defp scrape_favicon(%URI{} = instance_uri) do + try do + with {:ok, %Tesla.Env{body: html}} <- + Pleroma.HTTP.get(to_string(instance_uri), [{:Accept, "text/html"}]), + favicon_rel <- + html + |> Floki.parse_document!() + |> Floki.attribute("link[rel=icon]", "href") + |> List.first(), + favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(), + true <- is_binary(favicon) do + favicon + else + _ -> nil + end + rescue + _ -> nil + end + end end diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 3ee50dfd0..db5739254 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -209,8 +209,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> Map.get(:ap_id, "") |> URI.parse() |> URI.merge("/") - |> to_string() - |> Pleroma.Instances.get_cached_favicon() + |> Pleroma.Instances.Instance.get_or_update_favicon() |> MediaProxy.url() %{ -- cgit v1.2.3 From 8c9df2d2e6a5a3639f6b411cd3e9b57248a0650c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 7 Jul 2020 12:07:30 +0200 Subject: instance: Prevent loop of updates --- lib/pleroma/instances/instance.ex | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index b97e229e5..a1f935232 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -127,30 +127,23 @@ defmodule Pleroma.Instances.Instance do existing_record = Repo.get_by(Instance, %{host: host}) now = NaiveDateTime.utc_now() - if existing_record && existing_record.favicon && + if existing_record && existing_record.favicon_updated_at && NaiveDateTime.diff(now, existing_record.favicon_updated_at) < 86_400 do existing_record.favicon else favicon = scrape_favicon(instance_uri) - cond do - is_binary(favicon) && existing_record -> - existing_record - |> changeset(%{favicon: favicon, favicon_updated_at: now}) - |> Repo.update() - - favicon - - is_binary(favicon) -> - %Instance{} - |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) - |> Repo.insert() - - favicon - - true -> - nil + if existing_record do + existing_record + |> changeset(%{favicon: favicon, favicon_updated_at: now}) + |> Repo.update() + else + %Instance{} + |> changeset(%{host: host, favicon: favicon, favicon_updated_at: now}) + |> Repo.insert() end + + favicon end end -- cgit v1.2.3 From 312fc55f14e1b7f88ec43b72c577bf5df595beac Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 8 Jul 2020 05:56:24 +0200 Subject: Add [:instances_favicons, :enabled] setting, defaults to false --- lib/pleroma/web/mastodon_api/views/account_view.ex | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index db5739254..2feba4778 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -205,12 +205,16 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end favicon = - user - |> Map.get(:ap_id, "") - |> URI.parse() - |> URI.merge("/") - |> Pleroma.Instances.Instance.get_or_update_favicon() - |> MediaProxy.url() + if Pleroma.Config.get([:instances_favicons, :enabled]) do + user + |> Map.get(:ap_id, "") + |> URI.parse() + |> URI.merge("/") + |> Pleroma.Instances.Instance.get_or_update_favicon() + |> MediaProxy.url() + else + nil + end %{ id: to_string(user.id), -- cgit v1.2.3 From 31fef95e35d3cbc2d65c76a57ba8f4047809ce1b Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 8 Jul 2020 06:09:39 +0200 Subject: Add changelog and documentation --- lib/pleroma/web/api_spec/schemas/account.ex | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/account.ex b/lib/pleroma/web/api_spec/schemas/account.ex index 84f18f1b6..e6f163cb7 100644 --- a/lib/pleroma/web/api_spec/schemas/account.ex +++ b/lib/pleroma/web/api_spec/schemas/account.ex @@ -102,6 +102,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Account do type: :object, description: "A generic map of settings for frontends. Opaque to the backend. Only returned in `verify_credentials` and `update_credentials`" + }, + favicon: %Schema{ + type: :string, + format: :uri, + nullable: true, + description: "Favicon image of the user's instance" } } }, -- cgit v1.2.3 From e341f817850e51a29ec45607563323b6660f8da4 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 7 Jul 2020 09:10:02 +0300 Subject: fixed delete `Like` activity in remove user --- lib/pleroma/web/activity_pub/side_effects.ex | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 61feeae4d..70746f341 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -209,14 +209,20 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, object} end - def handle_undoing(%{data: %{"type" => "Like"}} = object) do - with %Object{} = liked_object <- Object.get_by_ap_id(object.data["object"]), - {:ok, _} <- Utils.remove_like_from_object(object, liked_object), - {:ok, _} <- Repo.delete(object) do - :ok + defp undo_like(nil, object), do: delete_object(object) + + defp undo_like(%Object{} = liked_object, object) do + with {:ok, _} <- Utils.remove_like_from_object(object, liked_object) do + delete_object(object) end end + def handle_undoing(%{data: %{"type" => "Like"}} = object) do + object.data["object"] + |> Object.get_by_ap_id() + |> undo_like(object) + end + def handle_undoing(%{data: %{"type" => "EmojiReact"}} = object) do with %Object{} = reacted_object <- Object.get_by_ap_id(object.data["object"]), {:ok, _} <- Utils.remove_emoji_reaction_from_object(object, reacted_object), @@ -246,6 +252,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do def handle_undoing(object), do: {:error, ["don't know how to handle", object]} + @spec delete_object(Object.t()) :: :ok | {:error, Ecto.Changeset.t()} + defp delete_object(object) do + with {:ok, _} <- Repo.delete(object), do: :ok + end + defp send_notifications(meta) do Keyword.get(meta, :notifications, []) |> Enum.each(fn notification -> -- cgit v1.2.3 From 3f8370a285d1ee705e4899656b88c442c50cb99b Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 8 Jul 2020 12:36:44 +0300 Subject: [#1895] Applied code review suggestion. --- .../web/mastodon_api/controllers/timeline_controller.ex | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index 4bbb82c23..a23f47e54 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -88,15 +88,12 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do ) end - defp restrict_unauthenticated?(local_only) do - config_key = - if local_only do - :local - else - :federated - end - - Pleroma.Config.get([:restrict_unauthenticated, :timelines, config_key]) + defp restrict_unauthenticated?(_local_only = true) do + Pleroma.Config.get([:restrict_unauthenticated, :timelines, :local]) + end + + defp restrict_unauthenticated?(_) do + Pleroma.Config.get([:restrict_unauthenticated, :timelines, :federated]) end # GET /api/v1/timelines/public -- cgit v1.2.3 From c0385cf47ae9c2dac527387225dee7d45dd33d8c Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 Jul 2020 11:52:29 +0200 Subject: AccountController: Fix muting / unmuting reblogs. --- .../web/api_spec/operations/account_operation.ex | 23 +++++++++++++++------- .../mastodon_api/controllers/account_controller.ex | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 9bde8fc0d..989bab122 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -203,14 +203,23 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do security: [%{"oAuth" => ["follow", "write:follows"]}], description: "Follow the given account", parameters: [ - %Reference{"$ref": "#/components/parameters/accountIdOrNickname"}, - Operation.parameter( - :reblogs, - :query, - BooleanLike, - "Receive this account's reblogs in home timeline? Defaults to true." - ) + %Reference{"$ref": "#/components/parameters/accountIdOrNickname"} ], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + properties: %{ + reblogs: %Schema{ + type: :boolean, + description: "Receive this account's reblogs in home timeline? Defaults to true.", + default: true + } + } + }, + required: false + ), responses: %{ 200 => Operation.response("Relationship", "application/json", AccountRelationship), 400 => Operation.response("Error", "application/json", ApiError), diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index d4532258c..fd89faf02 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -353,7 +353,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do {:error, "Can not follow yourself"} end - def follow(%{assigns: %{user: follower, account: followed}} = conn, params) do + def follow(%{body_params: params, assigns: %{user: follower, account: followed}} = conn, _) do with {:ok, follower} <- MastodonAPI.follow(follower, followed, params) do render(conn, "relationship.json", user: follower, target: followed) else -- cgit v1.2.3 From a6495f4a686feb3d4728432dd075f425c04c32dd Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 8 Jul 2020 12:54:23 +0300 Subject: [#1895] credo fix. --- lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex index a23f47e54..ab7b1d6aa 100644 --- a/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/timeline_controller.ex @@ -88,7 +88,7 @@ defmodule Pleroma.Web.MastodonAPI.TimelineController do ) end - defp restrict_unauthenticated?(_local_only = true) do + defp restrict_unauthenticated?(true = _local_only) do Pleroma.Config.get([:restrict_unauthenticated, :timelines, :local]) end -- cgit v1.2.3 From 704a3830556d94e0dbc39873480e9ba95a143be9 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 8 Jul 2020 13:14:18 +0300 Subject: Improved search results for localized nickname match. Tweaked user search to rank nickname matches higher than name matches. --- lib/pleroma/user/search.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 42ff1de78..7ff1c7e24 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -88,15 +88,21 @@ defmodule Pleroma.User.Search do |> Enum.join(" | ") end + # Considers nickname match, localized nickname match, name match; preferences nickname match defp trigram_rank(query, query_string) do from( u in query, select_merge: %{ search_rank: fragment( - "similarity(?, trim(? || ' ' || coalesce(?, '')))", + "similarity(?, ?) + \ + similarity(?, regexp_replace(?, '@.+', '')) + \ + similarity(?, trim(coalesce(?, '')))", ^query_string, u.nickname, + ^query_string, + u.nickname, + ^query_string, u.name ) } -- cgit v1.2.3 From 29fa75d00d1f550461b2ab1e59554e134208d419 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 Jul 2020 14:29:29 +0200 Subject: Notification: For follows, notify the followed. --- lib/pleroma/notification.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index fcb2144ae..32bcfcaba 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -497,6 +497,10 @@ defmodule Pleroma.Notification do end end + def get_potential_receiver_ap_ids(%{data: %{"type" => "Follow", "object" => object_id}}) do + [object_id] + end + def get_potential_receiver_ap_ids(activity) do [] |> Utils.maybe_notify_to_recipients(activity) -- cgit v1.2.3 From 172f4aff8ef573c54902dc8fa135d69f50fea47c Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 Jul 2020 14:30:53 +0200 Subject: Transmogrifier: Move following to the pipeline. --- .../object_validators/follow_validator.ex | 2 + lib/pleroma/web/activity_pub/side_effects.ex | 60 +++++++++++++++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 62 +--------------------- 3 files changed, 63 insertions(+), 61 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex index 2035ad9ba..ca2724616 100644 --- a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do field(:to, ObjectValidators.Recipients, default: []) field(:cc, ObjectValidators.Recipients, default: []) field(:object, ObjectValidators.ObjectID) + field(:state, :string, default: "pending") end def cast_data(data) do @@ -30,6 +31,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do cng |> validate_required([:id, :type, :actor, :to, :cc, :object]) |> validate_inclusion(:type, ["Follow"]) + |> validate_inclusion(:state, ~w{pending reject accept}) |> validate_actor_presence() |> validate_actor_presence(field_name: :object) end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 61feeae4d..284560913 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Activity.Ir.Topics alias Pleroma.Chat alias Pleroma.Chat.MessageReference + alias Pleroma.FollowingRelationship alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo @@ -21,6 +22,65 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do def handle(object, meta \\ []) + # Tasks this handle + # - Follows if possible + # - Sends a notification + # - Generates accept or reject if appropriate + def handle( + %{ + data: %{ + "id" => follow_id, + "type" => "Follow", + "object" => followed_user, + "actor" => following_user + } + } = object, + meta + ) do + with %User{} = follower <- User.get_cached_by_ap_id(following_user), + %User{} = followed <- User.get_cached_by_ap_id(followed_user), + {_, {:ok, _}, _, _} <- + {:following, User.follow(follower, followed, :follow_pending), follower, followed} do + if followed.local && !followed.locked do + Utils.update_follow_state_for_all(object, "accept") + FollowingRelationship.update(follower, followed, :follow_accept) + + %{ + to: [following_user], + actor: followed, + object: follow_id, + local: true + } + |> ActivityPub.accept() + end + else + {:following, {:error, _}, follower, followed} -> + Utils.update_follow_state_for_all(object, "reject") + FollowingRelationship.update(follower, followed, :follow_reject) + + if followed.local do + %{ + to: [follower.ap_id], + actor: followed, + object: follow_id, + local: true + } + |> ActivityPub.reject() + end + + _ -> + nil + end + + {:ok, notifications} = Notification.create_notifications(object, do_send: false) + + meta = + meta + |> add_notifications(notifications) + + {:ok, object, meta} + end + # Tasks this handles: # - Unfollow and block def handle( diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 117e930b3..884646ceb 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -529,66 +529,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def handle_incoming( - %{"type" => "Follow", "object" => followed, "actor" => follower, "id" => id} = data, - _options - ) do - with %User{local: true} = followed <- - User.get_cached_by_ap_id(Containment.get_actor(%{"actor" => followed})), - {:ok, %User{} = follower} <- - User.get_or_fetch_by_ap_id(Containment.get_actor(%{"actor" => follower})), - {:ok, activity} <- - ActivityPub.follow(follower, followed, id, false, skip_notify_and_stream: true) do - with deny_follow_blocked <- Pleroma.Config.get([:user, :deny_follow_blocked]), - {_, false} <- {:user_blocked, User.blocks?(followed, follower) && deny_follow_blocked}, - {_, false} <- {:user_locked, User.locked?(followed)}, - {_, {:ok, follower}} <- {:follow, User.follow(follower, followed)}, - {_, {:ok, _}} <- - {:follow_state_update, Utils.update_follow_state_for_all(activity, "accept")}, - {:ok, _relationship} <- - FollowingRelationship.update(follower, followed, :follow_accept) do - ActivityPub.accept(%{ - to: [follower.ap_id], - actor: followed, - object: data, - local: true - }) - else - {:user_blocked, true} -> - {:ok, _} = Utils.update_follow_state_for_all(activity, "reject") - {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_reject) - - ActivityPub.reject(%{ - to: [follower.ap_id], - actor: followed, - object: data, - local: true - }) - - {:follow, {:error, _}} -> - {:ok, _} = Utils.update_follow_state_for_all(activity, "reject") - {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_reject) - - ActivityPub.reject(%{ - to: [follower.ap_id], - actor: followed, - object: data, - local: true - }) - - {:user_locked, true} -> - {:ok, _relationship} = FollowingRelationship.update(follower, followed, :follow_pending) - :noop - end - - ActivityPub.notify_and_stream(activity) - {:ok, activity} - else - _e -> - :error - end - end - def handle_incoming( %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => id} = data, _options @@ -696,7 +636,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => type} = data, _options ) - when type in ~w{Update Block} do + when type in ~w{Update Block Follow} do with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} -- cgit v1.2.3 From 72ad3a66f48d4500be1f25dd7b02b834399d3bbe Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 3 Jul 2020 19:18:08 +0300 Subject: don't fully start pleroma in mix tasks --- lib/mix/pleroma.ex | 20 +++++++++++++++++++- lib/mix/tasks/pleroma/digest.ex | 2 ++ lib/mix/tasks/pleroma/email.ex | 1 + lib/mix/tasks/pleroma/relay.ex | 3 +++ lib/pleroma/application.ex | 3 ++- 5 files changed, 27 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 3ad6edbfb..553c74c25 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -3,6 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Pleroma do + @apps [:restarter, :ecto, :ecto_sql, :postgrex, :db_connection, :cachex] + @cachex_childs ["object", "user"] @doc "Common functions to be reused in mix tasks" def start_pleroma do Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) @@ -11,7 +13,23 @@ defmodule Mix.Pleroma do Application.put_env(:logger, :console, level: :debug) end - {:ok, _} = Application.ensure_all_started(:pleroma) + apps = + if Application.get_env(:tesla, :adapter) == Tesla.Adapter.Gun do + [:gun | @apps] + else + [:hackney | @apps] + end + + Enum.each(apps, &Application.ensure_all_started/1) + + childs = [Pleroma.Repo, Pleroma.Config.TransferTask, Pleroma.Web.Endpoint] + + cachex_childs = Enum.map(@cachex_childs, &Pleroma.Application.build_cachex(&1, [])) + + Supervisor.start_link(childs ++ cachex_childs, + strategy: :one_for_one, + name: Pleroma.Supervisor + ) if Pleroma.Config.get(:env) not in [:test, :benchmark] do pleroma_rebooted?() diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex index 3595f912d..8bde2d4f2 100644 --- a/lib/mix/tasks/pleroma/digest.ex +++ b/lib/mix/tasks/pleroma/digest.ex @@ -7,6 +7,8 @@ defmodule Mix.Tasks.Pleroma.Digest do def run(["test", nickname | opts]) do Mix.Pleroma.start_pleroma() + Application.ensure_all_started(:timex) + Application.ensure_all_started(:swoosh) user = Pleroma.User.get_by_nickname(nickname) diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex index d3fac6ec8..16fe31431 100644 --- a/lib/mix/tasks/pleroma/email.ex +++ b/lib/mix/tasks/pleroma/email.ex @@ -7,6 +7,7 @@ defmodule Mix.Tasks.Pleroma.Email do def run(["test" | args]) do Mix.Pleroma.start_pleroma() + Application.ensure_all_started(:swoosh) {options, [], []} = OptionParser.parse( diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index c3312507e..b67d256c3 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -12,6 +12,7 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["follow", target]) do start_pleroma() + Application.ensure_all_started(:flake_id) with {:ok, _activity} <- Relay.follow(target) do # put this task to sleep to allow the genserver to push out the messages @@ -23,6 +24,7 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["unfollow", target]) do start_pleroma() + Application.ensure_all_started(:flake_id) with {:ok, _activity} <- Relay.unfollow(target) do # put this task to sleep to allow the genserver to push out the messages @@ -34,6 +36,7 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["list"]) do start_pleroma() + Application.ensure_all_started(:flake_id) with {:ok, list} <- Relay.list(true) do list |> Enum.each(&shell_info(&1)) diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9615af122..7eb629abf 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -162,7 +162,8 @@ defmodule Pleroma.Application do defp seconds_valid_interval, do: :timer.seconds(Config.get!([Pleroma.Captcha, :seconds_valid])) - defp build_cachex(type, opts), + @spec build_cachex(String.t(), keyword()) :: map() + def build_cachex(type, opts), do: %{ id: String.to_atom("cachex_" <> type), start: {Cachex, :start_link, [String.to_atom(type <> "_cache"), opts]}, -- cgit v1.2.3 From b28cc154596ad9cbf5ef9708a5967672c61ddbdc Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 3 Jul 2020 20:12:00 +0300 Subject: don't restart pleroma in mix tasks --- lib/mix/pleroma.ex | 6 +++++- lib/pleroma/config/transfer_task.ex | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 553c74c25..0fbb6f1cd 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -22,7 +22,11 @@ defmodule Mix.Pleroma do Enum.each(apps, &Application.ensure_all_started/1) - childs = [Pleroma.Repo, Pleroma.Config.TransferTask, Pleroma.Web.Endpoint] + childs = [ + Pleroma.Repo, + {Pleroma.Config.TransferTask, false}, + Pleroma.Web.Endpoint + ] cachex_childs = Enum.map(@cachex_childs, &Pleroma.Application.build_cachex(&1, [])) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index eb86b8ff4..a0d7b7d71 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -31,8 +31,8 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, :gopher, [:enabled]} ] - def start_link(_) do - load_and_update_env() + def start_link(restart_pleroma? \\ true) do + load_and_update_env([], restart_pleroma?) if Config.get(:env) == :test, do: Ecto.Adapters.SQL.Sandbox.checkin(Repo) :ignore end -- cgit v1.2.3 From 9dda8b542723afae8dd5493b4082b8524873a14d Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 Jul 2020 15:40:56 +0200 Subject: CommonAPI: Switch to pipeline for following. --- lib/pleroma/web/activity_pub/side_effects.ex | 6 +++++- lib/pleroma/web/common_api/common_api.ex | 10 +++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 284560913..de02baf0f 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -44,6 +44,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if followed.local && !followed.locked do Utils.update_follow_state_for_all(object, "accept") FollowingRelationship.update(follower, followed, :follow_accept) + User.update_follower_count(followed) + User.update_following_count(follower) %{ to: [following_user], @@ -78,7 +80,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do meta |> add_notifications(notifications) - {:ok, object, meta} + updated_object = Activity.get_by_ap_id(follow_id) + + {:ok, updated_object, meta} end # Tasks this handles: diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index fd7149079..4d5b0decf 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -101,10 +101,14 @@ defmodule Pleroma.Web.CommonAPI do def follow(follower, followed) do timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout]) - with {:ok, follower} <- User.maybe_direct_follow(follower, followed), - {:ok, activity} <- ActivityPub.follow(follower, followed), + with {:ok, follow_data, _} <- Builder.follow(follower, followed), + {:ok, activity, _} <- Pipeline.common_pipeline(follow_data, local: true), {:ok, follower, followed} <- User.wait_and_refresh(timeout, follower, followed) do - {:ok, follower, followed, activity} + if activity.data["state"] == "reject" do + {:error, :rejected} + else + {:ok, follower, followed, activity} + end end end -- cgit v1.2.3 From 00e54f8fe7af098ba829f7f7cd5511569dcd1c0a Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 Jul 2020 17:07:24 +0200 Subject: ActivityPub: Remove `follow` and fix issues. --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 22 ---------------------- lib/pleroma/web/activity_pub/relay.ex | 2 +- 3 files changed, 2 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e98332744..9d1314f81 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1543,7 +1543,7 @@ defmodule Pleroma.User do fn followed_identifier -> with {:ok, %User{} = followed} <- get_or_fetch(followed_identifier), {:ok, follower} <- maybe_direct_follow(follower, followed), - {:ok, _} <- ActivityPub.follow(follower, followed) do + {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do followed else err -> diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8abbef487..1c2908805 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -322,28 +322,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - @spec follow(User.t(), User.t(), String.t() | nil, boolean(), keyword()) :: - {:ok, Activity.t()} | {:error, any()} - def follow(follower, followed, activity_id \\ nil, local \\ true, opts \\ []) do - with {:ok, result} <- - Repo.transaction(fn -> do_follow(follower, followed, activity_id, local, opts) end) do - result - end - end - - defp do_follow(follower, followed, activity_id, local, opts) do - skip_notify_and_stream = Keyword.get(opts, :skip_notify_and_stream, false) - data = make_follow_data(follower, followed, activity_id) - - with {:ok, activity} <- insert(data, local), - _ <- skip_notify_and_stream || notify_and_stream(activity), - :ok <- maybe_federate(activity) do - {:ok, activity} - else - {:error, error} -> Repo.rollback(error) - end - end - @spec unfollow(User.t(), User.t(), String.t() | nil, boolean()) :: {:ok, Activity.t()} | nil | {:error, any()} def unfollow(follower, followed, activity_id \\ nil, local \\ true) do diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index 484178edd..b09764d2b 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do def follow(target_instance) do with %User{} = local_user <- get_actor(), {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance), - {:ok, activity} <- ActivityPub.follow(local_user, target_user) do + {:ok, _, _, activity} <- CommonAPI.follow(local_user, target_user) do Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}") {:ok, activity} else -- cgit v1.2.3 From 123352ffa1c80aab658fca0c2276d1c06de43a02 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 8 Jul 2020 22:50:15 +0300 Subject: Removed unused trigram index on `users`. Fixed `users_fts_index` usage. --- lib/pleroma/user/search.ex | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex index 7ff1c7e24..d4fd31069 100644 --- a/lib/pleroma/user/search.ex +++ b/lib/pleroma/user/search.ex @@ -69,11 +69,15 @@ defmodule Pleroma.User.Search do u in query, where: fragment( + # The fragment must _exactly_ match `users_fts_index`, otherwise the index won't work """ - (to_tsvector('simple', ?) || to_tsvector('simple', ?)) @@ to_tsquery('simple', ?) + ( + setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') || + setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B') + ) @@ to_tsquery('simple', ?) """, - u.name, u.nickname, + u.name, ^query_string ) ) @@ -95,9 +99,11 @@ defmodule Pleroma.User.Search do select_merge: %{ search_rank: fragment( - "similarity(?, ?) + \ - similarity(?, regexp_replace(?, '@.+', '')) + \ - similarity(?, trim(coalesce(?, '')))", + """ + similarity(?, ?) + + similarity(?, regexp_replace(?, '@.+', '')) + + similarity(?, trim(coalesce(?, ''))) + """, ^query_string, u.nickname, ^query_string, -- cgit v1.2.3 From 4cbafcef0c3a0ce9ddf888b558c6691afb96ad94 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Jul 2020 11:17:43 +0300 Subject: load default config in mix tasks --- lib/mix/pleroma.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 0fbb6f1cd..de16cc52c 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -7,6 +7,7 @@ defmodule Mix.Pleroma do @cachex_childs ["object", "user"] @doc "Common functions to be reused in mix tasks" def start_pleroma do + Pleroma.Config.Holder.save_default() Application.put_env(:phoenix, :serve_endpoints, false, persistent: true) if Pleroma.Config.get(:env) != :test do -- cgit v1.2.3 From c2be0da79fcc95399647b292e0b9a4119d3dcdf1 Mon Sep 17 00:00:00 2001 From: eugenijm Date: Mon, 18 May 2020 22:56:09 +0300 Subject: Admin API: fix `GET /api/pleroma/admin/users/:nickname/credentials` returning 404 when getting the credentials of a remote user while `:instance, :limit_to_local_content` is set to `:unauthenticated` --- lib/pleroma/web/admin_api/controllers/admin_api_controller.ex | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index f9545d895..e5f14269a 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -206,8 +206,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do end end - def user_show(conn, %{"nickname" => nickname}) do - with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do + def user_show(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do conn |> put_view(AccountView) |> render("show.json", %{user: user}) @@ -233,11 +233,11 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do |> render("index.json", %{activities: activities, as: :activity}) end - def list_user_statuses(conn, %{"nickname" => nickname} = params) do + def list_user_statuses(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname} = params) do with_reblogs = params["with_reblogs"] == "true" || params["with_reblogs"] == true godmode = params["godmode"] == "true" || params["godmode"] == true - with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do {_, page_size} = page_params(params) activities = @@ -526,7 +526,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do @doc "Show a given user's credentials" def show_user_credentials(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do - with %User{} = user <- User.get_cached_by_nickname_or_id(nickname) do + with %User{} = user <- User.get_cached_by_nickname_or_id(nickname, for: admin) do conn |> put_view(AccountView) |> render("credentials.json", %{user: user, for: admin}) -- cgit v1.2.3 From d23804f191eb9e14cfb087863320ae90653c9544 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 9 Jul 2020 10:53:51 -0500 Subject: Use the Pleroma.Config alias --- lib/mix/tasks/pleroma/instance.ex | 4 +- lib/pleroma/application.ex | 2 +- lib/pleroma/emails/admin_email.ex | 2 +- lib/pleroma/emoji/loader.ex | 2 +- lib/pleroma/plugs/http_security_plug.ex | 4 +- lib/pleroma/user.ex | 52 +++++++++++----------- .../web/activity_pub/mrf/object_age_policy.ex | 2 +- .../web/activity_pub/mrf/reject_non_public.ex | 2 +- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 6 +-- lib/pleroma/web/media_proxy/media_proxy.ex | 2 +- .../web/twitter_api/controllers/util_controller.ex | 2 +- 12 files changed, 41 insertions(+), 41 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 86409738a..91440b453 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -145,7 +145,7 @@ defmodule Mix.Tasks.Pleroma.Instance do options, :uploads_dir, "What directory should media uploads go in (when using the local uploader)?", - Pleroma.Config.get([Pleroma.Uploaders.Local, :uploads]) + Config.get([Pleroma.Uploaders.Local, :uploads]) ) |> Path.expand() @@ -154,7 +154,7 @@ defmodule Mix.Tasks.Pleroma.Instance do options, :static_dir, "What directory should custom public files be read from (custom emojis, frontend bundle overrides, robots.txt, etc.)?", - Pleroma.Config.get([:instance, :static_dir]) + Config.get([:instance, :static_dir]) ) |> Path.expand() diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9615af122..32773d3c9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,7 +35,7 @@ defmodule Pleroma.Application do # See http://elixir-lang.org/docs/stable/elixir/Application.html # for more information on OTP Applications def start(_type, _args) do - Pleroma.Config.Holder.save_default() + Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() Config.DeprecationWarnings.warn() Pleroma.Plugs.HTTPSecurityPlug.warn_if_disabled() diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index 55f61024e..c67ba63ad 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Emails.AdminEmail do alias Pleroma.Config alias Pleroma.Web.Router.Helpers - defp instance_config, do: Pleroma.Config.get(:instance) + defp instance_config, do: Config.get(:instance) defp instance_name, do: instance_config()[:name] defp instance_notify_email do diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index 3de2dc762..03a6bca0b 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -108,7 +108,7 @@ defmodule Pleroma.Emoji.Loader do if File.exists?(emoji_txt) do load_from_file(emoji_txt, emoji_groups) else - extensions = Pleroma.Config.get([:emoji, :pack_extensions]) + extensions = Config.get([:emoji, :pack_extensions]) Logger.info( "No emoji.txt found for pack \"#{pack_name}\", assuming all #{ diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 472a3ff42..7d65cf078 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -82,14 +82,14 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] connect_src = - if Pleroma.Config.get(:env) == :dev do + if Config.get(:env) == :dev do [connect_src, " http://localhost:3035/"] else connect_src end script_src = - if Pleroma.Config.get(:env) == :dev do + if Config.get(:env) == :dev do "script-src 'self' 'unsafe-eval'" else "script-src 'self'" diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9d1314f81..0078f9831 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -388,8 +388,8 @@ defmodule Pleroma.User do defp fix_follower_address(params), do: params def remote_user_changeset(struct \\ %User{local: false}, params) do - bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) - name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) + bio_limit = Config.get([:instance, :user_bio_length], 5000) + name_limit = Config.get([:instance, :user_name_length], 100) name = case params[:name] do @@ -448,8 +448,8 @@ defmodule Pleroma.User do end def update_changeset(struct, params \\ %{}) do - bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) - name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) + bio_limit = Config.get([:instance, :user_bio_length], 5000) + name_limit = Config.get([:instance, :user_name_length], 100) struct |> cast( @@ -618,12 +618,12 @@ defmodule Pleroma.User do def force_password_reset(user), do: update_password_reset_pending(user, true) def register_changeset(struct, params \\ %{}, opts \\ []) do - bio_limit = Pleroma.Config.get([:instance, :user_bio_length], 5000) - name_limit = Pleroma.Config.get([:instance, :user_name_length], 100) + bio_limit = Config.get([:instance, :user_bio_length], 5000) + name_limit = Config.get([:instance, :user_name_length], 100) need_confirmation? = if is_nil(opts[:need_confirmation]) do - Pleroma.Config.get([:instance, :account_activation_required]) + Config.get([:instance, :account_activation_required]) else opts[:need_confirmation] end @@ -644,7 +644,7 @@ defmodule Pleroma.User do |> validate_confirmation(:password) |> unique_constraint(:email) |> unique_constraint(:nickname) - |> validate_exclusion(:nickname, Pleroma.Config.get([User, :restricted_nicknames])) + |> validate_exclusion(:nickname, Config.get([User, :restricted_nicknames])) |> validate_format(:nickname, local_nickname_regex()) |> validate_format(:email, @email_regex) |> validate_length(:bio, max: bio_limit) @@ -659,7 +659,7 @@ defmodule Pleroma.User do def maybe_validate_required_email(changeset, true), do: changeset def maybe_validate_required_email(changeset, _) do - if Pleroma.Config.get([:instance, :account_activation_required]) do + if Config.get([:instance, :account_activation_required]) do validate_required(changeset, [:email]) else changeset @@ -679,7 +679,7 @@ defmodule Pleroma.User do end defp autofollow_users(user) do - candidates = Pleroma.Config.get([:instance, :autofollowed_nicknames]) + candidates = Config.get([:instance, :autofollowed_nicknames]) autofollowed_users = User.Query.build(%{nickname: candidates, local: true, deactivated: false}) @@ -706,7 +706,7 @@ defmodule Pleroma.User do def try_send_confirmation_email(%User{} = user) do if user.confirmation_pending && - Pleroma.Config.get([:instance, :account_activation_required]) do + Config.get([:instance, :account_activation_required]) do user |> Pleroma.Emails.UserEmail.account_confirmation_email() |> Pleroma.Emails.Mailer.deliver_async() @@ -763,7 +763,7 @@ defmodule Pleroma.User do defdelegate following(user), to: FollowingRelationship def follow(%User{} = follower, %User{} = followed, state \\ :follow_accept) do - deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) + deny_follow_blocked = Config.get([:user, :deny_follow_blocked]) cond do followed.deactivated -> @@ -964,7 +964,7 @@ defmodule Pleroma.User do end def get_cached_by_nickname_or_id(nickname_or_id, opts \\ []) do - restrict_to_local = Pleroma.Config.get([:instance, :limit_to_local_content]) + restrict_to_local = Config.get([:instance, :limit_to_local_content]) cond do is_integer(nickname_or_id) or FlakeId.flake_id?(nickname_or_id) -> @@ -1160,7 +1160,7 @@ defmodule Pleroma.User do @spec update_follower_count(User.t()) :: {:ok, User.t()} def update_follower_count(%User{} = user) do - if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do + if user.local or !Config.get([:instance, :external_user_synchronization]) do follower_count = FollowingRelationship.follower_count(user) user @@ -1173,7 +1173,7 @@ defmodule Pleroma.User do @spec update_following_count(User.t()) :: {:ok, User.t()} def update_following_count(%User{local: false} = user) do - if Pleroma.Config.get([:instance, :external_user_synchronization]) do + if Config.get([:instance, :external_user_synchronization]) do {:ok, maybe_fetch_follow_information(user)} else {:ok, user} @@ -1260,7 +1260,7 @@ defmodule Pleroma.User do end def subscribe(%User{} = subscriber, %User{} = target) do - deny_follow_blocked = Pleroma.Config.get([:user, :deny_follow_blocked]) + deny_follow_blocked = Config.get([:user, :deny_follow_blocked]) if blocks?(target, subscriber) and deny_follow_blocked do {:error, "Could not subscribe: #{target.nickname} is blocking you"} @@ -1651,7 +1651,7 @@ defmodule Pleroma.User do Pleroma.HTML.Scrubber.TwitterText end - def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy]) + def html_filter_policy(_), do: Config.get([:markup, :scrub_policy]) def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) @@ -1833,7 +1833,7 @@ defmodule Pleroma.User do end defp local_nickname_regex do - if Pleroma.Config.get([:instance, :extended_nickname_format]) do + if Config.get([:instance, :extended_nickname_format]) do @extended_local_nickname_regex else @strict_local_nickname_regex @@ -1961,8 +1961,8 @@ defmodule Pleroma.User do def get_mascot(%{mascot: mascot}) when is_nil(mascot) do # use instance-default - config = Pleroma.Config.get([:assets, :mascots]) - default_mascot = Pleroma.Config.get([:assets, :default_mascot]) + config = Config.get([:assets, :mascots]) + default_mascot = Config.get([:assets, :default_mascot]) mascot = Keyword.get(config, default_mascot) %{ @@ -2057,7 +2057,7 @@ defmodule Pleroma.User do def validate_fields(changeset, remote? \\ false) do limit_name = if remote?, do: :max_remote_account_fields, else: :max_account_fields - limit = Pleroma.Config.get([:instance, limit_name], 0) + limit = Config.get([:instance, limit_name], 0) changeset |> validate_length(:fields, max: limit) @@ -2071,8 +2071,8 @@ defmodule Pleroma.User do end defp valid_field?(%{"name" => name, "value" => value}) do - name_limit = Pleroma.Config.get([:instance, :account_field_name_length], 255) - value_limit = Pleroma.Config.get([:instance, :account_field_value_length], 255) + name_limit = Config.get([:instance, :account_field_name_length], 255) + value_limit = Config.get([:instance, :account_field_value_length], 255) is_binary(name) && is_binary(value) && String.length(name) <= name_limit && String.length(value) <= value_limit @@ -2082,10 +2082,10 @@ defmodule Pleroma.User do defp truncate_field(%{"name" => name, "value" => value}) do {name, _chopped} = - String.split_at(name, Pleroma.Config.get([:instance, :account_field_name_length], 255)) + String.split_at(name, Config.get([:instance, :account_field_name_length], 255)) {value, _chopped} = - String.split_at(value, Pleroma.Config.get([:instance, :account_field_value_length], 255)) + String.split_at(value, Config.get([:instance, :account_field_value_length], 255)) %{"name" => name, "value" => value} end @@ -2140,7 +2140,7 @@ defmodule Pleroma.User do def add_pinnned_activity(user, %Pleroma.Activity{id: id}) do if id not in user.pinned_activities do - max_pinned_statuses = Pleroma.Config.get([:instance, :max_pinned_statuses], 0) + max_pinned_statuses = Config.get([:instance, :max_pinned_statuses], 0) params = %{pinned_activities: user.pinned_activities ++ [id]} user diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index b0ccb63c8..a62914135 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -98,7 +98,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do @impl true def describe do mrf_object_age = - Pleroma.Config.get(:mrf_object_age) + Config.get(:mrf_object_age) |> Enum.into(%{}) {:ok, %{mrf_object_age: mrf_object_age}} diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 3092f3272..4fd63106d 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -47,5 +47,5 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do @impl true def describe, - do: {:ok, %{mrf_rejectnonpublic: Pleroma.Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}} + do: {:ok, %{mrf_rejectnonpublic: Config.get(:mrf_rejectnonpublic) |> Enum.into(%{})}} end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 9cea6bcf9..70a2ca053 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -155,7 +155,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do %{host: actor_host} = URI.parse(actor) reject_deletes = - Pleroma.Config.get([:mrf_simple, :reject_deletes]) + Config.get([:mrf_simple, :reject_deletes]) |> MRF.subdomains_regex() if MRF.subdomain_match?(reject_deletes, actor_host) do diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 15594125f..9c38b73eb 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -143,7 +143,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_poll_data(%{poll: %{options: options, expires_in: expires_in}} = data) when is_list(options) do - limits = Pleroma.Config.get([:instance, :poll_limits]) + limits = Config.get([:instance, :poll_limits]) with :ok <- validate_poll_expiration(expires_in, limits), :ok <- validate_poll_options_amount(options, limits), @@ -502,7 +502,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_report_content_html(nil), do: {:ok, {nil, [], []}} def make_report_content_html(comment) do - max_size = Pleroma.Config.get([:instance, :max_report_comment_size], 1000) + max_size = Config.get([:instance, :max_report_comment_size], 1000) if String.length(comment) <= max_size do {:ok, format_input(comment, "text/plain")} @@ -564,7 +564,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def validate_character_limit(full_payload, _attachments) do - limit = Pleroma.Config.get([:instance, :limit]) + limit = Config.get([:instance, :limit]) length = String.length(full_payload) if length <= limit do diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 077fabe47..6f35826da 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -106,7 +106,7 @@ defmodule Pleroma.Web.MediaProxy do def build_url(sig_base64, url_base64, filename \\ nil) do [ - Pleroma.Config.get([:media_proxy, :base_url], Web.base_url()), + Config.get([:media_proxy, :base_url], Web.base_url()), "proxy", sig_base64, url_base64, diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 8314e75b4..f02c4075c 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do def frontend_configurations(conn, _params) do config = - Pleroma.Config.get(:frontend_configurations, %{}) + Config.get(:frontend_configurations, %{}) |> Enum.into(%{}) json(conn, config) -- cgit v1.2.3 From d5fcec8315a5f48d94a083b38abcc615834dc532 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Jul 2020 18:59:48 +0300 Subject: fix for info after tag/untag user --- lib/mix/tasks/pleroma/user.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index bca7e87bf..a9370b5e7 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -15,6 +15,8 @@ defmodule Mix.Tasks.Pleroma.User do @moduledoc File.read!("docs/administration/CLI_tasks/user.md") def run(["new", nickname, email | rest]) do + Application.ensure_all_started(:flake_id) + {options, [], []} = OptionParser.parse( rest, @@ -97,6 +99,7 @@ defmodule Mix.Tasks.Pleroma.User do def run(["rm", nickname]) do start_pleroma() + Application.ensure_all_started(:flake_id) with %User{local: true} = user <- User.get_cached_by_nickname(nickname), {:ok, delete_data, _} <- Builder.delete(user, user.ap_id), @@ -232,7 +235,7 @@ defmodule Mix.Tasks.Pleroma.User do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.tag(tags) - shell_info("Tags of #{user.nickname}: #{inspect(tags)}") + shell_info("Tags of #{user.nickname}: #{inspect(user.tags)}") else _ -> shell_error("Could not change user tags for #{nickname}") @@ -245,7 +248,7 @@ defmodule Mix.Tasks.Pleroma.User do with %User{} = user <- User.get_cached_by_nickname(nickname) do user = user |> User.untag(tags) - shell_info("Tags of #{user.nickname}: #{inspect(tags)}") + shell_info("Tags of #{user.nickname}: #{inspect(user.tags)}") else _ -> shell_error("Could not change user tags for #{nickname}") @@ -328,6 +331,7 @@ defmodule Mix.Tasks.Pleroma.User do def run(["delete_activities", nickname]) do start_pleroma() + Application.ensure_all_started(:flake_id) with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do User.delete_user_activities(user) -- cgit v1.2.3 From 79707e879d3af359be9e1f6ac10717cc9cb72b2c Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 9 Jul 2020 19:13:16 +0300 Subject: cleap up --- lib/mix/pleroma.ex | 20 +++++++++++++++----- lib/mix/tasks/pleroma/digest.ex | 2 -- lib/mix/tasks/pleroma/email.ex | 1 - lib/mix/tasks/pleroma/relay.ex | 3 --- lib/mix/tasks/pleroma/user.ex | 4 ---- 5 files changed, 15 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index de16cc52c..9f0bf6ecb 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -3,8 +3,18 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Pleroma do - @apps [:restarter, :ecto, :ecto_sql, :postgrex, :db_connection, :cachex] - @cachex_childs ["object", "user"] + @apps [ + :restarter, + :ecto, + :ecto_sql, + :postgrex, + :db_connection, + :cachex, + :flake_id, + :swoosh, + :timex + ] + @cachex_children ["object", "user"] @doc "Common functions to be reused in mix tasks" def start_pleroma do Pleroma.Config.Holder.save_default() @@ -23,15 +33,15 @@ defmodule Mix.Pleroma do Enum.each(apps, &Application.ensure_all_started/1) - childs = [ + children = [ Pleroma.Repo, {Pleroma.Config.TransferTask, false}, Pleroma.Web.Endpoint ] - cachex_childs = Enum.map(@cachex_childs, &Pleroma.Application.build_cachex(&1, [])) + cachex_children = Enum.map(@cachex_children, &Pleroma.Application.build_cachex(&1, [])) - Supervisor.start_link(childs ++ cachex_childs, + Supervisor.start_link(children ++ cachex_children, strategy: :one_for_one, name: Pleroma.Supervisor ) diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex index 8bde2d4f2..3595f912d 100644 --- a/lib/mix/tasks/pleroma/digest.ex +++ b/lib/mix/tasks/pleroma/digest.ex @@ -7,8 +7,6 @@ defmodule Mix.Tasks.Pleroma.Digest do def run(["test", nickname | opts]) do Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:timex) - Application.ensure_all_started(:swoosh) user = Pleroma.User.get_by_nickname(nickname) diff --git a/lib/mix/tasks/pleroma/email.ex b/lib/mix/tasks/pleroma/email.ex index 16fe31431..d3fac6ec8 100644 --- a/lib/mix/tasks/pleroma/email.ex +++ b/lib/mix/tasks/pleroma/email.ex @@ -7,7 +7,6 @@ defmodule Mix.Tasks.Pleroma.Email do def run(["test" | args]) do Mix.Pleroma.start_pleroma() - Application.ensure_all_started(:swoosh) {options, [], []} = OptionParser.parse( diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex index b67d256c3..c3312507e 100644 --- a/lib/mix/tasks/pleroma/relay.ex +++ b/lib/mix/tasks/pleroma/relay.ex @@ -12,7 +12,6 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["follow", target]) do start_pleroma() - Application.ensure_all_started(:flake_id) with {:ok, _activity} <- Relay.follow(target) do # put this task to sleep to allow the genserver to push out the messages @@ -24,7 +23,6 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["unfollow", target]) do start_pleroma() - Application.ensure_all_started(:flake_id) with {:ok, _activity} <- Relay.unfollow(target) do # put this task to sleep to allow the genserver to push out the messages @@ -36,7 +34,6 @@ defmodule Mix.Tasks.Pleroma.Relay do def run(["list"]) do start_pleroma() - Application.ensure_all_started(:flake_id) with {:ok, list} <- Relay.list(true) do list |> Enum.each(&shell_info(&1)) diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index a9370b5e7..01824aa18 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -15,8 +15,6 @@ defmodule Mix.Tasks.Pleroma.User do @moduledoc File.read!("docs/administration/CLI_tasks/user.md") def run(["new", nickname, email | rest]) do - Application.ensure_all_started(:flake_id) - {options, [], []} = OptionParser.parse( rest, @@ -99,7 +97,6 @@ defmodule Mix.Tasks.Pleroma.User do def run(["rm", nickname]) do start_pleroma() - Application.ensure_all_started(:flake_id) with %User{local: true} = user <- User.get_cached_by_nickname(nickname), {:ok, delete_data, _} <- Builder.delete(user, user.ap_id), @@ -331,7 +328,6 @@ defmodule Mix.Tasks.Pleroma.User do def run(["delete_activities", nickname]) do start_pleroma() - Application.ensure_all_started(:flake_id) with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do User.delete_user_activities(user) -- cgit v1.2.3 From 2b979cc90c4e466a8d0a83706e642b325cc24d0e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 9 Jul 2020 11:55:40 -0500 Subject: Add AdminFE reports URL to report emails --- lib/pleroma/emails/admin_email.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index c67ba63ad..aa0b2a66b 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -72,6 +72,8 @@ defmodule Pleroma.Emails.AdminEmail do

Reported Account: #{account.nickname}

#{comment_html} #{statuses_html} +

+ View Reports in AdminFE """ new() -- cgit v1.2.3 From cc7153cd828afef1564b58649875b5529c078054 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 9 Jul 2020 19:07:07 +0200 Subject: user: Add support for custom emojis in profile fields --- lib/pleroma/user.ex | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 9d1314f81..19b361b88 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -527,11 +527,21 @@ defmodule Pleroma.User do end defp put_emoji(changeset) do - bio = get_change(changeset, :bio) - name = get_change(changeset, :name) + emojified_fields = [:bio, :name, :raw_fields] + + if Enum.any?(changeset.changes, fn {k, _} -> k in emojified_fields end) do + bio = Emoji.Formatter.get_emoji_map(get_field(changeset, :bio)) + name = Emoji.Formatter.get_emoji_map(get_field(changeset, :name)) + + emoji = Map.merge(bio, name) + + emoji = + changeset + |> get_field(:raw_fields) + |> Enum.reduce(emoji, fn x, acc -> + Map.merge(acc, Emoji.Formatter.get_emoji_map(x["name"] <> x["value"])) + end) - if bio || name do - emoji = Map.merge(Emoji.Formatter.get_emoji_map(bio), Emoji.Formatter.get_emoji_map(name)) put_change(changeset, :emoji, emoji) else changeset -- cgit v1.2.3 From 93e494ec212b5bb6aae31a3b43304ed230d095e2 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 10 Jul 2020 14:10:44 +0200 Subject: ActivityPub: Don't rename a clashing nickname with the same ap id. --- lib/pleroma/web/activity_pub/activity_pub.ex | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 8da5cf938..bc7b5d95a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1376,13 +1376,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def maybe_handle_clashing_nickname(nickname) do - with %User{} = old_user <- User.get_by_nickname(nickname) do - Logger.info("Found an old user for #{nickname}, ap id is #{old_user.ap_id}, renaming.") + def maybe_handle_clashing_nickname(data) do + nickname = data[:nickname] + + with %User{} = old_user <- User.get_by_nickname(nickname), + {_, false} <- {:ap_id_comparison, data[:ap_id] == old_user.ap_id} do + Logger.info( + "Found an old user for #{nickname}, the old ap id is #{old_user.ap_id}, new one is #{ + data[:ap_id] + }, renaming." + ) old_user |> User.remote_user_changeset(%{nickname: "#{old_user.id}.#{old_user.nickname}"}) |> User.update_and_set_cache() + else + {:ap_id_comparison, true} -> + Logger.info( + "Found an old user for #{nickname}, but the ap id #{data[:ap_id]} is the same as the new user. Race condition? Not changing anything." + ) + + _ -> + nil end end @@ -1398,7 +1413,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> User.remote_user_changeset(data) |> User.update_and_set_cache() else - maybe_handle_clashing_nickname(data[:nickname]) + maybe_handle_clashing_nickname(data) data |> User.remote_user_changeset() -- cgit v1.2.3 From a1dace088cb8d17e074e38689196793aa2c46a57 Mon Sep 17 00:00:00 2001 From: href Date: Fri, 10 Jul 2020 17:10:48 +0200 Subject: ReverseProxy: Streaming and disable encoding if Range Fixes #1823 Fixes #1860 --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 54 ++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 18 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 4bbeb493c..76a321c3a 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -3,11 +3,12 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.ReverseProxy do + @range_headers ~w(range if-range) @keep_req_headers ~w(accept user-agent accept-encoding cache-control if-modified-since) ++ - ~w(if-unmodified-since if-none-match if-range range) + ~w(if-unmodified-since if-none-match) ++ @range_headers @resp_cache_headers ~w(etag date last-modified) @keep_resp_headers @resp_cache_headers ++ - ~w(content-type content-disposition content-encoding content-range) ++ + ~w(content-length content-type content-disposition content-encoding content-range) ++ ~w(accept-ranges vary) @default_cache_control_header "public, max-age=1209600" @valid_resp_codes [200, 206, 304] @@ -170,6 +171,8 @@ defmodule Pleroma.ReverseProxy do end defp response(conn, client, url, status, headers, opts) do + Logger.debug("#{__MODULE__} #{status} #{url} #{inspect(headers)}") + result = conn |> put_resp_headers(build_resp_headers(headers, opts)) @@ -220,7 +223,9 @@ defmodule Pleroma.ReverseProxy do end end - defp head_response(conn, _url, code, headers, opts) do + defp head_response(conn, url, code, headers, opts) do + Logger.debug("#{__MODULE__} #{code} #{url} #{inspect(headers)}") + conn |> put_resp_headers(build_resp_headers(headers, opts)) |> send_resp(code, "") @@ -262,20 +267,33 @@ defmodule Pleroma.ReverseProxy do headers |> downcase_headers() |> Enum.filter(fn {k, _} -> k in @keep_req_headers end) - |> (fn headers -> - headers = headers ++ Keyword.get(opts, :req_headers, []) - - if Keyword.get(opts, :keep_user_agent, false) do - List.keystore( - headers, - "user-agent", - 0, - {"user-agent", Pleroma.Application.user_agent()} - ) - else - headers - end - end).() + |> build_req_range_or_encoding_header(opts) + |> build_req_user_agent_header(opts) + |> Keyword.merge(Keyword.get(opts, :req_headers, [])) + end + + # Disable content-encoding if any @range_headers are requested (see #1823). + defp build_req_range_or_encoding_header(headers, _opts) do + range? = Enum.any?(headers, fn {header, _} -> Enum.member?(@range_headers, header) end) + + if range? && List.keymember?(headers, "accept-encoding", 0) do + List.keydelete(headers, "accept-encoding", 0) + else + headers + end + end + + defp build_req_user_agent_header(headers, opts) do + if Keyword.get(opts, :keep_user_agent, false) do + List.keystore( + headers, + "user-agent", + 0, + {"user-agent", Pleroma.Application.user_agent()} + ) + else + headers + end end defp build_resp_headers(headers, opts) do @@ -283,7 +301,7 @@ defmodule Pleroma.ReverseProxy do |> Enum.filter(fn {k, _} -> k in @keep_resp_headers end) |> build_resp_cache_headers(opts) |> build_resp_content_disposition_header(opts) - |> (fn headers -> headers ++ Keyword.get(opts, :resp_headers, []) end).() + |> Keyword.merge(Keyword.get(opts, :resp_headers, [])) end defp build_resp_cache_headers(headers, _opts) do -- cgit v1.2.3 From 72b3dbf4d1c457747c362eda3dddada6e02f1568 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 10 Jul 2020 11:04:19 -0500 Subject: Credo line length complaint --- lib/pleroma/reverse_proxy/reverse_proxy.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex index 76a321c3a..28ad4c846 100644 --- a/lib/pleroma/reverse_proxy/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex @@ -8,8 +8,8 @@ defmodule Pleroma.ReverseProxy do ~w(if-unmodified-since if-none-match) ++ @range_headers @resp_cache_headers ~w(etag date last-modified) @keep_resp_headers @resp_cache_headers ++ - ~w(content-length content-type content-disposition content-encoding content-range) ++ - ~w(accept-ranges vary) + ~w(content-length content-type content-disposition content-encoding) ++ + ~w(content-range accept-ranges vary) @default_cache_control_header "public, max-age=1209600" @valid_resp_codes [200, 206, 304] @max_read_duration :timer.seconds(30) -- cgit v1.2.3 From 61675938811a8f2121c857f285d31bd4c7ef3336 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 10 Jul 2020 16:46:26 -0500 Subject: Support Exiftool for stripping EXIF data We really only want to strip location data anyway, and mogrify strips color profiles. --- lib/pleroma/upload/filter/exiftool.ex | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 lib/pleroma/upload/filter/exiftool.ex (limited to 'lib') diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex new file mode 100644 index 000000000..833d8cab4 --- /dev/null +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.Exiftool do + @behaviour Pleroma.Upload.Filter + + @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} + @type conversions :: conversion() | [conversion()] + + def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do + System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) + :ok + end + + def filter(_), do: :ok +end -- cgit v1.2.3 From 9e4567267443b801ab2f4f58f3897b2834387a80 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 10 Jul 2020 17:07:28 -0500 Subject: Add a moduledoc --- lib/pleroma/upload/filter/exiftool.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index 833d8cab4..eb199709a 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -3,6 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Upload.Filter.Exiftool do + @moduledoc """ + Strips GPS related EXIF tags and overwrites the file in place. + Also strips or replaces filesystem metadata e.g., timestamps. + """ @behaviour Pleroma.Upload.Filter @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} -- cgit v1.2.3 From b329f05ed63657aa19b227b9b919662dad56ecae Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 10 Jul 2020 17:08:54 -0500 Subject: Remove unused @types --- lib/pleroma/upload/filter/exiftool.ex | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/exiftool.ex b/lib/pleroma/upload/filter/exiftool.ex index eb199709a..c7fb6aefa 100644 --- a/lib/pleroma/upload/filter/exiftool.ex +++ b/lib/pleroma/upload/filter/exiftool.ex @@ -9,9 +9,6 @@ defmodule Pleroma.Upload.Filter.Exiftool do """ @behaviour Pleroma.Upload.Filter - @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} - @type conversions :: conversion() | [conversion()] - def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) :ok -- cgit v1.2.3 From b3764423251c963a5ca007517189f556bfe95155 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sat, 11 Jul 2020 10:36:36 +0300 Subject: MediaProxy whitelist setting now supports hosts with scheme added deprecation warning about using bare domains --- lib/pleroma/config/deprecation_warnings.ex | 15 +++++++++- lib/pleroma/plugs/http_security_plug.ex | 47 ++++++++++++++++++++---------- lib/pleroma/web/media_proxy/media_proxy.ex | 26 ++++++++++------- 3 files changed, 62 insertions(+), 26 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 0a6c724fb..026871c4f 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -54,6 +54,7 @@ defmodule Pleroma.Config.DeprecationWarnings do check_hellthread_threshold() mrf_user_allowlist() check_old_mrf_config() + check_media_proxy_whitelist_config() end def check_old_mrf_config do @@ -65,7 +66,7 @@ defmodule Pleroma.Config.DeprecationWarnings do move_namespace_and_warn(@mrf_config_map, warning_preface) end - @spec move_namespace_and_warn([config_map()], String.t()) :: :ok + @spec move_namespace_and_warn([config_map()], String.t()) :: :ok | nil def move_namespace_and_warn(config_map, warning_preface) do warning = Enum.reduce(config_map, "", fn @@ -84,4 +85,16 @@ defmodule Pleroma.Config.DeprecationWarnings do Logger.warn(warning_preface <> warning) end end + + @spec check_media_proxy_whitelist_config() :: :ok | nil + def check_media_proxy_whitelist_config do + whitelist = Config.get([:media_proxy, :whitelist]) + + if Enum.any?(whitelist, &(not String.starts_with?(&1, "http"))) do + Logger.warn(""" + !!!DEPRECATION WARNING!!! + Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. + """) + end + end end diff --git a/lib/pleroma/plugs/http_security_plug.ex b/lib/pleroma/plugs/http_security_plug.ex index 7d65cf078..c363b193b 100644 --- a/lib/pleroma/plugs/http_security_plug.ex +++ b/lib/pleroma/plugs/http_security_plug.ex @@ -108,31 +108,48 @@ defmodule Pleroma.Plugs.HTTPSecurityPlug do |> :erlang.iolist_to_binary() end - defp build_csp_multimedia_source_list do - media_proxy_whitelist = - Enum.reduce(Config.get([:media_proxy, :whitelist]), [], fn host, acc -> - add_source(acc, host) - end) + defp build_csp_from_whitelist([], acc), do: acc - media_proxy_base_url = build_csp_param(Config.get([:media_proxy, :base_url])) + defp build_csp_from_whitelist([last], acc) do + [build_csp_param_from_whitelist(last) | acc] + end - upload_base_url = build_csp_param(Config.get([Pleroma.Upload, :base_url])) + defp build_csp_from_whitelist([head | tail], acc) do + build_csp_from_whitelist(tail, [[?\s, build_csp_param_from_whitelist(head)] | acc]) + end - s3_endpoint = build_csp_param(Config.get([Pleroma.Uploaders.S3, :public_endpoint])) + # TODO: use `build_csp_param/1` after removing support bare domains for media proxy whitelist + defp build_csp_param_from_whitelist("http" <> _ = url) do + build_csp_param(url) + end - captcha_method = Config.get([Pleroma.Captcha, :method]) + defp build_csp_param_from_whitelist(url), do: url - captcha_endpoint = build_csp_param(Config.get([captcha_method, :endpoint])) + defp build_csp_multimedia_source_list do + media_proxy_whitelist = + [:media_proxy, :whitelist] + |> Config.get() + |> build_csp_from_whitelist([]) - [] - |> add_source(media_proxy_base_url) - |> add_source(upload_base_url) - |> add_source(s3_endpoint) + captcha_method = Config.get([Pleroma.Captcha, :method]) + captcha_endpoint = Config.get([captcha_method, :endpoint]) + + base_endpoints = + [ + [:media_proxy, :base_url], + [Pleroma.Upload, :base_url], + [Pleroma.Uploaders.S3, :public_endpoint] + ] + |> Enum.map(&Config.get/1) + + [captcha_endpoint | base_endpoints] + |> Enum.map(&build_csp_param/1) + |> Enum.reduce([], &add_source(&2, &1)) |> add_source(media_proxy_whitelist) - |> add_source(captcha_endpoint) end defp add_source(iodata, nil), do: iodata + defp add_source(iodata, []), do: iodata defp add_source(iodata, source), do: [[?\s, source] | iodata] defp add_csp_param(csp_iodata, nil), do: csp_iodata diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 6f35826da..dfbfcea6b 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -60,22 +60,28 @@ defmodule Pleroma.Web.MediaProxy do defp whitelisted?(url) do %{host: domain} = URI.parse(url) - mediaproxy_whitelist = Config.get([:media_proxy, :whitelist]) - - upload_base_url_domain = - if !is_nil(Config.get([Upload, :base_url])) do - [URI.parse(Config.get([Upload, :base_url])).host] + mediaproxy_whitelist_domains = + [:media_proxy, :whitelist] + |> Config.get() + |> Enum.map(&maybe_get_domain_from_url/1) + + whitelist_domains = + if base_url = Config.get([Upload, :base_url]) do + %{host: base_domain} = URI.parse(base_url) + [base_domain | mediaproxy_whitelist_domains] else - [] + mediaproxy_whitelist_domains end - whitelist = mediaproxy_whitelist ++ upload_base_url_domain + domain in whitelist_domains + end - Enum.any?(whitelist, fn pattern -> - String.equivalent?(domain, pattern) - end) + defp maybe_get_domain_from_url("http" <> _ = url) do + URI.parse(url).host end + defp maybe_get_domain_from_url(domain), do: domain + def encode_url(url) do base64 = Base.url_encode64(url, @base64_opts) -- cgit v1.2.3 From 4347d2de5eb609bbfa1a206a5de5df925d3a0696 Mon Sep 17 00:00:00 2001 From: href Date: Sun, 12 Jul 2020 17:23:33 +0200 Subject: Config/Docs: Expand behaviour suggestions at runtime --- lib/pleroma/application.ex | 1 + lib/pleroma/docs/generator.ex | 31 +++++++++++++++------- lib/pleroma/docs/json.ex | 21 ++++++++++----- lib/pleroma/docs/markdown.ex | 5 ++++ .../web/admin_api/controllers/config_controller.ex | 4 +-- 5 files changed, 42 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 84f3aa82d..b68a373a4 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -42,6 +42,7 @@ defmodule Pleroma.Application do Pleroma.ApplicationRequirements.verify!() setup_instrumenters() load_custom_modules() + Pleroma.Docs.JSON.compile() adapter = Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index e0fc8cd02..a671a6278 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -6,16 +6,21 @@ defmodule Pleroma.Docs.Generator do implementation.process(descriptions) end - @spec list_modules_in_dir(String.t(), String.t()) :: [module()] - def list_modules_in_dir(dir, start) do - with {:ok, files} <- File.ls(dir) do - files - |> Enum.filter(&String.ends_with?(&1, ".ex")) - |> Enum.map(fn filename -> - module = filename |> String.trim_trailing(".ex") |> Macro.camelize() - String.to_atom(start <> module) - end) - end + @spec list_behaviour_implementations(behaviour :: module()) :: [module()] + def list_behaviour_implementations(behaviour) do + :code.all_loaded() + |> Enum.filter(fn {module, _} -> + # This shouldn't be needed as all modules are expected to have module_info/1, + # but in test enviroments some transient modules `:elixir_compiler_XX` + # are loaded for some reason (where XX is a random integer). + if function_exported?(module, :module_info, 1) do + module.module_info(:attributes) + |> Keyword.get_values(:behaviour) + |> List.flatten() + |> Enum.member?(behaviour) + end + end) + |> Enum.map(fn {module, _} -> module end) end @doc """ @@ -87,6 +92,12 @@ defmodule Pleroma.Docs.Generator do else: string end + defp format_suggestions({:list_behaviour_implementations, behaviour}) do + behaviour + |> list_behaviour_implementations() + |> format_suggestions() + end + defp format_suggestions([]), do: [] defp format_suggestions([suggestion | tail]) do diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex index d1cf1f487..feeb4320e 100644 --- a/lib/pleroma/docs/json.ex +++ b/lib/pleroma/docs/json.ex @@ -1,5 +1,19 @@ defmodule Pleroma.Docs.JSON do @behaviour Pleroma.Docs.Generator + @external_resource "config/description.exs" + @raw_config Pleroma.Config.Loader.read("config/description.exs") + @raw_descriptions @raw_config[:pleroma][:config_description] + @term __MODULE__.Compiled + + @spec compile :: :ok + def compile do + :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(@raw_descriptions)) + end + + @spec compiled_descriptions :: Map.t() + def compiled_descriptions do + :persistent_term.get(@term) + end @spec process(keyword()) :: {:ok, String.t()} def process(descriptions) do @@ -13,11 +27,4 @@ defmodule Pleroma.Docs.JSON do {:ok, path} end end - - def compile do - with config <- Pleroma.Config.Loader.read("config/description.exs") do - config[:pleroma][:config_description] - |> Pleroma.Docs.Generator.convert_to_strings() - end - end end diff --git a/lib/pleroma/docs/markdown.ex b/lib/pleroma/docs/markdown.ex index 68b106499..da3f20f43 100644 --- a/lib/pleroma/docs/markdown.ex +++ b/lib/pleroma/docs/markdown.ex @@ -68,6 +68,11 @@ defmodule Pleroma.Docs.Markdown do IO.write(file, " #{list_mark}`#{inspect(suggestion)}`\n") end + defp print_suggestions(file, {:list_behaviour_implementations, behaviour}) do + suggestions = Pleroma.Docs.Generator.list_behaviour_implementations(behaviour) + print_suggestions(file, suggestions) + end + defp print_suggestions(_file, nil), do: nil defp print_suggestions(_file, ""), do: nil diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex index 7f60470cb..0df13007f 100644 --- a/lib/pleroma/web/admin_api/controllers/config_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do alias Pleroma.ConfigDB alias Pleroma.Plugs.OAuthScopesPlug - @descriptions Pleroma.Docs.JSON.compile() - plug(Pleroma.Web.ApiSpec.CastAndValidate) plug(OAuthScopesPlug, %{scopes: ["write"], admin: true} when action == :update) @@ -25,7 +23,7 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.ConfigOperation def descriptions(conn, _params) do - descriptions = Enum.filter(@descriptions, &whitelisted_config?/1) + descriptions = Enum.filter(Pleroma.Docs.JSON.compiled_descriptions(), &whitelisted_config?/1) json(conn, descriptions) end -- cgit v1.2.3 From 133004e22d74e7cdfd13a69f88b509b395985a5d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 12 Jul 2020 10:38:07 -0500 Subject: Improve database config migration and add documentation --- lib/mix/tasks/pleroma/config.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index d5129d410..343438add 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -83,7 +83,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp migrate_from_db(opts) do if Pleroma.Config.get([:configurable_from_database]) do - env = opts[:env] || "prod" + env = Mix.env() config_path = if Pleroma.Config.get(:release) do @@ -105,6 +105,7 @@ defmodule Mix.Tasks.Pleroma.Config do :ok = File.close(file) System.cmd("mix", ["format", config_path]) + shell_info("Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs") else migration_error() end @@ -112,7 +113,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp migration_error do shell_error( - "Migration is not allowed in config. You can change this behavior by setting `configurable_from_database` to true." + "Migration is not allowed in config. You can change this behavior by setting `config :pleroma, configurable_from_database: true`" ) end -- cgit v1.2.3 From 46b123cded5f572851652cecedcce22aa87b97e7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 12 Jul 2020 10:59:12 -0500 Subject: Still allow passing the arg, but fallback to MIX_ENV --- lib/mix/tasks/pleroma/config.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 343438add..38c6a6f1d 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -83,7 +83,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp migrate_from_db(opts) do if Pleroma.Config.get([:configurable_from_database]) do - env = Mix.env() + env = opts[:env] || Mix.env() config_path = if Pleroma.Config.get(:release) do -- cgit v1.2.3 From e1908a5270d7b060238c9bc8bcd2808c705c27d9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 13 Jul 2020 08:39:56 -0500 Subject: Pick up env for both source and OTP installs --- lib/mix/tasks/pleroma/config.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 38c6a6f1d..7e2164181 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -83,7 +83,7 @@ defmodule Mix.Tasks.Pleroma.Config do defp migrate_from_db(opts) do if Pleroma.Config.get([:configurable_from_database]) do - env = opts[:env] || Mix.env() + env = opts[:env] || Pleroma.Config.get(:env) config_path = if Pleroma.Config.get(:release) do -- cgit v1.2.3 From d1cd3f4ec06214dc85e11dca30f193ee7d093488 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 13 Jul 2020 10:32:17 -0500 Subject: Lint --- lib/mix/tasks/pleroma/config.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 7e2164181..904c5a74b 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -105,7 +105,10 @@ defmodule Mix.Tasks.Pleroma.Config do :ok = File.close(file) System.cmd("mix", ["format", config_path]) - shell_info("Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs") + + shell_info( + "Database configuration settings have been exported to config/#{env}.exported_from_db.secret.exs" + ) else migration_error() end -- cgit v1.2.3 From b221b640a2dd443e3c2274b16ed5b62566329d09 Mon Sep 17 00:00:00 2001 From: = <=> Date: Mon, 13 Jul 2020 22:19:13 +0300 Subject: Transmogrifier: filtering weirdness in address fields --- lib/pleroma/web/activity_pub/transmogrifier.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 884646ceb..f37bcab3e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -62,15 +62,17 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_summary(object), do: Map.put(object, "summary", "") def fix_addressing_list(map, field) do + addrs = map[field] + cond do - is_binary(map[field]) -> - Map.put(map, field, [map[field]]) + is_list(addrs) -> + Map.put(map, field, Enum.filter(addrs, &is_binary/1)) - is_nil(map[field]) -> - Map.put(map, field, []) + is_binary(addrs) -> + Map.put(map, field, [addrs]) true -> - map + Map.put(map, field, []) end end -- cgit v1.2.3 From cf3f8cb72a46f0c8c798d4022cff442fae4ab401 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Sun, 19 Jul 2020 21:35:57 +0300 Subject: [#1940] Reinstated OAuth-less `admin_token` authentication. Refactored UserIsAdminPlug (freed from checking admin scopes presence). --- .../plugs/admin_secret_authentication_plug.ex | 12 +++++++++-- lib/pleroma/plugs/user_is_admin_plug.ex | 25 +++------------------- 2 files changed, 13 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/admin_secret_authentication_plug.ex b/lib/pleroma/plugs/admin_secret_authentication_plug.ex index b4b47a31f..ff0328d4a 100644 --- a/lib/pleroma/plugs/admin_secret_authentication_plug.ex +++ b/lib/pleroma/plugs/admin_secret_authentication_plug.ex @@ -4,7 +4,9 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do import Plug.Conn + alias Pleroma.User + alias Pleroma.Plugs.OAuthScopesPlug def init(options) do options @@ -26,7 +28,7 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do def authenticate(%{params: %{"admin_token" => admin_token}} = conn) do if admin_token == secret_token() do - assign(conn, :user, %User{is_admin: true}) + assign_admin_user(conn) else conn end @@ -36,8 +38,14 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do token = secret_token() case get_req_header(conn, "x-admin-token") do - [^token] -> assign(conn, :user, %User{is_admin: true}) + [^token] -> assign_admin_user(conn) _ -> conn end end + + defp assign_admin_user(conn) do + conn + |> assign(:user, %User{is_admin: true}) + |> OAuthScopesPlug.skip_plug() + end end diff --git a/lib/pleroma/plugs/user_is_admin_plug.ex b/lib/pleroma/plugs/user_is_admin_plug.ex index 2748102df..488a61d1d 100644 --- a/lib/pleroma/plugs/user_is_admin_plug.ex +++ b/lib/pleroma/plugs/user_is_admin_plug.ex @@ -7,37 +7,18 @@ defmodule Pleroma.Plugs.UserIsAdminPlug do import Plug.Conn alias Pleroma.User - alias Pleroma.Web.OAuth def init(options) do options end - def call(%{assigns: %{user: %User{is_admin: true}} = assigns} = conn, _) do - token = assigns[:token] - - cond do - not Pleroma.Config.enforce_oauth_admin_scope_usage?() -> - conn - - token && OAuth.Scopes.contains_admin_scopes?(token.scopes) -> - # Note: checking for _any_ admin scope presence, not necessarily fitting requested action. - # Thus, controller must explicitly invoke OAuthScopesPlug to verify scope requirements. - # Admin might opt out of admin scope for some apps to block any admin actions from them. - conn - - true -> - fail(conn) - end + def call(%{assigns: %{user: %User{is_admin: true}}} = conn, _) do + conn end def call(conn, _) do - fail(conn) - end - - defp fail(conn) do conn - |> render_error(:forbidden, "User is not an admin or OAuth admin scope is not granted.") + |> render_error(:forbidden, "User is not an admin.") |> halt() end end -- cgit v1.2.3 From 9b225db7d86289fb9d9c51f62e6ec29f6c07f60d Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 14 Jul 2020 11:58:41 +0300 Subject: [#1940] Applied rate limit for requests with bad `admin_token`. Added doc warnings on `admin_token` setting. --- lib/pleroma/plugs/admin_secret_authentication_plug.ex | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/plugs/admin_secret_authentication_plug.ex b/lib/pleroma/plugs/admin_secret_authentication_plug.ex index ff0328d4a..2e54df47a 100644 --- a/lib/pleroma/plugs/admin_secret_authentication_plug.ex +++ b/lib/pleroma/plugs/admin_secret_authentication_plug.ex @@ -5,15 +5,19 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do import Plug.Conn - alias Pleroma.User alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Plugs.RateLimiter + alias Pleroma.User def init(options) do options end def secret_token do - Pleroma.Config.get(:admin_token) + case Pleroma.Config.get(:admin_token) do + blank when blank in [nil, ""] -> nil + token -> token + end end def call(%{assigns: %{user: %User{}}} = conn, _), do: conn @@ -30,7 +34,7 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do if admin_token == secret_token() do assign_admin_user(conn) else - conn + handle_bad_token(conn) end end @@ -38,8 +42,9 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do token = secret_token() case get_req_header(conn, "x-admin-token") do + blank when blank in [[], [""]] -> conn [^token] -> assign_admin_user(conn) - _ -> conn + _ -> handle_bad_token(conn) end end @@ -48,4 +53,8 @@ defmodule Pleroma.Plugs.AdminSecretAuthenticationPlug do |> assign(:user, %User{is_admin: true}) |> OAuthScopesPlug.skip_plug() end + + defp handle_bad_token(conn) do + RateLimiter.call(conn, name: :authentication) + end end -- cgit v1.2.3 From 858d9fc7e8e722604676c90cf2707f0209f935ec Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 13 Jul 2020 15:47:13 +0200 Subject: MRF Policies: Return a {:reject, reason} instead of {:reject, nil} --- .../web/activity_pub/mrf/anti_followbot_policy.ex | 2 +- .../web/activity_pub/mrf/anti_link_spam_policy.ex | 7 +++---- lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex | 4 ++-- lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 7 ++++--- lib/pleroma/web/activity_pub/mrf/mention_policy.ex | 5 +++-- lib/pleroma/web/activity_pub/mrf/object_age_policy.ex | 8 +++----- lib/pleroma/web/activity_pub/mrf/reject_non_public.ex | 2 +- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 16 ++++++++++------ lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 7 ++++--- .../web/activity_pub/mrf/user_allow_list_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex | 18 +++++++++++------- 11 files changed, 43 insertions(+), 35 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 0270b96ae..b96388489 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -60,7 +60,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do if score < 0.8 do {:ok, message} else - {:reject, nil} + {:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index a7e187b5e..b22464111 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -39,14 +39,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do {:ok, message} {:old_user, false} -> - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] User has no posts nor followers"} {:error, _} -> - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] Failed to get or fetch user by ap_id"} e -> - Logger.warn("[MRF anti-link-spam] WTF: unhandled error #{inspect(e)}") - {:reject, nil} + {:reject, "[AntiLinkSpamPolicy] Unhandled error #{inspect(e)}"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index f6b2c4415..9ba07b4e3 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do defp reject_message(message, threshold) when threshold > 0 do with {_, recipients} <- get_recipient_count(message) do if recipients > threshold do - {:reject, nil} + {:reject, "[HellthreadPolicy] #{recipients} recipients is over the limit of #{threshold}"} else {:ok, message} end @@ -87,7 +87,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do {:ok, message} <- delist_message(message, delist_threshold) do {:ok, message} else - _e -> {:reject, nil} + e -> e end end diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 88b0d2b39..15e09dcf0 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do if Enum.any?(Pleroma.Config.get([:mrf_keyword, :reject]), fn pattern -> string_matches?(content, pattern) or string_matches?(summary, pattern) end) do - {:reject, nil} + {:reject, "[KeywordPolicy] Matches with rejected keyword"} else {:ok, message} end @@ -89,8 +89,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do {:ok, message} <- check_replace(message) do {:ok, message} else - _e -> - {:reject, nil} + {:reject, nil} -> {:reject, "[KeywordPolicy] "} + {:reject, _} = e -> e + _e -> {:reject, "[KeywordPolicy] "} end end diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index 06f003921..7910ca131 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -12,8 +12,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) recipients = (message["to"] || []) ++ (message["cc"] || []) - if Enum.any?(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do - {:reject, nil} + if rejected_mention = + Enum.find(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do + {:reject, "[MentionPolicy] Rejected for mention of #{rejected_mention}"} else {:ok, message} end diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index a62914135..5f111c72f 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do defp check_reject(message, actions) do if :reject in actions do - {:reject, nil} + {:reject, "[ObjectAgePolicy]"} else {:ok, message} end @@ -47,9 +47,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do {:ok, message} else - # Unhandleable error: somebody is messing around, just drop the message. _e -> - {:reject, nil} + {:reject, "[ObjectAgePolicy] Unhandled error"} end else {:ok, message} @@ -69,9 +68,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do {:ok, message} else - # Unhandleable error: somebody is messing around, just drop the message. _e -> - {:reject, nil} + {:reject, "[ObjectAgePolicy] Unhandled error"} end else {:ok, message} diff --git a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex index 4fd63106d..0b9ed2224 100644 --- a/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex +++ b/lib/pleroma/web/activity_pub/mrf/reject_non_public.ex @@ -38,7 +38,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.RejectNonPublic do {:ok, object} true -> - {:reject, nil} + {:reject, "[RejectNonPublic] visibility: #{visibility}"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 70a2ca053..b77b8c7b4 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do accepts == [] -> {:ok, object} actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} MRF.subdomain_match?(accepts, actor_host) -> {:ok, object} - true -> {:reject, nil} + true -> {:reject, "[SimplePolicy] host not in accept list"} end end @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do |> MRF.subdomains_regex() if MRF.subdomain_match?(rejects, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in reject list"} else {:ok, object} end @@ -114,7 +114,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do |> MRF.subdomains_regex() if MRF.subdomain_match?(report_removal, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in report_removal list"} else {:ok, object} end @@ -159,7 +159,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do |> MRF.subdomains_regex() if MRF.subdomain_match?(reject_deletes, actor_host) do - {:reject, nil} + {:reject, "[SimplePolicy] host in reject_deletes list"} else {:ok, object} end @@ -177,7 +177,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_report_removal(actor_info, object) do {:ok, object} else - _e -> {:reject, nil} + {:reject, nil} -> {:reject, "[SimplePolicy]"} + {:reject, _} = e -> e + _ -> {:reject, "[SimplePolicy]"} end end @@ -191,7 +193,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_banner_removal(actor_info, object) do {:ok, object} else - _e -> {:reject, nil} + {:reject, nil} -> {:reject, "[SimplePolicy]"} + {:reject, _} = e -> e + _ -> {:reject, "[SimplePolicy]"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index c310462cb..febabda08 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -134,12 +134,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do if user.local == true do {:ok, message} else - {:reject, nil} + {:reject, + "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-remote-subscription"} end end - defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow"}), - do: {:reject, nil} + defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow", "actor" => actor}), + do: {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-any-subscription"} defp process_tag(_, message), do: {:ok, message} diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index 651aed70f..1a28f2ba2 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do if actor in allow_list do {:ok, object} else - {:reject, nil} + {:reject, "[UserAllowListPolicy] #{actor} not in the list"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 6167a74e2..a6c545570 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -11,22 +11,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do with {:ok, _} <- filter(child_message) do {:ok, message} else - {:reject, nil} -> - {:reject, nil} + {:reject, _} = e -> e end end def filter(%{"type" => message_type} = message) do with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]), rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]), - true <- - Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type), - false <- - length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type), + {_, true} <- + {:accepted, + Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type)}, + {_, false} <- + {:rejected, + length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type)}, {:ok, _} <- filter(message["object"]) do {:ok, message} else - _ -> {:reject, nil} + {:reject, _} = e -> e + {:accepted, _} -> {:reject, "[VocabularyPolicy] #{message_type} not in accept list"} + {:rejected, _} -> {:reject, "[VocabularyPolicy] #{message_type} in reject list"} + _ -> {:reject, "[VocabularyPolicy]"} end end -- cgit v1.2.3 From e6ccc2556568f2180c3ce1945bdc7a0cba97e924 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 14 Jul 2020 11:41:30 +0300 Subject: Fix in-db configuration in dev environment Previously, in-db configuration only worked when `warnings_as_errors` was disabled because re-compiling scrubbers on application restart created a warning about module conflicts. This patch fixes that by enabling `ignore_module_conflict` option of the compiler at runtime, and enables `warnings_as_errors` in prod since there is no reason to keep it disabled anymore. --- lib/pleroma/application.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index b68a373a4..3282c6882 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -35,6 +35,10 @@ defmodule Pleroma.Application do # See http://elixir-lang.org/docs/stable/elixir/Application.html # for more information on OTP Applications def start(_type, _args) do + # Scrubbers are compiled at runtime and therefore will cause a conflict + # every time the application is restarted, so we disable module + # conflicts at runtime + Code.compiler_options(ignore_module_conflict: true) Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() Config.DeprecationWarnings.warn() -- cgit v1.2.3 From 124b4709dcf12a417f5164e53ef3ba67e538d4c7 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Tue, 14 Jul 2020 19:31:05 +0300 Subject: [#1940] Added `admin_token` param (as `admin_api_params/0`) to existing Admin API OpenAPI operations. --- lib/pleroma/web/api_spec/helpers.ex | 4 ++++ lib/pleroma/web/api_spec/operations/admin/config_operation.ex | 3 +++ lib/pleroma/web/api_spec/operations/admin/invite_operation.ex | 4 ++++ .../web/api_spec/operations/admin/media_proxy_cache_operation.ex | 3 +++ lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex | 6 ++++-- lib/pleroma/web/api_spec/operations/admin/relay_operation.ex | 3 +++ lib/pleroma/web/api_spec/operations/admin/report_operation.ex | 7 +++++-- lib/pleroma/web/api_spec/operations/admin/status_operation.ex | 7 ++++--- 8 files changed, 30 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex index a258e8421..2a7f1a706 100644 --- a/lib/pleroma/web/api_spec/helpers.ex +++ b/lib/pleroma/web/api_spec/helpers.ex @@ -29,6 +29,10 @@ defmodule Pleroma.Web.ApiSpec.Helpers do } end + def admin_api_params do + [Operation.parameter(:admin_token, :query, :string, "Allows authorization via admin token.")] + end + def pagination_params do [ Operation.parameter(:max_id, :query, :string, "Return items older than this ID"), diff --git a/lib/pleroma/web/api_spec/operations/admin/config_operation.ex b/lib/pleroma/web/api_spec/operations/admin/config_operation.ex index 7b38a2ef4..3a8380797 100644 --- a/lib/pleroma/web/api_spec/operations/admin/config_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/config_operation.ex @@ -26,6 +26,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do %Schema{type: :boolean, default: false}, "Get only saved in database settings" ) + | admin_api_params() ], security: [%{"oAuth" => ["read"]}], responses: %{ @@ -41,6 +42,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do summary: "Update config settings", operationId: "AdminAPI.ConfigController.update", security: [%{"oAuth" => ["write"]}], + parameters: admin_api_params(), requestBody: request_body("Parameters", %Schema{ type: :object, @@ -73,6 +75,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ConfigOperation do summary: "Get JSON with config descriptions.", operationId: "AdminAPI.ConfigController.descriptions", security: [%{"oAuth" => ["read"]}], + parameters: admin_api_params(), responses: %{ 200 => Operation.response("Config Descriptions", "application/json", %Schema{ diff --git a/lib/pleroma/web/api_spec/operations/admin/invite_operation.ex b/lib/pleroma/web/api_spec/operations/admin/invite_operation.ex index d3af9db49..801024d75 100644 --- a/lib/pleroma/web/api_spec/operations/admin/invite_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/invite_operation.ex @@ -20,6 +20,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do summary: "Get a list of generated invites", operationId: "AdminAPI.InviteController.index", security: [%{"oAuth" => ["read:invites"]}], + parameters: admin_api_params(), responses: %{ 200 => Operation.response("Invites", "application/json", %Schema{ @@ -51,6 +52,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do summary: "Create an account registration invite token", operationId: "AdminAPI.InviteController.create", security: [%{"oAuth" => ["write:invites"]}], + parameters: admin_api_params(), requestBody: request_body("Parameters", %Schema{ type: :object, @@ -71,6 +73,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do summary: "Revoke invite by token", operationId: "AdminAPI.InviteController.revoke", security: [%{"oAuth" => ["write:invites"]}], + parameters: admin_api_params(), requestBody: request_body( "Parameters", @@ -97,6 +100,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.InviteOperation do summary: "Sends registration invite via email", operationId: "AdminAPI.InviteController.email", security: [%{"oAuth" => ["write:invites"]}], + parameters: admin_api_params(), requestBody: request_body( "Parameters", diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex index 0358cfbad..20d033f66 100644 --- a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex @@ -33,6 +33,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do %Schema{type: :integer, default: 50}, "Number of statuses to return" ) + | admin_api_params() ], responses: %{ 200 => success_response() @@ -46,6 +47,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do summary: "Remove a banned MediaProxy URL from Cachex", operationId: "AdminAPI.MediaProxyCacheController.delete", security: [%{"oAuth" => ["write:media_proxy_caches"]}], + parameters: admin_api_params(), requestBody: request_body( "Parameters", @@ -71,6 +73,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do summary: "Purge and optionally ban a MediaProxy URL", operationId: "AdminAPI.MediaProxyCacheController.purge", security: [%{"oAuth" => ["write:media_proxy_caches"]}], + parameters: admin_api_params(), requestBody: request_body( "Parameters", diff --git a/lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex b/lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex index fbc9f80d7..a75f3e622 100644 --- a/lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/oauth_app_operation.ex @@ -36,6 +36,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do %Schema{type: :integer, default: 50}, "Number of apps to return" ) + | admin_api_params() ], responses: %{ 200 => @@ -72,6 +73,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do summary: "Create OAuth App", operationId: "AdminAPI.OAuthAppController.create", requestBody: request_body("Parameters", create_request()), + parameters: admin_api_params(), security: [%{"oAuth" => ["write"]}], responses: %{ 200 => Operation.response("App", "application/json", oauth_app()), @@ -85,7 +87,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do tags: ["Admin", "oAuth Apps"], summary: "Update OAuth App", operationId: "AdminAPI.OAuthAppController.update", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["write"]}], requestBody: request_body("Parameters", update_request()), responses: %{ @@ -103,7 +105,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do tags: ["Admin", "oAuth Apps"], summary: "Delete OAuth App", operationId: "AdminAPI.OAuthAppController.delete", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["write"]}], responses: %{ 204 => no_content_response(), diff --git a/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex b/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex index 7672cb467..67ee5eee0 100644 --- a/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/relay_operation.ex @@ -19,6 +19,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do summary: "List Relays", operationId: "AdminAPI.RelayController.index", security: [%{"oAuth" => ["read"]}], + parameters: admin_api_params(), responses: %{ 200 => Operation.response("Response", "application/json", %Schema{ @@ -41,6 +42,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do summary: "Follow a Relay", operationId: "AdminAPI.RelayController.follow", security: [%{"oAuth" => ["write:follows"]}], + parameters: admin_api_params(), requestBody: request_body("Parameters", %Schema{ type: :object, @@ -64,6 +66,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RelayOperation do summary: "Unfollow a Relay", operationId: "AdminAPI.RelayController.unfollow", security: [%{"oAuth" => ["write:follows"]}], + parameters: admin_api_params(), requestBody: request_body("Parameters", %Schema{ type: :object, diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index 15e78bfaf..3bb7ec49e 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -48,6 +48,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do %Schema{type: :integer, default: 50}, "Number number of log entries per page" ) + | admin_api_params() ], responses: %{ 200 => @@ -71,7 +72,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do tags: ["Admin", "Reports"], summary: "Get an individual report", operationId: "AdminAPI.ReportController.show", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["read:reports"]}], responses: %{ 200 => Operation.response("Report", "application/json", report()), @@ -86,6 +87,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do summary: "Change the state of one or multiple reports", operationId: "AdminAPI.ReportController.update", security: [%{"oAuth" => ["write:reports"]}], + parameters: admin_api_params(), requestBody: request_body("Parameters", update_request(), required: true), responses: %{ 204 => no_content_response(), @@ -100,7 +102,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do tags: ["Admin", "Reports"], summary: "Create report note", operationId: "AdminAPI.ReportController.notes_create", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], requestBody: request_body("Parameters", %Schema{ type: :object, @@ -124,6 +126,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do parameters: [ Operation.parameter(:report_id, :path, :string, "Report ID"), Operation.parameter(:id, :path, :string, "Note ID") + | admin_api_params() ], security: [%{"oAuth" => ["write:reports"]}], responses: %{ diff --git a/lib/pleroma/web/api_spec/operations/admin/status_operation.ex b/lib/pleroma/web/api_spec/operations/admin/status_operation.ex index 745399b4b..c105838a4 100644 --- a/lib/pleroma/web/api_spec/operations/admin/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/status_operation.ex @@ -55,6 +55,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do %Schema{type: :integer, default: 50}, "Number of statuses to return" ) + | admin_api_params() ], responses: %{ 200 => @@ -71,7 +72,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do tags: ["Admin", "Statuses"], summary: "Show Status", operationId: "AdminAPI.StatusController.show", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["read:statuses"]}], responses: %{ 200 => Operation.response("Status", "application/json", status()), @@ -85,7 +86,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do tags: ["Admin", "Statuses"], summary: "Change the scope of an individual reported status", operationId: "AdminAPI.StatusController.update", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["write:statuses"]}], requestBody: request_body("Parameters", update_request(), required: true), responses: %{ @@ -100,7 +101,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.StatusOperation do tags: ["Admin", "Statuses"], summary: "Delete an individual reported status", operationId: "AdminAPI.StatusController.delete", - parameters: [id_param()], + parameters: [id_param() | admin_api_params()], security: [%{"oAuth" => ["write:statuses"]}], responses: %{ 200 => empty_object_response(), -- cgit v1.2.3 From 1dd767b8c7b7565ad94ccb85324e97fa9885923e Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Tue, 14 Jul 2020 21:44:08 +0300 Subject: Include port in host for signatures --- lib/pleroma/web/activity_pub/publisher.ex | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index b70cbd043..d88f7f3ee 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -49,7 +49,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do """ def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do Logger.debug("Federating #{id} to #{inbox}") - %{host: host, path: path} = URI.parse(inbox) + + uri = URI.parse(inbox) digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64()) @@ -57,8 +58,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do signature = Pleroma.Signature.sign(actor, %{ - "(request-target)": "post #{path}", - host: host, + "(request-target)": "post #{uri.path}", + host: signature_host(uri), "content-length": byte_size(json), digest: digest, date: date @@ -76,8 +77,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {"digest", digest} ] ) do - if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since], - do: Instances.set_reachable(inbox) + if not Map.has_key?(params, :unreachable_since) || params[:unreachable_since] do + Instances.set_reachable(inbox) + end result else @@ -96,6 +98,14 @@ defmodule Pleroma.Web.ActivityPub.Publisher do |> publish_one() end + defp signature_host(%URI{port: port, scheme: scheme, host: host}) do + if port == URI.default_port(scheme) do + host + else + "#{host}:#{port}" + end + end + defp should_federate?(inbox, public) do if public do true -- cgit v1.2.3 From 58a4f350a8bc361d793cb96442f856362c18f195 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 6 May 2020 01:51:10 +0300 Subject: Refactor gun pooling and simplify adapter option insertion This patch refactors gun pooling to use Elixir process registry and simplifies adapter option insertion. Having the pool use process registry instead of a GenServer has a number of advantages: - Simpler code: the initial implementation adds about half the lines of code it deletes - Concurrency: unlike a GenServer, ETS-based registry can handle multiple checkout/checkin requests at the same time - Precise and easy idle connection clousure: current proposal for closing idle connections in the GenServer-based pool needs to filter through all connections once a minute and compare their last active time with closing time. With Elixir process registry this can be done by just using `Process.send_after`/`Process.cancel_timer` in the worker process. - Lower memory footprint: In my tests `gun-memory-leak` branch uses about 290mb on peak load (250 connections) and 235mb on idle (5-10 connections). Registry-based pool uses 210mb on idle and 240mb on peak load --- lib/pleroma/application.ex | 8 +- lib/pleroma/gun/conn.ex | 78 +------- lib/pleroma/gun/connection_pool.ex | 129 +++++++++++++ lib/pleroma/gun/connection_pool/worker.ex | 95 ++++++++++ lib/pleroma/http/adapter_helper.ex | 133 ++++++++++++-- lib/pleroma/http/adapter_helper/default.ex | 17 ++ lib/pleroma/http/adapter_helper/gun.ex | 32 +--- lib/pleroma/http/adapter_helper/hackney.ex | 3 + lib/pleroma/http/connection.ex | 124 ------------- lib/pleroma/http/http.ex | 53 ++---- lib/pleroma/pool/connections.ex | 283 ----------------------------- lib/pleroma/pool/pool.ex | 22 --- lib/pleroma/pool/request.ex | 65 ------- lib/pleroma/pool/supervisor.ex | 42 ----- lib/pleroma/reverse_proxy/client/tesla.ex | 2 +- 15 files changed, 400 insertions(+), 686 deletions(-) create mode 100644 lib/pleroma/gun/connection_pool.ex create mode 100644 lib/pleroma/gun/connection_pool/worker.ex create mode 100644 lib/pleroma/http/adapter_helper/default.ex delete mode 100644 lib/pleroma/http/connection.ex delete mode 100644 lib/pleroma/pool/connections.ex delete mode 100644 lib/pleroma/pool/pool.ex delete mode 100644 lib/pleroma/pool/request.ex delete mode 100644 lib/pleroma/pool/supervisor.ex (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 3282c6882..be14c1f9f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -223,9 +223,7 @@ defmodule Pleroma.Application do # start hackney and gun pools in tests defp http_children(_, :test) do - hackney_options = Config.get([:hackney_pools, :federation]) - hackney_pool = :hackney_pool.child_spec(:federation, hackney_options) - [hackney_pool, Pleroma.Pool.Supervisor] + http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil) end defp http_children(Tesla.Adapter.Hackney, _) do @@ -244,7 +242,9 @@ defmodule Pleroma.Application do end end - defp http_children(Tesla.Adapter.Gun, _), do: [Pleroma.Pool.Supervisor] + defp http_children(Tesla.Adapter.Gun, _) do + [{Registry, keys: :unique, name: Pleroma.Gun.ConnectionPool}] + end defp http_children(_, _), do: [] end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index cd25a2e74..77f78c7ff 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -3,40 +3,11 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Gun.Conn do - @moduledoc """ - Struct for gun connection data - """ alias Pleroma.Gun - alias Pleroma.Pool.Connections require Logger - @type gun_state :: :up | :down - @type conn_state :: :active | :idle - - @type t :: %__MODULE__{ - conn: pid(), - gun_state: gun_state(), - conn_state: conn_state(), - used_by: [pid()], - last_reference: pos_integer(), - crf: float(), - retries: pos_integer() - } - - defstruct conn: nil, - gun_state: :open, - conn_state: :init, - used_by: [], - last_reference: 0, - crf: 1, - retries: 0 - - @spec open(String.t() | URI.t(), atom(), keyword()) :: :ok | nil - def open(url, name, opts \\ []) - def open(url, name, opts) when is_binary(url), do: open(URI.parse(url), name, opts) - - def open(%URI{} = uri, name, opts) do + def open(%URI{} = uri, opts) do pool_opts = Pleroma.Config.get([:connections_pool], []) opts = @@ -45,30 +16,10 @@ defmodule Pleroma.Gun.Conn do |> Map.put_new(:retry, pool_opts[:retry] || 1) |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) + |> Map.put_new(:supervise, false) |> maybe_add_tls_opts(uri) - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - - max_connections = pool_opts[:max_connections] || 250 - - conn_pid = - if Connections.count(name) < max_connections do - do_open(uri, opts) - else - close_least_used_and_do_open(name, uri, opts) - end - - if is_pid(conn_pid) do - conn = %Pleroma.Gun.Conn{ - conn: conn_pid, - gun_state: :up, - conn_state: :active, - last_reference: :os.system_time(:second) - } - - :ok = Gun.set_owner(conn_pid, Process.whereis(name)) - Connections.add_conn(name, key, conn) - end + do_open(uri, opts) end defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts @@ -81,7 +32,7 @@ defmodule Pleroma.Gun.Conn do reuse_sessions: false, verify_fun: {&:ssl_verify_hostname.verify_fun/3, - [check_hostname: Pleroma.HTTP.Connection.format_host(host)]} + [check_hostname: Pleroma.HTTP.AdapterHelper.format_host(host)]} ] tls_opts = @@ -105,7 +56,7 @@ defmodule Pleroma.Gun.Conn do {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]), stream <- Gun.connect(conn, connect_opts), {:response, :fin, 200, _} <- Gun.await(conn, stream) do - conn + {:ok, conn} else error -> Logger.warn( @@ -141,7 +92,7 @@ defmodule Pleroma.Gun.Conn do with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do - conn + {:ok, conn} else error -> Logger.warn( @@ -155,11 +106,11 @@ defmodule Pleroma.Gun.Conn do end defp do_open(%URI{host: host, port: port} = uri, opts) do - host = Pleroma.HTTP.Connection.parse_host(host) + host = Pleroma.HTTP.AdapterHelper.parse_host(host) with {:ok, conn} <- Gun.open(host, port, opts), {:ok, _} <- Gun.await_up(conn, opts[:await_up_timeout]) do - conn + {:ok, conn} else error -> Logger.warn( @@ -171,7 +122,7 @@ defmodule Pleroma.Gun.Conn do end defp destination_opts(%URI{host: host, port: port}) do - host = Pleroma.HTTP.Connection.parse_host(host) + host = Pleroma.HTTP.AdapterHelper.parse_host(host) %{host: host, port: port} end @@ -181,17 +132,6 @@ defmodule Pleroma.Gun.Conn do defp add_http2_opts(opts, _, _), do: opts - defp close_least_used_and_do_open(name, uri, opts) do - with [{key, conn} | _conns] <- Connections.get_unused_conns(name), - :ok <- Gun.close(conn.conn) do - Connections.remove_conn(name, key) - - do_open(uri, opts) - else - [] -> {:error, :pool_overflowed} - end - end - def compose_uri_log(%URI{scheme: scheme, host: host, path: path}) do "#{scheme}://#{host}#{path}" end diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex new file mode 100644 index 000000000..e6abee69c --- /dev/null +++ b/lib/pleroma/gun/connection_pool.ex @@ -0,0 +1,129 @@ +defmodule Pleroma.Gun.ConnectionPool do + @registry __MODULE__ + + def get_conn(uri, opts) do + case enforce_pool_limits() do + :ok -> + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + + case Registry.lookup(@registry, key) do + # The key has already been registered, but connection is not up yet + [{worker_pid, {nil, _used_by, _crf, _last_reference}}] -> + get_gun_pid_from_worker(worker_pid) + + [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> + GenServer.cast(worker_pid, {:add_client, self(), false}) + {:ok, gun_pid} + + [] -> + # :gun.set_owner fails in :connected state for whatevever reason, + # so we open the connection in the process directly and send it's pid back + # We trust gun to handle timeouts by itself + case GenServer.start(Pleroma.Gun.ConnectionPool.Worker, [uri, key, opts, self()], + timeout: :infinity + ) do + {:ok, _worker_pid} -> + receive do + {:conn_pid, pid} -> {:ok, pid} + end + + {:error, {:error, {:already_registered, worker_pid}}} -> + get_gun_pid_from_worker(worker_pid) + + err -> + err + end + end + + :error -> + {:error, :pool_full} + end + end + + @enforcer_key "enforcer" + defp enforce_pool_limits() do + max_connections = Pleroma.Config.get([:connections_pool, :max_connections]) + + if Registry.count(@registry) >= max_connections do + case Registry.lookup(@registry, @enforcer_key) do + [] -> + pid = + spawn(fn -> + {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil) + + reclaim_max = + [:connections_pool, :reclaim_multiplier] + |> Pleroma.Config.get() + |> Kernel.*(max_connections) + |> round + |> max(1) + + unused_conns = + Registry.select( + @registry, + [ + {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], + [{{:"$1", :"$3", :"$4"}}]} + ] + ) + + case unused_conns do + [] -> + exit(:pool_full) + + unused_conns -> + unused_conns + |> Enum.sort(fn {_pid1, crf1, last_reference1}, + {_pid2, crf2, last_reference2} -> + crf1 <= crf2 and last_reference1 <= last_reference2 + end) + |> Enum.take(reclaim_max) + |> Enum.each(fn {pid, _, _} -> GenServer.call(pid, :idle_close) end) + end + end) + + wait_for_enforcer_finish(pid) + + [{pid, _}] -> + wait_for_enforcer_finish(pid) + end + else + :ok + end + end + + defp wait_for_enforcer_finish(pid) do + ref = Process.monitor(pid) + + receive do + {:DOWN, ^ref, :process, ^pid, :pool_full} -> + :error + + {:DOWN, ^ref, :process, ^pid, :normal} -> + :ok + end + end + + defp get_gun_pid_from_worker(worker_pid) do + # GenServer.call will block the process for timeout length if + # the server crashes on startup (which will happen if gun fails to connect) + # so instead we use cast + monitor + + ref = Process.monitor(worker_pid) + GenServer.cast(worker_pid, {:add_client, self(), true}) + + receive do + {:conn_pid, pid} -> {:ok, pid} + {:DOWN, ^ref, :process, ^worker_pid, reason} -> reason + end + end + + def release_conn(conn_pid) do + [worker_pid] = + Registry.select(@registry, [ + {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} + ]) + + GenServer.cast(worker_pid, {:remove_client, self()}) + end +end diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex new file mode 100644 index 000000000..ebde4bbf6 --- /dev/null +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -0,0 +1,95 @@ +defmodule Pleroma.Gun.ConnectionPool.Worker do + alias Pleroma.Gun + use GenServer + + @registry Pleroma.Gun.ConnectionPool + + @impl true + def init([uri, key, opts, client_pid]) do + time = :os.system_time(:second) + # Register before opening connection to prevent race conditions + with {:ok, _owner} <- Registry.register(@registry, key, {nil, [client_pid], 1, time}), + {:ok, conn_pid} <- Gun.Conn.open(uri, opts), + Process.link(conn_pid) do + {_, _} = + Registry.update_value(@registry, key, fn {_, used_by, crf, last_reference} -> + {conn_pid, used_by, crf, last_reference} + end) + + send(client_pid, {:conn_pid, conn_pid}) + {:ok, %{key: key, timer: nil}, :hibernate} + else + err -> {:stop, err} + end + end + + @impl true + def handle_cast({:add_client, client_pid, send_pid_back}, %{key: key} = state) do + time = :os.system_time(:second) + + {{conn_pid, _, _, _}, _} = + Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> + {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time} + end) + + if send_pid_back, do: send(client_pid, {:conn_pid, conn_pid}) + + state = + if state.timer != nil do + Process.cancel_timer(state[:timer]) + %{state | timer: nil} + else + state + end + + {:noreply, state, :hibernate} + end + + @impl true + def handle_cast({:remove_client, client_pid}, %{key: key} = state) do + {{_conn_pid, used_by, _crf, _last_reference}, _} = + Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> + {conn_pid, List.delete(used_by, client_pid), crf, last_reference} + end) + + timer = + if used_by == [] do + max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) + Process.send_after(self(), :idle_close, max_idle) + else + nil + end + + {:noreply, %{state | timer: timer}, :hibernate} + end + + @impl true + def handle_info(:idle_close, state) do + # Gun monitors the owner process, and will close the connection automatically + # when it's terminated + {:stop, :normal, state} + end + + # Gracefully shutdown if the connection got closed without any streams left + @impl true + def handle_info({:gun_down, _pid, _protocol, _reason, []}, state) do + {:stop, :normal, state} + end + + # Otherwise, shutdown with an error + @impl true + def handle_info({:gun_down, _pid, _protocol, _reason, _killed_streams} = down_message, state) do + {:stop, {:error, down_message}, state} + end + + @impl true + def handle_call(:idle_close, _, %{key: key} = state) do + Registry.unregister(@registry, key) + {:stop, :normal, state} + end + + # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478 + defp crf(time_delta, prev_crf) do + 1 + :math.pow(0.5, time_delta / 100) * prev_crf + end +end diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 510722ff9..0532ea31d 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -3,7 +3,21 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.HTTP.AdapterHelper do - alias Pleroma.HTTP.Connection + @moduledoc """ + Configure Tesla.Client with default and customized adapter options. + """ + @defaults [pool: :federation] + + @type ip_address :: ipv4_address() | ipv6_address() + @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} + @type ipv6_address :: + {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} + @type proxy_type() :: :socks4 | :socks5 + @type host() :: charlist() | ip_address() + + alias Pleroma.Config + alias Pleroma.HTTP.AdapterHelper + require Logger @type proxy :: {Connection.host(), pos_integer()} @@ -11,24 +25,13 @@ defmodule Pleroma.HTTP.AdapterHelper do @callback options(keyword(), URI.t()) :: keyword() @callback after_request(keyword()) :: :ok - - @spec options(keyword(), URI.t()) :: keyword() - def options(opts, _uri) do - proxy = Pleroma.Config.get([:http, :proxy_url], nil) - maybe_add_proxy(opts, format_proxy(proxy)) - end - - @spec maybe_get_conn(URI.t(), keyword()) :: keyword() - def maybe_get_conn(_uri, opts), do: opts - - @spec after_request(keyword()) :: :ok - def after_request(_opts), do: :ok + @callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()} @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil def format_proxy(nil), do: nil def format_proxy(proxy_url) do - case Connection.parse_proxy(proxy_url) do + case parse_proxy(proxy_url) do {:ok, host, port} -> {host, port} {:ok, type, host, port} -> {type, host, port} _ -> nil @@ -38,4 +41,106 @@ defmodule Pleroma.HTTP.AdapterHelper do @spec maybe_add_proxy(keyword(), proxy() | nil) :: keyword() def maybe_add_proxy(opts, nil), do: opts def maybe_add_proxy(opts, proxy), do: Keyword.put_new(opts, :proxy, proxy) + + @doc """ + Merge default connection & adapter options with received ones. + """ + + @spec options(URI.t(), keyword()) :: keyword() + def options(%URI{} = uri, opts \\ []) do + @defaults + |> pool_timeout() + |> Keyword.merge(opts) + |> adapter_helper().options(uri) + end + + defp pool_timeout(opts) do + {config_key, default} = + if adapter() == Tesla.Adapter.Gun do + {:pools, Config.get([:pools, :default, :timeout])} + else + {:hackney_pools, 10_000} + end + + timeout = Config.get([config_key, opts[:pool], :timeout], default) + + Keyword.merge(opts, timeout: timeout) + end + + @spec after_request(keyword()) :: :ok + def after_request(opts), do: adapter_helper().after_request(opts) + + def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts) + defp adapter, do: Application.get_env(:tesla, :adapter) + + defp adapter_helper do + case adapter() do + Tesla.Adapter.Gun -> AdapterHelper.Gun + Tesla.Adapter.Hackney -> AdapterHelper.Hackney + _ -> AdapterHelper.Default + end + end + + @spec parse_proxy(String.t() | tuple() | nil) :: + {:ok, host(), pos_integer()} + | {:ok, proxy_type(), host(), pos_integer()} + | {:error, atom()} + | nil + + def parse_proxy(nil), do: nil + + def parse_proxy(proxy) when is_binary(proxy) do + with [host, port] <- String.split(proxy, ":"), + {port, ""} <- Integer.parse(port) do + {:ok, parse_host(host), port} + else + {_, _} -> + Logger.warn("Parsing port failed #{inspect(proxy)}") + {:error, :invalid_proxy_port} + + :error -> + Logger.warn("Parsing port failed #{inspect(proxy)}") + {:error, :invalid_proxy_port} + + _ -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}") + {:error, :invalid_proxy} + end + end + + def parse_proxy(proxy) when is_tuple(proxy) do + with {type, host, port} <- proxy do + {:ok, type, parse_host(host), port} + else + _ -> + Logger.warn("Parsing proxy failed #{inspect(proxy)}") + {:error, :invalid_proxy} + end + end + + @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() + def parse_host(host) when is_list(host), do: host + def parse_host(host) when is_atom(host), do: to_charlist(host) + + def parse_host(host) when is_binary(host) do + host = to_charlist(host) + + case :inet.parse_address(host) do + {:error, :einval} -> host + {:ok, ip} -> ip + end + end + + @spec format_host(String.t()) :: charlist() + def format_host(host) do + host_charlist = to_charlist(host) + + case :inet.parse_address(host_charlist) do + {:error, :einval} -> + :idna.encode(host_charlist) + + {:ok, _ip} -> + host_charlist + end + end end diff --git a/lib/pleroma/http/adapter_helper/default.ex b/lib/pleroma/http/adapter_helper/default.ex new file mode 100644 index 000000000..218cfacc0 --- /dev/null +++ b/lib/pleroma/http/adapter_helper/default.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.HTTP.AdapterHelper.Default do + alias Pleroma.HTTP.AdapterHelper + + @behaviour Pleroma.HTTP.AdapterHelper + + @spec options(keyword(), URI.t()) :: keyword() + def options(opts, _uri) do + proxy = Pleroma.Config.get([:http, :proxy_url], nil) + AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy)) + end + + @spec after_request(keyword()) :: :ok + def after_request(_opts), do: :ok + + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} + def get_conn(_uri, opts), do: {:ok, opts} +end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index ead7cdc6b..6f7cc9784 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -5,8 +5,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do @behaviour Pleroma.HTTP.AdapterHelper + alias Pleroma.Gun.ConnectionPool alias Pleroma.HTTP.AdapterHelper - alias Pleroma.Pool.Connections require Logger @@ -31,13 +31,13 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do |> Keyword.merge(config_opts) |> add_scheme_opts(uri) |> AdapterHelper.maybe_add_proxy(proxy) - |> maybe_get_conn(uri, incoming_opts) + |> Keyword.merge(incoming_opts) end @spec after_request(keyword()) :: :ok def after_request(opts) do if opts[:conn] && opts[:body_as] != :chunks do - Connections.checkout(opts[:conn], self(), :gun_connections) + ConnectionPool.release_conn(opts[:conn]) end :ok @@ -51,27 +51,11 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do |> Keyword.put(:tls_opts, log_level: :warning) end - defp maybe_get_conn(adapter_opts, uri, incoming_opts) do - {receive_conn?, opts} = - adapter_opts - |> Keyword.merge(incoming_opts) - |> Keyword.pop(:receive_conn, true) - - if Connections.alive?(:gun_connections) and receive_conn? do - checkin_conn(uri, opts) - else - opts - end - end - - defp checkin_conn(uri, opts) do - case Connections.checkin(uri, :gun_connections) do - nil -> - Task.start(Pleroma.Gun.Conn, :open, [uri, :gun_connections, opts]) - opts - - conn when is_pid(conn) -> - Keyword.merge(opts, conn: conn, close_conn: false) + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} + def get_conn(uri, opts) do + case ConnectionPool.get_conn(uri, opts) do + {:ok, conn_pid} -> {:ok, Keyword.merge(opts, conn: conn_pid, close_conn: false)} + err -> err end end end diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index 3972a03a9..42d552740 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -25,4 +25,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do defp add_scheme_opts(opts, _), do: opts def after_request(_), do: :ok + + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} + def get_conn(_uri, opts), do: {:ok, opts} end diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex deleted file mode 100644 index ebacf7902..000000000 --- a/lib/pleroma/http/connection.ex +++ /dev/null @@ -1,124 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.HTTP.Connection do - @moduledoc """ - Configure Tesla.Client with default and customized adapter options. - """ - - alias Pleroma.Config - alias Pleroma.HTTP.AdapterHelper - - require Logger - - @defaults [pool: :federation] - - @type ip_address :: ipv4_address() | ipv6_address() - @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} - @type ipv6_address :: - {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} - @type proxy_type() :: :socks4 | :socks5 - @type host() :: charlist() | ip_address() - - @doc """ - Merge default connection & adapter options with received ones. - """ - - @spec options(URI.t(), keyword()) :: keyword() - def options(%URI{} = uri, opts \\ []) do - @defaults - |> pool_timeout() - |> Keyword.merge(opts) - |> adapter_helper().options(uri) - end - - defp pool_timeout(opts) do - {config_key, default} = - if adapter() == Tesla.Adapter.Gun do - {:pools, Config.get([:pools, :default, :timeout])} - else - {:hackney_pools, 10_000} - end - - timeout = Config.get([config_key, opts[:pool], :timeout], default) - - Keyword.merge(opts, timeout: timeout) - end - - @spec after_request(keyword()) :: :ok - def after_request(opts), do: adapter_helper().after_request(opts) - - defp adapter, do: Application.get_env(:tesla, :adapter) - - defp adapter_helper do - case adapter() do - Tesla.Adapter.Gun -> AdapterHelper.Gun - Tesla.Adapter.Hackney -> AdapterHelper.Hackney - _ -> AdapterHelper - end - end - - @spec parse_proxy(String.t() | tuple() | nil) :: - {:ok, host(), pos_integer()} - | {:ok, proxy_type(), host(), pos_integer()} - | {:error, atom()} - | nil - - def parse_proxy(nil), do: nil - - def parse_proxy(proxy) when is_binary(proxy) do - with [host, port] <- String.split(proxy, ":"), - {port, ""} <- Integer.parse(port) do - {:ok, parse_host(host), port} - else - {_, _} -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - :error -> - Logger.warn("Parsing port failed #{inspect(proxy)}") - {:error, :invalid_proxy_port} - - _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") - {:error, :invalid_proxy} - end - end - - def parse_proxy(proxy) when is_tuple(proxy) do - with {type, host, port} <- proxy do - {:ok, type, parse_host(host), port} - else - _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") - {:error, :invalid_proxy} - end - end - - @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() - def parse_host(host) when is_list(host), do: host - def parse_host(host) when is_atom(host), do: to_charlist(host) - - def parse_host(host) when is_binary(host) do - host = to_charlist(host) - - case :inet.parse_address(host) do - {:error, :einval} -> host - {:ok, ip} -> ip - end - end - - @spec format_host(String.t()) :: charlist() - def format_host(host) do - host_charlist = to_charlist(host) - - case :inet.parse_address(host_charlist) do - {:error, :einval} -> - :idna.encode(host_charlist) - - {:ok, _ip} -> - host_charlist - end - end -end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 66ca75367..8ded76601 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -7,7 +7,7 @@ defmodule Pleroma.HTTP do Wrapper for `Tesla.request/2`. """ - alias Pleroma.HTTP.Connection + alias Pleroma.HTTP.AdapterHelper alias Pleroma.HTTP.Request alias Pleroma.HTTP.RequestBuilder, as: Builder alias Tesla.Client @@ -60,49 +60,26 @@ defmodule Pleroma.HTTP do {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) - adapter_opts = Connection.options(uri, options[:adapter] || []) - options = put_in(options[:adapter], adapter_opts) - params = options[:params] || [] - request = build_request(method, headers, options, url, body, params) + adapter_opts = AdapterHelper.options(uri, options[:adapter] || []) - adapter = Application.get_env(:tesla, :adapter) - client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) + case AdapterHelper.get_conn(uri, adapter_opts) do + {:ok, adapter_opts} -> + options = put_in(options[:adapter], adapter_opts) + params = options[:params] || [] + request = build_request(method, headers, options, url, body, params) - pid = Process.whereis(adapter_opts[:pool]) + adapter = Application.get_env(:tesla, :adapter) + client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) - pool_alive? = - if adapter == Tesla.Adapter.Gun && pid do - Process.alive?(pid) - else - false - end + response = request(client, request) - request_opts = - adapter_opts - |> Enum.into(%{}) - |> Map.put(:env, Pleroma.Config.get([:env])) - |> Map.put(:pool_alive?, pool_alive?) + AdapterHelper.after_request(adapter_opts) - response = request(client, request, request_opts) + response - Connection.after_request(adapter_opts) - - response - end - - @spec request(Client.t(), keyword(), map()) :: {:ok, Env.t()} | {:error, any()} - def request(%Client{} = client, request, %{env: :test}), do: request(client, request) - - def request(%Client{} = client, request, %{body_as: :chunks}), do: request(client, request) - - def request(%Client{} = client, request, %{pool_alive?: false}), do: request(client, request) - - def request(%Client{} = client, request, %{pool: pool, timeout: timeout}) do - :poolboy.transaction( - pool, - &Pleroma.Pool.Request.execute(&1, client, request, timeout), - timeout - ) + err -> + err + end end @spec request(Client.t(), keyword()) :: {:ok, Env.t()} | {:error, any()} diff --git a/lib/pleroma/pool/connections.ex b/lib/pleroma/pool/connections.ex deleted file mode 100644 index acafe1bea..000000000 --- a/lib/pleroma/pool/connections.ex +++ /dev/null @@ -1,283 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Connections do - use GenServer - - alias Pleroma.Config - alias Pleroma.Gun - - require Logger - - @type domain :: String.t() - @type conn :: Pleroma.Gun.Conn.t() - - @type t :: %__MODULE__{ - conns: %{domain() => conn()}, - opts: keyword() - } - - defstruct conns: %{}, opts: [] - - @spec start_link({atom(), keyword()}) :: {:ok, pid()} - def start_link({name, opts}) do - GenServer.start_link(__MODULE__, opts, name: name) - end - - @impl true - def init(opts), do: {:ok, %__MODULE__{conns: %{}, opts: opts}} - - @spec checkin(String.t() | URI.t(), atom()) :: pid() | nil - def checkin(url, name) - def checkin(url, name) when is_binary(url), do: checkin(URI.parse(url), name) - - def checkin(%URI{} = uri, name) do - timeout = Config.get([:connections_pool, :checkin_timeout], 250) - - GenServer.call(name, {:checkin, uri}, timeout) - end - - @spec alive?(atom()) :: boolean() - def alive?(name) do - if pid = Process.whereis(name) do - Process.alive?(pid) - else - false - end - end - - @spec get_state(atom()) :: t() - def get_state(name) do - GenServer.call(name, :state) - end - - @spec count(atom()) :: pos_integer() - def count(name) do - GenServer.call(name, :count) - end - - @spec get_unused_conns(atom()) :: [{domain(), conn()}] - def get_unused_conns(name) do - GenServer.call(name, :unused_conns) - end - - @spec checkout(pid(), pid(), atom()) :: :ok - def checkout(conn, pid, name) do - GenServer.cast(name, {:checkout, conn, pid}) - end - - @spec add_conn(atom(), String.t(), Pleroma.Gun.Conn.t()) :: :ok - def add_conn(name, key, conn) do - GenServer.cast(name, {:add_conn, key, conn}) - end - - @spec remove_conn(atom(), String.t()) :: :ok - def remove_conn(name, key) do - GenServer.cast(name, {:remove_conn, key}) - end - - @impl true - def handle_cast({:add_conn, key, conn}, state) do - state = put_in(state.conns[key], conn) - - Process.monitor(conn.conn) - {:noreply, state} - end - - @impl true - def handle_cast({:checkout, conn_pid, pid}, state) do - state = - with true <- Process.alive?(conn_pid), - {key, conn} <- find_conn(state.conns, conn_pid), - used_by <- List.keydelete(conn.used_by, pid, 0) do - conn_state = if used_by == [], do: :idle, else: conn.conn_state - - put_in(state.conns[key], %{conn | conn_state: conn_state, used_by: used_by}) - else - false -> - Logger.debug("checkout for closed conn #{inspect(conn_pid)}") - state - - nil -> - Logger.debug("checkout for alive conn #{inspect(conn_pid)}, but is not in state") - state - end - - {:noreply, state} - end - - @impl true - def handle_cast({:remove_conn, key}, state) do - state = put_in(state.conns, Map.delete(state.conns, key)) - {:noreply, state} - end - - @impl true - def handle_call({:checkin, uri}, from, state) do - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - - case state.conns[key] do - %{conn: pid, gun_state: :up} = conn -> - time = :os.system_time(:second) - last_reference = time - conn.last_reference - crf = crf(last_reference, 100, conn.crf) - - state = - put_in(state.conns[key], %{ - conn - | last_reference: time, - crf: crf, - conn_state: :active, - used_by: [from | conn.used_by] - }) - - {:reply, pid, state} - - %{gun_state: :down} -> - {:reply, nil, state} - - nil -> - {:reply, nil, state} - end - end - - @impl true - def handle_call(:state, _from, state), do: {:reply, state, state} - - @impl true - def handle_call(:count, _from, state) do - {:reply, Enum.count(state.conns), state} - end - - @impl true - def handle_call(:unused_conns, _from, state) do - unused_conns = - state.conns - |> Enum.filter(&filter_conns/1) - |> Enum.sort(&sort_conns/2) - - {:reply, unused_conns, state} - end - - defp filter_conns({_, %{conn_state: :idle, used_by: []}}), do: true - defp filter_conns(_), do: false - - defp sort_conns({_, c1}, {_, c2}) do - c1.crf <= c2.crf and c1.last_reference <= c2.last_reference - end - - @impl true - def handle_info({:gun_up, conn_pid, _protocol}, state) do - %{origin_host: host, origin_scheme: scheme, origin_port: port} = Gun.info(conn_pid) - - host = - case :inet.ntoa(host) do - {:error, :einval} -> host - ip -> ip - end - - key = "#{scheme}:#{host}:#{port}" - - state = - with {key, conn} <- find_conn(state.conns, conn_pid, key), - {true, key} <- {Process.alive?(conn_pid), key} do - put_in(state.conns[key], %{ - conn - | gun_state: :up, - conn_state: :active, - retries: 0 - }) - else - {false, key} -> - put_in( - state.conns, - Map.delete(state.conns, key) - ) - - nil -> - :ok = Gun.close(conn_pid) - - state - end - - {:noreply, state} - end - - @impl true - def handle_info({:gun_down, conn_pid, _protocol, _reason, _killed}, state) do - retries = Config.get([:connections_pool, :retry], 1) - # we can't get info on this pid, because pid is dead - state = - with {key, conn} <- find_conn(state.conns, conn_pid), - {true, key} <- {Process.alive?(conn_pid), key} do - if conn.retries == retries do - :ok = Gun.close(conn.conn) - - put_in( - state.conns, - Map.delete(state.conns, key) - ) - else - put_in(state.conns[key], %{ - conn - | gun_state: :down, - retries: conn.retries + 1 - }) - end - else - {false, key} -> - put_in( - state.conns, - Map.delete(state.conns, key) - ) - - nil -> - Logger.debug(":gun_down for conn which isn't found in state") - - state - end - - {:noreply, state} - end - - @impl true - def handle_info({:DOWN, _ref, :process, conn_pid, reason}, state) do - Logger.debug("received DOWN message for #{inspect(conn_pid)} reason -> #{inspect(reason)}") - - state = - with {key, conn} <- find_conn(state.conns, conn_pid) do - Enum.each(conn.used_by, fn {pid, _ref} -> - Process.exit(pid, reason) - end) - - put_in( - state.conns, - Map.delete(state.conns, key) - ) - else - nil -> - Logger.debug(":DOWN for conn which isn't found in state") - - state - end - - {:noreply, state} - end - - defp find_conn(conns, conn_pid) do - Enum.find(conns, fn {_key, conn} -> - conn.conn == conn_pid - end) - end - - defp find_conn(conns, conn_pid, conn_key) do - Enum.find(conns, fn {key, conn} -> - key == conn_key and conn.conn == conn_pid - end) - end - - def crf(current, steps, crf) do - 1 + :math.pow(0.5, current / steps) * crf - end -end diff --git a/lib/pleroma/pool/pool.ex b/lib/pleroma/pool/pool.ex deleted file mode 100644 index 21a6fbbc5..000000000 --- a/lib/pleroma/pool/pool.ex +++ /dev/null @@ -1,22 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool do - def child_spec(opts) do - poolboy_opts = - opts - |> Keyword.put(:worker_module, Pleroma.Pool.Request) - |> Keyword.put(:name, {:local, opts[:name]}) - |> Keyword.put(:size, opts[:size]) - |> Keyword.put(:max_overflow, opts[:max_overflow]) - - %{ - id: opts[:id] || {__MODULE__, make_ref()}, - start: {:poolboy, :start_link, [poolboy_opts, [name: opts[:name]]]}, - restart: :permanent, - shutdown: 5000, - type: :worker - } - end -end diff --git a/lib/pleroma/pool/request.ex b/lib/pleroma/pool/request.ex deleted file mode 100644 index 3fb930db7..000000000 --- a/lib/pleroma/pool/request.ex +++ /dev/null @@ -1,65 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Request do - use GenServer - - require Logger - - def start_link(args) do - GenServer.start_link(__MODULE__, args) - end - - @impl true - def init(_), do: {:ok, []} - - @spec execute(pid() | atom(), Tesla.Client.t(), keyword(), pos_integer()) :: - {:ok, Tesla.Env.t()} | {:error, any()} - def execute(pid, client, request, timeout) do - GenServer.call(pid, {:execute, client, request}, timeout) - end - - @impl true - def handle_call({:execute, client, request}, _from, state) do - response = Pleroma.HTTP.request(client, request) - - {:reply, response, state} - end - - @impl true - def handle_info({:gun_data, _conn, _stream, _, _}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_up, _conn, _protocol}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_down, _conn, _protocol, _reason, _killed}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_error, _conn, _stream, _error}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_push, _conn, _stream, _new_stream, _method, _uri, _headers}, state) do - {:noreply, state} - end - - @impl true - def handle_info({:gun_response, _conn, _stream, _, _status, _headers}, state) do - {:noreply, state} - end - - @impl true - def handle_info(msg, state) do - Logger.warn("Received unexpected message #{inspect(__MODULE__)} #{inspect(msg)}") - {:noreply, state} - end -end diff --git a/lib/pleroma/pool/supervisor.ex b/lib/pleroma/pool/supervisor.ex deleted file mode 100644 index faf646cb2..000000000 --- a/lib/pleroma/pool/supervisor.ex +++ /dev/null @@ -1,42 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Pool.Supervisor do - use Supervisor - - alias Pleroma.Config - alias Pleroma.Pool - - def start_link(args) do - Supervisor.start_link(__MODULE__, args, name: __MODULE__) - end - - def init(_) do - conns_child = %{ - id: Pool.Connections, - start: - {Pool.Connections, :start_link, [{:gun_connections, Config.get([:connections_pool])}]} - } - - Supervisor.init([conns_child | pools()], strategy: :one_for_one) - end - - defp pools do - pools = Config.get(:pools) - - pools = - if Config.get([Pleroma.Upload, :proxy_remote]) == false do - Keyword.delete(pools, :upload) - else - pools - end - - for {pool_name, pool_opts} <- pools do - pool_opts - |> Keyword.put(:id, {Pool, pool_name}) - |> Keyword.put(:name, pool_name) - |> Pool.child_spec() - end - end -end diff --git a/lib/pleroma/reverse_proxy/client/tesla.ex b/lib/pleroma/reverse_proxy/client/tesla.ex index e81ea8bde..65785445d 100644 --- a/lib/pleroma/reverse_proxy/client/tesla.ex +++ b/lib/pleroma/reverse_proxy/client/tesla.ex @@ -48,7 +48,7 @@ defmodule Pleroma.ReverseProxy.Client.Tesla do # if there were redirects we need to checkout old conn conn = opts[:old_conn] || opts[:conn] - if conn, do: :ok = Pleroma.Pool.Connections.checkout(conn, self(), :gun_connections) + if conn, do: :ok = Pleroma.Gun.ConnectionPool.release_conn(conn) :done end -- cgit v1.2.3 From fffbcffb8c9ce1e96de5d1a5e15005e271deacd4 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 6 May 2020 21:41:34 +0300 Subject: Connection Pool: don't enforce pool limits if no new connection needs to be opened --- lib/pleroma/gun/connection_pool.ex | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex index e6abee69c..ed7ddff81 100644 --- a/lib/pleroma/gun/connection_pool.ex +++ b/lib/pleroma/gun/connection_pool.ex @@ -2,20 +2,20 @@ defmodule Pleroma.Gun.ConnectionPool do @registry __MODULE__ def get_conn(uri, opts) do - case enforce_pool_limits() do - :ok -> - key = "#{uri.scheme}:#{uri.host}:#{uri.port}" + key = "#{uri.scheme}:#{uri.host}:#{uri.port}" - case Registry.lookup(@registry, key) do - # The key has already been registered, but connection is not up yet - [{worker_pid, {nil, _used_by, _crf, _last_reference}}] -> - get_gun_pid_from_worker(worker_pid) + case Registry.lookup(@registry, key) do + # The key has already been registered, but connection is not up yet + [{worker_pid, {nil, _used_by, _crf, _last_reference}}] -> + get_gun_pid_from_worker(worker_pid) - [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> - GenServer.cast(worker_pid, {:add_client, self(), false}) - {:ok, gun_pid} + [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> + GenServer.cast(worker_pid, {:add_client, self(), false}) + {:ok, gun_pid} - [] -> + [] -> + case enforce_pool_limits() do + :ok -> # :gun.set_owner fails in :connected state for whatevever reason, # so we open the connection in the process directly and send it's pid back # We trust gun to handle timeouts by itself @@ -33,10 +33,10 @@ defmodule Pleroma.Gun.ConnectionPool do err -> err end - end - :error -> - {:error, :pool_full} + :error -> + {:error, :pool_full} + end end end -- cgit v1.2.3 From d08b1576990ca33ac4178fb757ec03a777c55b5b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 6 May 2020 21:51:10 +0300 Subject: Connection pool: check that there actually is a result Sometimes connections died before being released to the pool, resulting in MatchErrors --- lib/pleroma/gun/connection_pool.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex index ed7ddff81..0daf1da44 100644 --- a/lib/pleroma/gun/connection_pool.ex +++ b/lib/pleroma/gun/connection_pool.ex @@ -119,11 +119,17 @@ defmodule Pleroma.Gun.ConnectionPool do end def release_conn(conn_pid) do - [worker_pid] = + query_result = Registry.select(@registry, [ {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} ]) - GenServer.cast(worker_pid, {:remove_client, self()}) + case query_result do + [worker_pid] -> + GenServer.cast(worker_pid, {:remove_client, self()}) + + [] -> + :ok + end end end -- cgit v1.2.3 From ec9d0d146b4ec6752f8f2896ace9bb5585469773 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 6 May 2020 23:14:24 +0300 Subject: Connection pool: Fix race conditions in limit enforcement Fixes race conditions in limit enforcement by putting worker processes in a DynamicSupervisor --- lib/pleroma/application.ex | 2 +- lib/pleroma/gun/connection_pool.ex | 105 +++++---------------- lib/pleroma/gun/connection_pool/worker.ex | 12 +-- .../gun/connection_pool/worker_supervisor.ex | 91 ++++++++++++++++++ 4 files changed, 118 insertions(+), 92 deletions(-) create mode 100644 lib/pleroma/gun/connection_pool/worker_supervisor.ex (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index be14c1f9f..cfdaf1770 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -243,7 +243,7 @@ defmodule Pleroma.Application do end defp http_children(Tesla.Adapter.Gun, _) do - [{Registry, keys: :unique, name: Pleroma.Gun.ConnectionPool}] + Pleroma.Gun.ConnectionPool.children() end defp http_children(_, _), do: [] diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex index 0daf1da44..545bfaf7f 100644 --- a/lib/pleroma/gun/connection_pool.ex +++ b/lib/pleroma/gun/connection_pool.ex @@ -1,6 +1,15 @@ defmodule Pleroma.Gun.ConnectionPool do @registry __MODULE__ + alias Pleroma.Gun.ConnectionPool.WorkerSupervisor + + def children do + [ + {Registry, keys: :unique, name: @registry}, + Pleroma.Gun.ConnectionPool.WorkerSupervisor + ] + end + def get_conn(uri, opts) do key = "#{uri.scheme}:#{uri.host}:#{uri.port}" @@ -14,93 +23,21 @@ defmodule Pleroma.Gun.ConnectionPool do {:ok, gun_pid} [] -> - case enforce_pool_limits() do - :ok -> - # :gun.set_owner fails in :connected state for whatevever reason, - # so we open the connection in the process directly and send it's pid back - # We trust gun to handle timeouts by itself - case GenServer.start(Pleroma.Gun.ConnectionPool.Worker, [uri, key, opts, self()], - timeout: :infinity - ) do - {:ok, _worker_pid} -> - receive do - {:conn_pid, pid} -> {:ok, pid} - end - - {:error, {:error, {:already_registered, worker_pid}}} -> - get_gun_pid_from_worker(worker_pid) - - err -> - err + # :gun.set_owner fails in :connected state for whatevever reason, + # so we open the connection in the process directly and send it's pid back + # We trust gun to handle timeouts by itself + case WorkerSupervisor.start_worker([uri, key, opts, self()]) do + {:ok, _worker_pid} -> + receive do + {:conn_pid, pid} -> {:ok, pid} end - :error -> - {:error, :pool_full} - end - end - end - - @enforcer_key "enforcer" - defp enforce_pool_limits() do - max_connections = Pleroma.Config.get([:connections_pool, :max_connections]) - - if Registry.count(@registry) >= max_connections do - case Registry.lookup(@registry, @enforcer_key) do - [] -> - pid = - spawn(fn -> - {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil) - - reclaim_max = - [:connections_pool, :reclaim_multiplier] - |> Pleroma.Config.get() - |> Kernel.*(max_connections) - |> round - |> max(1) - - unused_conns = - Registry.select( - @registry, - [ - {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], - [{{:"$1", :"$3", :"$4"}}]} - ] - ) + {:error, {:error, {:already_registered, worker_pid}}} -> + get_gun_pid_from_worker(worker_pid) - case unused_conns do - [] -> - exit(:pool_full) - - unused_conns -> - unused_conns - |> Enum.sort(fn {_pid1, crf1, last_reference1}, - {_pid2, crf2, last_reference2} -> - crf1 <= crf2 and last_reference1 <= last_reference2 - end) - |> Enum.take(reclaim_max) - |> Enum.each(fn {pid, _, _} -> GenServer.call(pid, :idle_close) end) - end - end) - - wait_for_enforcer_finish(pid) - - [{pid, _}] -> - wait_for_enforcer_finish(pid) - end - else - :ok - end - end - - defp wait_for_enforcer_finish(pid) do - ref = Process.monitor(pid) - - receive do - {:DOWN, ^ref, :process, ^pid, :pool_full} -> - :error - - {:DOWN, ^ref, :process, ^pid, :normal} -> - :ok + err -> + err + end end end diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index ebde4bbf6..25fafc64c 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -1,9 +1,13 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do alias Pleroma.Gun - use GenServer + use GenServer, restart: :temporary @registry Pleroma.Gun.ConnectionPool + def start_link(opts) do + GenServer.start_link(__MODULE__, opts) + end + @impl true def init([uri, key, opts, client_pid]) do time = :os.system_time(:second) @@ -82,12 +86,6 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do {:stop, {:error, down_message}, state} end - @impl true - def handle_call(:idle_close, _, %{key: key} = state) do - Registry.unregister(@registry, key) - {:stop, :normal, state} - end - # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478 defp crf(time_delta, prev_crf) do 1 + :math.pow(0.5, time_delta / 100) * prev_crf diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex new file mode 100644 index 000000000..5b546bd87 --- /dev/null +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -0,0 +1,91 @@ +defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do + @doc "Supervisor for pool workers. Does not do anything except enforce max connection limit" + + use DynamicSupervisor + + def start_link(opts) do + DynamicSupervisor.start_link(__MODULE__, opts, name: __MODULE__) + end + + def init(_opts) do + DynamicSupervisor.init( + strategy: :one_for_one, + max_children: Pleroma.Config.get([:connections_pool, :max_connections]) + ) + end + + def start_worker(opts) do + case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do + {:error, :max_children} -> + case free_pool() do + :ok -> start_worker(opts) + :error -> {:error, :pool_full} + end + + res -> + res + end + end + + @registry Pleroma.Gun.ConnectionPool + @enforcer_key "enforcer" + defp free_pool do + case Registry.lookup(@registry, @enforcer_key) do + [] -> + pid = + spawn(fn -> + {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil) + + max_connections = Pleroma.Config.get([:connections_pool, :max_connections]) + + reclaim_max = + [:connections_pool, :reclaim_multiplier] + |> Pleroma.Config.get() + |> Kernel.*(max_connections) + |> round + |> max(1) + + unused_conns = + Registry.select( + @registry, + [ + {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], + [{{:"$1", :"$3", :"$4"}}]} + ] + ) + + case unused_conns do + [] -> + exit(:no_unused_conns) + + unused_conns -> + unused_conns + |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> + crf1 <= crf2 and last_reference1 <= last_reference2 + end) + |> Enum.take(reclaim_max) + |> Enum.each(fn {pid, _, _} -> + DynamicSupervisor.terminate_child(__MODULE__, pid) + end) + end + end) + + wait_for_enforcer_finish(pid) + + [{pid, _}] -> + wait_for_enforcer_finish(pid) + end + end + + defp wait_for_enforcer_finish(pid) do + ref = Process.monitor(pid) + + receive do + {:DOWN, ^ref, :process, ^pid, :no_unused_conns} -> + :error + + {:DOWN, ^ref, :process, ^pid, :normal} -> + :ok + end + end +end -- cgit v1.2.3 From 0ffde499b8a8f31c82183253bdd692c75733ca2f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 15 Jul 2020 15:24:47 +0300 Subject: Connection Pool: register workers using :via --- lib/pleroma/gun/connection_pool.ex | 8 +++++--- lib/pleroma/gun/connection_pool/worker.ex | 17 ++++++++--------- lib/pleroma/gun/connection_pool/worker_supervisor.ex | 3 +-- 3 files changed, 14 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex index 545bfaf7f..e951872fe 100644 --- a/lib/pleroma/gun/connection_pool.ex +++ b/lib/pleroma/gun/connection_pool.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Gun.ConnectionPool do case Registry.lookup(@registry, key) do # The key has already been registered, but connection is not up yet - [{worker_pid, {nil, _used_by, _crf, _last_reference}}] -> + [{worker_pid, nil}] -> get_gun_pid_from_worker(worker_pid) [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> @@ -26,13 +26,13 @@ defmodule Pleroma.Gun.ConnectionPool do # :gun.set_owner fails in :connected state for whatevever reason, # so we open the connection in the process directly and send it's pid back # We trust gun to handle timeouts by itself - case WorkerSupervisor.start_worker([uri, key, opts, self()]) do + case WorkerSupervisor.start_worker([key, uri, opts, self()]) do {:ok, _worker_pid} -> receive do {:conn_pid, pid} -> {:ok, pid} end - {:error, {:error, {:already_registered, worker_pid}}} -> + {:error, {:already_started, worker_pid}} -> get_gun_pid_from_worker(worker_pid) err -> @@ -56,6 +56,8 @@ defmodule Pleroma.Gun.ConnectionPool do end def release_conn(conn_pid) do + # :ets.fun2ms(fn {_, {worker_pid, {gun_pid, _, _, _}}} when gun_pid == conn_pid -> + # worker_pid end) query_result = Registry.select(@registry, [ {{:_, :"$1", {:"$2", :_, :_, :_}}, [{:==, :"$2", conn_pid}], [:"$1"]} diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 25fafc64c..0a94f16a2 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -4,20 +4,19 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do @registry Pleroma.Gun.ConnectionPool - def start_link(opts) do - GenServer.start_link(__MODULE__, opts) + def start_link([key | _] = opts) do + GenServer.start_link(__MODULE__, opts, name: {:via, Registry, {@registry, key}}) end @impl true - def init([uri, key, opts, client_pid]) do - time = :os.system_time(:second) - # Register before opening connection to prevent race conditions - with {:ok, _owner} <- Registry.register(@registry, key, {nil, [client_pid], 1, time}), - {:ok, conn_pid} <- Gun.Conn.open(uri, opts), + def init([key, uri, opts, client_pid]) do + with {:ok, conn_pid} <- Gun.Conn.open(uri, opts), Process.link(conn_pid) do + time = :os.system_time(:second) + {_, _} = - Registry.update_value(@registry, key, fn {_, used_by, crf, last_reference} -> - {conn_pid, used_by, crf, last_reference} + Registry.update_value(@registry, key, fn _ -> + {conn_pid, [client_pid], 1, time} end) send(client_pid, {:conn_pid, conn_pid}) diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index 5b546bd87..d090c034e 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -1,5 +1,5 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do - @doc "Supervisor for pool workers. Does not do anything except enforce max connection limit" + @moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit" use DynamicSupervisor @@ -35,7 +35,6 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do pid = spawn(fn -> {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil) - max_connections = Pleroma.Config.get([:connections_pool, :max_connections]) reclaim_max = -- cgit v1.2.3 From 7738fbbaf5a6fcd6a10b4ef0a2dcea731a3d4192 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 15 Jul 2020 15:26:25 +0300 Subject: Connection pool: implement logging and telemetry events --- lib/pleroma/application.ex | 1 + .../gun/connection_pool/worker_supervisor.ex | 44 ++++++++++++--- lib/pleroma/telemetry/logger.ex | 62 ++++++++++++++++++++++ 3 files changed, 100 insertions(+), 7 deletions(-) create mode 100644 lib/pleroma/telemetry/logger.ex (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index cfdaf1770..37fcdf293 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -39,6 +39,7 @@ defmodule Pleroma.Application do # every time the application is restarted, so we disable module # conflicts at runtime Code.compiler_options(ignore_module_conflict: true) + Pleroma.Telemetry.Logger.attach() Config.Holder.save_default() Pleroma.HTML.compile_scrubbers() Config.DeprecationWarnings.warn() diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index d090c034e..4b5d10d2a 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -18,8 +18,12 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do {:error, :max_children} -> case free_pool() do - :ok -> start_worker(opts) - :error -> {:error, :pool_full} + :ok -> + start_worker(opts) + + :error -> + :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) + {:error, :pool_full} end res -> @@ -44,6 +48,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do |> round |> max(1) + :telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{ + max_connections: max_connections, + reclaim_max: reclaim_max + }) + + # :ets.fun2ms( + # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] -> + # {worker_pid, crf, last_reference} end) unused_conns = Registry.select( @registry, @@ -55,17 +67,35 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do case unused_conns do [] -> + :telemetry.execute( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: 0}, + %{ + max_connections: max_connections + } + ) + exit(:no_unused_conns) unused_conns -> - unused_conns - |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> - crf1 <= crf2 and last_reference1 <= last_reference2 - end) - |> Enum.take(reclaim_max) + reclaimed = + unused_conns + |> Enum.sort(fn {_pid1, crf1, last_reference1}, + {_pid2, crf2, last_reference2} -> + crf1 <= crf2 and last_reference1 <= last_reference2 + end) + |> Enum.take(reclaim_max) + + reclaimed |> Enum.each(fn {pid, _, _} -> DynamicSupervisor.terminate_child(__MODULE__, pid) end) + + :telemetry.execute( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: Enum.count(reclaimed)}, + %{max_connections: max_connections} + ) end end) diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex new file mode 100644 index 000000000..d76dd37b5 --- /dev/null +++ b/lib/pleroma/telemetry/logger.ex @@ -0,0 +1,62 @@ +defmodule Pleroma.Telemetry.Logger do + @moduledoc "Transforms Pleroma telemetry events to logs" + + require Logger + + @events [ + [:pleroma, :connection_pool, :reclaim, :start], + [:pleroma, :connection_pool, :reclaim, :stop], + [:pleroma, :connection_pool, :provision_failure] + ] + def attach do + :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, []) + end + + # Passing anonymous functions instead of strings to logger is intentional, + # that way strings won't be concatenated if the message is going to be thrown + # out anyway due to higher log level configured + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :start], + _, + %{max_connections: max_connections, reclaim_max: reclaim_max}, + _ + ) do + Logger.debug(fn -> + "Connection pool is exhausted (reached #{max_connections} connections). Starting idle connection cleanup to reclaim as much as #{ + reclaim_max + } connections" + end) + end + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: 0}, + _, + _ + ) do + Logger.error(fn -> + "Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts" + end) + end + + def handle_event( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: reclaimed_count}, + _, + _ + ) do + Logger.debug(fn -> "Connection pool cleaned up #{reclaimed_count} idle connections" end) + end + + def handle_event( + [:pleroma, :connection_pool, :provision_failure], + %{opts: [key | _]}, + _, + _ + ) do + Logger.error(fn -> + "Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion" + end) + end +end -- cgit v1.2.3 From e94ba05e523d735cd7a357a3aa30e433f60ef9a3 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 7 May 2020 16:11:48 +0300 Subject: Connection pool: Fix a possible infinite recursion if the pool is exhausted --- lib/pleroma/gun/connection_pool/worker_supervisor.ex | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index 4b5d10d2a..5cb8d488a 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -14,16 +14,14 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do ) end - def start_worker(opts) do + def start_worker(opts, retry \\ false) do case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do {:error, :max_children} -> - case free_pool() do - :ok -> - start_worker(opts) - - :error -> - :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) - {:error, :pool_full} + if retry or free_pool() == :error do + :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) + {:error, :pool_full} + else + start_worker(opts, true) end res -> -- cgit v1.2.3 From 1b15cb066c612c72d106e7e7026819ea14e0ceab Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 8 May 2020 18:18:59 +0300 Subject: Connection pool: Add client death tracking While running this in production I noticed a number of ghost processes with all their clients dead before they released the connection, so let's track them to log it and remove them from clients --- lib/pleroma/gun/connection_pool/worker.ex | 31 ++++++++++++++++++++++++++++++- lib/pleroma/telemetry/logger.ex | 16 +++++++++++++++- 2 files changed, 45 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 0a94f16a2..8467325f3 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -20,7 +20,10 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do end) send(client_pid, {:conn_pid, conn_pid}) - {:ok, %{key: key, timer: nil}, :hibernate} + + {:ok, + %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}}, + :hibernate} else err -> {:stop, err} end @@ -45,6 +48,9 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do state end + ref = Process.monitor(client_pid) + + state = put_in(state.client_monitors[client_pid], ref) {:noreply, state, :hibernate} end @@ -55,6 +61,9 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do {conn_pid, List.delete(used_by, client_pid), crf, last_reference} end) + {ref, state} = pop_in(state.client_monitors[client_pid]) + Process.demonitor(ref) + timer = if used_by == [] do max_idle = Pleroma.Config.get([:connections_pool, :max_idle_time], 30_000) @@ -85,6 +94,26 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do {:stop, {:error, down_message}, state} end + @impl true + def handle_info({:DOWN, _ref, :process, pid, reason}, state) do + # Sometimes the client is dead before we demonitor it in :remove_client, so the message + # arrives anyway + + case state.client_monitors[pid] do + nil -> + {:noreply, state, :hibernate} + + _ref -> + :telemetry.execute( + [:pleroma, :connection_pool, :client_death], + %{client_pid: pid, reason: reason}, + %{key: state.key} + ) + + handle_cast({:remove_client, pid}, state) + end + end + # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478 defp crf(time_delta, prev_crf) do 1 + :math.pow(0.5, time_delta / 100) * prev_crf diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index d76dd37b5..4cacae02f 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -6,7 +6,8 @@ defmodule Pleroma.Telemetry.Logger do @events [ [:pleroma, :connection_pool, :reclaim, :start], [:pleroma, :connection_pool, :reclaim, :stop], - [:pleroma, :connection_pool, :provision_failure] + [:pleroma, :connection_pool, :provision_failure], + [:pleroma, :connection_pool, :client_death] ] def attach do :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, []) @@ -59,4 +60,17 @@ defmodule Pleroma.Telemetry.Logger do "Connection pool had to refuse opening a connection to #{key} due to connection limit exhaustion" end) end + + def handle_event( + [:pleroma, :connection_pool, :client_death], + %{client_pid: client_pid, reason: reason}, + %{key: key}, + _ + ) do + Logger.warn(fn -> + "Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{ + inspect(reason) + }" + end) + end end -- cgit v1.2.3 From 281ddd5e371c5698489774e703106bd7c3ccb56b Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 8 May 2020 19:57:11 +0300 Subject: Connection pool: fix connections being supervised by gun_sup --- lib/pleroma/gun/api.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/gun/api.ex b/lib/pleroma/gun/api.ex index f51cd7db8..09be74392 100644 --- a/lib/pleroma/gun/api.ex +++ b/lib/pleroma/gun/api.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Gun.API do :tls_opts, :tcp_opts, :socks_opts, - :ws_opts + :ws_opts, + :supervise ] @impl Gun -- cgit v1.2.3 From 94c8f3cfafb92c6d092549b24bb69f3870e1c0d8 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 16 May 2020 11:49:19 +0300 Subject: Use a custom pool-aware FollowRedirects middleware --- lib/pleroma/http/adapter_helper.ex | 4 - lib/pleroma/http/adapter_helper/default.ex | 3 - lib/pleroma/http/adapter_helper/gun.ex | 9 -- lib/pleroma/http/adapter_helper/hackney.ex | 2 - lib/pleroma/http/http.ex | 9 +- lib/pleroma/tesla/middleware/follow_redirects.ex | 106 +++++++++++++++++++++++ 6 files changed, 109 insertions(+), 24 deletions(-) create mode 100644 lib/pleroma/tesla/middleware/follow_redirects.ex (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 0532ea31d..bcb9b2b1e 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -24,7 +24,6 @@ defmodule Pleroma.HTTP.AdapterHelper do | {Connection.proxy_type(), Connection.host(), pos_integer()} @callback options(keyword(), URI.t()) :: keyword() - @callback after_request(keyword()) :: :ok @callback get_conn(URI.t(), keyword()) :: {:ok, term()} | {:error, term()} @spec format_proxy(String.t() | tuple() | nil) :: proxy() | nil @@ -67,9 +66,6 @@ defmodule Pleroma.HTTP.AdapterHelper do Keyword.merge(opts, timeout: timeout) end - @spec after_request(keyword()) :: :ok - def after_request(opts), do: adapter_helper().after_request(opts) - def get_conn(uri, opts), do: adapter_helper().get_conn(uri, opts) defp adapter, do: Application.get_env(:tesla, :adapter) diff --git a/lib/pleroma/http/adapter_helper/default.ex b/lib/pleroma/http/adapter_helper/default.ex index 218cfacc0..e13441316 100644 --- a/lib/pleroma/http/adapter_helper/default.ex +++ b/lib/pleroma/http/adapter_helper/default.ex @@ -9,9 +9,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Default do AdapterHelper.maybe_add_proxy(opts, AdapterHelper.format_proxy(proxy)) end - @spec after_request(keyword()) :: :ok - def after_request(_opts), do: :ok - @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} def get_conn(_uri, opts), do: {:ok, opts} end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 6f7cc9784..5b4629978 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -34,15 +34,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do |> Keyword.merge(incoming_opts) end - @spec after_request(keyword()) :: :ok - def after_request(opts) do - if opts[:conn] && opts[:body_as] != :chunks do - ConnectionPool.release_conn(opts[:conn]) - end - - :ok - end - defp add_scheme_opts(opts, %{scheme: "http"}), do: opts defp add_scheme_opts(opts, %{scheme: "https"}) do diff --git a/lib/pleroma/http/adapter_helper/hackney.ex b/lib/pleroma/http/adapter_helper/hackney.ex index 42d552740..cd569422b 100644 --- a/lib/pleroma/http/adapter_helper/hackney.ex +++ b/lib/pleroma/http/adapter_helper/hackney.ex @@ -24,8 +24,6 @@ defmodule Pleroma.HTTP.AdapterHelper.Hackney do defp add_scheme_opts(opts, _), do: opts - def after_request(_), do: :ok - @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} def get_conn(_uri, opts), do: {:ok, opts} end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index 8ded76601..afcb4d738 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -69,14 +69,11 @@ defmodule Pleroma.HTTP do request = build_request(method, headers, options, url, body, params) adapter = Application.get_env(:tesla, :adapter) - client = Tesla.client([Tesla.Middleware.FollowRedirects], adapter) + client = Tesla.client([Pleroma.HTTP.Middleware.FollowRedirects], adapter) - response = request(client, request) - - AdapterHelper.after_request(adapter_opts) - - response + request(client, request) + # Connection release is handled in a custom FollowRedirects middleware err -> err end diff --git a/lib/pleroma/tesla/middleware/follow_redirects.ex b/lib/pleroma/tesla/middleware/follow_redirects.ex new file mode 100644 index 000000000..f2c502c69 --- /dev/null +++ b/lib/pleroma/tesla/middleware/follow_redirects.ex @@ -0,0 +1,106 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2015-2020 Tymon Tobolski +# Copyright © 2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.HTTP.Middleware.FollowRedirects do + @moduledoc """ + Pool-aware version of https://github.com/teamon/tesla/blob/master/lib/tesla/middleware/follow_redirects.ex + + Follow 3xx redirects + ## Options + - `:max_redirects` - limit number of redirects (default: `5`) + """ + + alias Pleroma.Gun.ConnectionPool + + @behaviour Tesla.Middleware + + @max_redirects 5 + @redirect_statuses [301, 302, 303, 307, 308] + + @impl Tesla.Middleware + def call(env, next, opts \\ []) do + max = Keyword.get(opts, :max_redirects, @max_redirects) + + redirect(env, next, max) + end + + defp redirect(env, next, left) do + opts = env.opts[:adapter] + + case Tesla.run(env, next) do + {:ok, %{status: status} = res} when status in @redirect_statuses and left > 0 -> + release_conn(opts) + + case Tesla.get_header(res, "location") do + nil -> + {:ok, res} + + location -> + location = parse_location(location, res) + + case get_conn(location, opts) do + {:ok, opts} -> + %{env | opts: Keyword.put(env.opts, :adapter, opts)} + |> new_request(res.status, location) + |> redirect(next, left - 1) + + e -> + e + end + end + + {:ok, %{status: status}} when status in @redirect_statuses -> + release_conn(opts) + {:error, {__MODULE__, :too_many_redirects}} + + other -> + unless opts[:body_as] == :chunks do + release_conn(opts) + end + + other + end + end + + defp get_conn(location, opts) do + uri = URI.parse(location) + + case ConnectionPool.get_conn(uri, opts) do + {:ok, conn} -> + {:ok, Keyword.merge(opts, conn: conn)} + + e -> + e + end + end + + defp release_conn(opts) do + ConnectionPool.release_conn(opts[:conn]) + end + + # The 303 (See Other) redirect was added in HTTP/1.1 to indicate that the originally + # requested resource is not available, however a related resource (or another redirect) + # available via GET is available at the specified location. + # https://tools.ietf.org/html/rfc7231#section-6.4.4 + defp new_request(env, 303, location), do: %{env | url: location, method: :get, query: []} + + # The 307 (Temporary Redirect) status code indicates that the target + # resource resides temporarily under a different URI and the user agent + # MUST NOT change the request method (...) + # https://tools.ietf.org/html/rfc7231#section-6.4.7 + defp new_request(env, 307, location), do: %{env | url: location} + + defp new_request(env, _, location), do: %{env | url: location, query: []} + + defp parse_location("https://" <> _rest = location, _env), do: location + defp parse_location("http://" <> _rest = location, _env), do: location + + defp parse_location(location, env) do + env.url + |> URI.parse() + |> URI.merge(location) + |> URI.to_string() + end +end -- cgit v1.2.3 From 4128e3a84a2b6d75a8f92759e65ee673b47cec01 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sun, 17 May 2020 22:16:02 +0300 Subject: HTTP: Implement max request limits --- lib/pleroma/application.ex | 3 ++- lib/pleroma/http/adapter_helper/gun.ex | 21 +++++++++++++++++++++ lib/pleroma/http/http.ex | 17 ++++++++++++++++- 3 files changed, 39 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 37fcdf293..0ffb55358 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -244,7 +244,8 @@ defmodule Pleroma.Application do end defp http_children(Tesla.Adapter.Gun, _) do - Pleroma.Gun.ConnectionPool.children() + Pleroma.Gun.ConnectionPool.children() ++ + [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] end defp http_children(_, _), do: [] diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 5b4629978..883f7f6f7 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -49,4 +49,25 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do err -> err end end + + @prefix Pleroma.Gun.ConnectionPool + def limiter_setup do + wait = Pleroma.Config.get([:connections_pool, :connection_acquisition_wait]) + retries = Pleroma.Config.get([:connections_pool, :connection_acquisition_retries]) + + :pools + |> Pleroma.Config.get([]) + |> Enum.each(fn {name, opts} -> + max_running = Keyword.get(opts, :size, 50) + max_waiting = Keyword.get(opts, :max_waiting, 10) + + :ok = + ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting, + wait: wait, + max_retries: retries + ) + end) + + :ok + end end diff --git a/lib/pleroma/http/http.ex b/lib/pleroma/http/http.ex index afcb4d738..6128bc4cf 100644 --- a/lib/pleroma/http/http.ex +++ b/lib/pleroma/http/http.ex @@ -71,7 +71,13 @@ defmodule Pleroma.HTTP do adapter = Application.get_env(:tesla, :adapter) client = Tesla.client([Pleroma.HTTP.Middleware.FollowRedirects], adapter) - request(client, request) + maybe_limit( + fn -> + request(client, request) + end, + adapter, + adapter_opts + ) # Connection release is handled in a custom FollowRedirects middleware err -> @@ -92,4 +98,13 @@ defmodule Pleroma.HTTP do |> Builder.add_param(:query, :query, params) |> Builder.convert_to_keyword() end + + @prefix Pleroma.Gun.ConnectionPool + defp maybe_limit(fun, Tesla.Adapter.Gun, opts) do + ConcurrentLimiter.limit(:"#{@prefix}.#{opts[:pool] || :default}", fun) + end + + defp maybe_limit(fun, _, _) do + fun.() + end end -- cgit v1.2.3 From 00926a63fb174a8bcb2f496921c5d17e04e44b1d Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 16 Jun 2020 16:20:28 +0300 Subject: Adapter Helper: Use built-in ip address type --- lib/pleroma/http/adapter_helper.ex | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index bcb9b2b1e..8ca433732 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -8,12 +8,8 @@ defmodule Pleroma.HTTP.AdapterHelper do """ @defaults [pool: :federation] - @type ip_address :: ipv4_address() | ipv6_address() - @type ipv4_address :: {0..255, 0..255, 0..255, 0..255} - @type ipv6_address :: - {0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535, 0..65_535} @type proxy_type() :: :socks4 | :socks5 - @type host() :: charlist() | ip_address() + @type host() :: charlist() | :inet.ip_address() alias Pleroma.Config alias Pleroma.HTTP.AdapterHelper @@ -114,7 +110,7 @@ defmodule Pleroma.HTTP.AdapterHelper do end end - @spec parse_host(String.t() | atom() | charlist()) :: charlist() | ip_address() + @spec parse_host(String.t() | atom() | charlist()) :: charlist() | :inet.ip_address() def parse_host(host) when is_list(host), do: host def parse_host(host) when is_atom(host), do: to_charlist(host) -- cgit v1.2.3 From 7882f28569bfaee2996d059990eec279415f0785 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 17 Jun 2020 12:54:13 +0300 Subject: Use erlang monotonic time for CRF calculation --- lib/pleroma/gun/connection_pool/worker.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 8467325f3..418cb18c1 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do def init([key, uri, opts, client_pid]) do with {:ok, conn_pid} <- Gun.Conn.open(uri, opts), Process.link(conn_pid) do - time = :os.system_time(:second) + time = :erlang.monotonic_time() {_, _} = Registry.update_value(@registry, key, fn _ -> @@ -31,7 +31,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do @impl true def handle_cast({:add_client, client_pid, send_pid_back}, %{key: key} = state) do - time = :os.system_time(:second) + time = :erlang.monotonic_time() {{conn_pid, _, _, _}, _} = Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> -- cgit v1.2.3 From 007843b75e0c7087dad1ef932224b21327d81793 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 23 Jun 2020 15:38:45 +0300 Subject: Add documentation for new connection pool settings and remove some `:retry_timeout` and `:retry` got removed because reconnecting on failure is something the new pool intentionally doesn't do. `:max_overflow` had to go in favor of `:max_waiting`, I didn't reuse the key because the settings are very different in their behaviour. `:checkin_timeout` got removed in favor of `:connection_acquisition_wait`, I didn't reuse the key because the settings are somewhat different. I didn't do any migrations/deprecation warnings/changelog entries because these settings were never in stable. --- lib/pleroma/gun/conn.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 77f78c7ff..9dc8880db 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -13,8 +13,6 @@ defmodule Pleroma.Gun.Conn do opts = opts |> Enum.into(%{}) - |> Map.put_new(:retry, pool_opts[:retry] || 1) - |> Map.put_new(:retry_timeout, pool_opts[:retry_timeout] || 1000) |> Map.put_new(:await_up_timeout, pool_opts[:await_up_timeout] || 5_000) |> Map.put_new(:supervise, false) |> maybe_add_tls_opts(uri) -- cgit v1.2.3 From 37f1e781cb19594a6534efbc4d28e793d5960915 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 23 Jun 2020 20:36:21 +0300 Subject: Gun adapter helper: fix wildcard cert issues on OTP 23 See https://bugs.erlang.org/browse/ERL-1260 for more info. The ssl match function is basically copied from mint, except that `:string.lowercase/1` was replaced by `:string.casefold`. It was a TODO in mint's code, so might as well do it since we don't need to support OTP <20. Closes #1834 --- lib/pleroma/http/adapter_helper/gun.ex | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 883f7f6f7..07aaed7f6 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -39,9 +39,36 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do defp add_scheme_opts(opts, %{scheme: "https"}) do opts |> Keyword.put(:certificates_verification, true) - |> Keyword.put(:tls_opts, log_level: :warning) + |> Keyword.put(:tls_opts, + log_level: :warning, + customize_hostname_check: [match_fun: &ssl_match_fun/2] + ) end + # ssl_match_fun is adapted from [Mint](https://github.com/elixir-mint/mint) + # Copyright 2018 Eric Meadows-Jönsson and Andrea Leopardi + + # Wildcard domain handling for DNS ID entries in the subjectAltName X.509 + # extension. Note that this is a subset of the wildcard patterns implemented + # by OTP when matching against the subject CN attribute, but this is the only + # wildcard usage defined by the CA/Browser Forum's Baseline Requirements, and + # therefore the only pattern used in commercially issued certificates. + defp ssl_match_fun({:dns_id, reference}, {:dNSName, [?*, ?. | presented]}) do + case domain_without_host(reference) do + '' -> + :default + + domain -> + :string.casefold(domain) == :string.casefold(presented) + end + end + + defp ssl_match_fun(_reference, _presented), do: :default + + defp domain_without_host([]), do: [] + defp domain_without_host([?. | domain]), do: domain + defp domain_without_host([_ | more]), do: domain_without_host(more) + @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} def get_conn(uri, opts) do case ConnectionPool.get_conn(uri, opts) do -- cgit v1.2.3 From 12fa5541f01ca5cfe082a62dac3317da78043e8f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 30 Jun 2020 15:58:53 +0300 Subject: FollowRedirects: Unconditionally release the connection if there is an error There is no need for streaming the body if there is no body --- lib/pleroma/tesla/middleware/follow_redirects.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/tesla/middleware/follow_redirects.ex b/lib/pleroma/tesla/middleware/follow_redirects.ex index f2c502c69..5a7032215 100644 --- a/lib/pleroma/tesla/middleware/follow_redirects.ex +++ b/lib/pleroma/tesla/middleware/follow_redirects.ex @@ -55,6 +55,10 @@ defmodule Pleroma.HTTP.Middleware.FollowRedirects do release_conn(opts) {:error, {__MODULE__, :too_many_redirects}} + {:error, _} = e -> + release_conn(opts) + e + other -> unless opts[:body_as] == :chunks do release_conn(opts) -- cgit v1.2.3 From 9b73c35ca8b051316815461247b802bc8567854f Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 30 Jun 2020 18:35:15 +0300 Subject: Request limiter setup: consider {:error, :existing} a success When the application restarts (which happens after certain config changes), the limiters are not destroyed, so `ConcurrentLimiter.new` will produce {:error, :existing} --- lib/pleroma/http/adapter_helper/gun.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 07aaed7f6..b8c4cc59c 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -88,11 +88,17 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do max_running = Keyword.get(opts, :size, 50) max_waiting = Keyword.get(opts, :max_waiting, 10) - :ok = + result = ConcurrentLimiter.new(:"#{@prefix}.#{name}", max_running, max_waiting, wait: wait, max_retries: retries ) + + case result do + :ok -> :ok + {:error, :existing} -> :ok + e -> raise e + end end) :ok -- cgit v1.2.3 From a705637dcf7ffe063c9c0f3f190f57e44aaa63f2 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 2 Jul 2020 01:53:27 +0300 Subject: Connection Pool: fix LRFU implementation to not actually be LRU The numbers of the native time unit were so small the CRF was always 1, making it an LRU. This commit switches the time to miliseconds and changes the time delta multiplier to the one yielding mostly highest hit rates according to the paper --- lib/pleroma/gun/connection_pool/worker.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 418cb18c1..ec0502621 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do def init([key, uri, opts, client_pid]) do with {:ok, conn_pid} <- Gun.Conn.open(uri, opts), Process.link(conn_pid) do - time = :erlang.monotonic_time() + time = :erlang.monotonic_time(:millisecond) {_, _} = Registry.update_value(@registry, key, fn _ -> @@ -31,7 +31,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do @impl true def handle_cast({:add_client, client_pid, send_pid_back}, %{key: key} = state) do - time = :erlang.monotonic_time() + time = :erlang.monotonic_time(:millisecond) {{conn_pid, _, _, _}, _} = Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> @@ -116,6 +116,6 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do # LRFU policy: https://citeseerx.ist.psu.edu/viewdoc/summary?doi=10.1.1.55.1478 defp crf(time_delta, prev_crf) do - 1 + :math.pow(0.5, time_delta / 100) * prev_crf + 1 + :math.pow(0.5, 0.0001 * time_delta) * prev_crf end end -- cgit v1.2.3 From 46dd276d686e49676101e2af743aad61393f4b70 Mon Sep 17 00:00:00 2001 From: href Date: Tue, 7 Jul 2020 18:56:17 +0200 Subject: ConnectionPool.Worker: Open gun conn in continue instead of init --- lib/pleroma/gun/connection_pool/worker.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index ec0502621..6ee622fb0 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -9,7 +9,12 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do end @impl true - def init([key, uri, opts, client_pid]) do + def init([_key, _uri, _opts, _client_pid] = opts) do + {:ok, nil, {:continue, {:connect, opts}}} + end + + @impl true + def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do with {:ok, conn_pid} <- Gun.Conn.open(uri, opts), Process.link(conn_pid) do time = :erlang.monotonic_time(:millisecond) @@ -21,7 +26,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do send(client_pid, {:conn_pid, conn_pid}) - {:ok, + {:noreply, %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}}, :hibernate} else -- cgit v1.2.3 From 6a0f2bdf8ceb4127678cc55406a02d41c7fb0ed7 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 8 Jul 2020 13:01:02 +0200 Subject: Ensure connections error get known by the caller --- lib/pleroma/gun/connection_pool.ex | 22 ++++++++++++---------- lib/pleroma/gun/connection_pool/worker.ex | 3 ++- lib/pleroma/http/adapter_helper/gun.ex | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex index e951872fe..d3eead7d8 100644 --- a/lib/pleroma/gun/connection_pool.ex +++ b/lib/pleroma/gun/connection_pool.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Gun.ConnectionPool do case Registry.lookup(@registry, key) do # The key has already been registered, but connection is not up yet [{worker_pid, nil}] -> - get_gun_pid_from_worker(worker_pid) + get_gun_pid_from_worker(worker_pid, true) [{worker_pid, {gun_pid, _used_by, _crf, _last_reference}}] -> GenServer.cast(worker_pid, {:add_client, self(), false}) @@ -27,13 +27,11 @@ defmodule Pleroma.Gun.ConnectionPool do # so we open the connection in the process directly and send it's pid back # We trust gun to handle timeouts by itself case WorkerSupervisor.start_worker([key, uri, opts, self()]) do - {:ok, _worker_pid} -> - receive do - {:conn_pid, pid} -> {:ok, pid} - end + {:ok, worker_pid} -> + get_gun_pid_from_worker(worker_pid, false) {:error, {:already_started, worker_pid}} -> - get_gun_pid_from_worker(worker_pid) + get_gun_pid_from_worker(worker_pid, true) err -> err @@ -41,17 +39,21 @@ defmodule Pleroma.Gun.ConnectionPool do end end - defp get_gun_pid_from_worker(worker_pid) do + defp get_gun_pid_from_worker(worker_pid, register) do # GenServer.call will block the process for timeout length if # the server crashes on startup (which will happen if gun fails to connect) # so instead we use cast + monitor ref = Process.monitor(worker_pid) - GenServer.cast(worker_pid, {:add_client, self(), true}) + if register, do: GenServer.cast(worker_pid, {:add_client, self(), true}) receive do - {:conn_pid, pid} -> {:ok, pid} - {:DOWN, ^ref, :process, ^worker_pid, reason} -> reason + {:conn_pid, pid} -> + Process.demonitor(ref) + {:ok, pid} + + {:DOWN, ^ref, :process, ^worker_pid, reason} -> + {:error, reason} end end diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 6ee622fb0..16a508ad9 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -30,7 +30,8 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}}, :hibernate} else - err -> {:stop, err} + err -> + {:stop, err, nil} end end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index b8c4cc59c..74677ddb5 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -14,7 +14,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do connect_timeout: 5_000, domain_lookup_timeout: 5_000, tls_handshake_timeout: 5_000, - retry: 1, + retry: 0, retry_timeout: 1000, await_up_timeout: 5_000 ] -- cgit v1.2.3 From 23d714ed3038a24eb78314d52807c46d8b8de2f3 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 8 Jul 2020 13:22:42 +0200 Subject: Fix race in enforcer/reclaimer start --- lib/pleroma/gun/connection_pool/reclaimer.ex | 85 ++++++++++++++++++++++ .../gun/connection_pool/worker_supervisor.ex | 81 +-------------------- 2 files changed, 89 insertions(+), 77 deletions(-) create mode 100644 lib/pleroma/gun/connection_pool/reclaimer.ex (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex new file mode 100644 index 000000000..1793ac3ee --- /dev/null +++ b/lib/pleroma/gun/connection_pool/reclaimer.ex @@ -0,0 +1,85 @@ +defmodule Pleroma.Gun.ConnectionPool.Reclaimer do + use GenServer, restart: :temporary + + @registry Pleroma.Gun.ConnectionPool + + def start_monitor() do + pid = + case :gen_server.start(__MODULE__, [], name: {:via, Registry, {@registry, "reclaimer"}}) do + {:ok, pid} -> + pid + + {:error, {:already_registered, pid}} -> + pid + end + + {pid, Process.monitor(pid)} + end + + @impl true + def init(_) do + {:ok, nil, {:continue, :reclaim}} + end + + @impl true + def handle_continue(:reclaim, _) do + max_connections = Pleroma.Config.get([:connections_pool, :max_connections]) + + reclaim_max = + [:connections_pool, :reclaim_multiplier] + |> Pleroma.Config.get() + |> Kernel.*(max_connections) + |> round + |> max(1) + + :telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{ + max_connections: max_connections, + reclaim_max: reclaim_max + }) + + # :ets.fun2ms( + # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] -> + # {worker_pid, crf, last_reference} end) + unused_conns = + Registry.select( + @registry, + [ + {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], [{{:"$1", :"$3", :"$4"}}]} + ] + ) + + case unused_conns do + [] -> + :telemetry.execute( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: 0}, + %{ + max_connections: max_connections + } + ) + + {:stop, :no_unused_conns, nil} + + unused_conns -> + reclaimed = + unused_conns + |> Enum.sort(fn {_pid1, crf1, last_reference1}, {_pid2, crf2, last_reference2} -> + crf1 <= crf2 and last_reference1 <= last_reference2 + end) + |> Enum.take(reclaim_max) + + reclaimed + |> Enum.each(fn {pid, _, _} -> + DynamicSupervisor.terminate_child(Pleroma.Gun.ConnectionPool.WorkerSupervisor, pid) + end) + + :telemetry.execute( + [:pleroma, :connection_pool, :reclaim, :stop], + %{reclaimed_count: Enum.count(reclaimed)}, + %{max_connections: max_connections} + ) + + {:stop, :normal, nil} + end + end +end diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index 5cb8d488a..39615c956 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -29,89 +29,16 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do end end - @registry Pleroma.Gun.ConnectionPool - @enforcer_key "enforcer" defp free_pool do - case Registry.lookup(@registry, @enforcer_key) do - [] -> - pid = - spawn(fn -> - {:ok, _pid} = Registry.register(@registry, @enforcer_key, nil) - max_connections = Pleroma.Config.get([:connections_pool, :max_connections]) - - reclaim_max = - [:connections_pool, :reclaim_multiplier] - |> Pleroma.Config.get() - |> Kernel.*(max_connections) - |> round - |> max(1) - - :telemetry.execute([:pleroma, :connection_pool, :reclaim, :start], %{}, %{ - max_connections: max_connections, - reclaim_max: reclaim_max - }) - - # :ets.fun2ms( - # fn {_, {worker_pid, {_, used_by, crf, last_reference}}} when used_by == [] -> - # {worker_pid, crf, last_reference} end) - unused_conns = - Registry.select( - @registry, - [ - {{:_, :"$1", {:_, :"$2", :"$3", :"$4"}}, [{:==, :"$2", []}], - [{{:"$1", :"$3", :"$4"}}]} - ] - ) - - case unused_conns do - [] -> - :telemetry.execute( - [:pleroma, :connection_pool, :reclaim, :stop], - %{reclaimed_count: 0}, - %{ - max_connections: max_connections - } - ) - - exit(:no_unused_conns) - - unused_conns -> - reclaimed = - unused_conns - |> Enum.sort(fn {_pid1, crf1, last_reference1}, - {_pid2, crf2, last_reference2} -> - crf1 <= crf2 and last_reference1 <= last_reference2 - end) - |> Enum.take(reclaim_max) - - reclaimed - |> Enum.each(fn {pid, _, _} -> - DynamicSupervisor.terminate_child(__MODULE__, pid) - end) - - :telemetry.execute( - [:pleroma, :connection_pool, :reclaim, :stop], - %{reclaimed_count: Enum.count(reclaimed)}, - %{max_connections: max_connections} - ) - end - end) - - wait_for_enforcer_finish(pid) - - [{pid, _}] -> - wait_for_enforcer_finish(pid) - end + wait_for_reclaimer_finish(Pleroma.Gun.ConnectionPool.Reclaimer.start_monitor()) end - defp wait_for_enforcer_finish(pid) do - ref = Process.monitor(pid) - + defp wait_for_reclaimer_finish({pid, mon}) do receive do - {:DOWN, ^ref, :process, ^pid, :no_unused_conns} -> + {:DOWN, ^mon, :process, ^pid, :no_unused_conns} -> :error - {:DOWN, ^ref, :process, ^pid, :normal} -> + {:DOWN, ^mon, :process, ^pid, :normal} -> :ok end end -- cgit v1.2.3 From 53ba6815b170d7d5c11282933b99730209f526ea Mon Sep 17 00:00:00 2001 From: href Date: Wed, 8 Jul 2020 13:58:38 +0200 Subject: parentheses... --- lib/pleroma/gun/connection_pool/reclaimer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex index 1793ac3ee..cea800882 100644 --- a/lib/pleroma/gun/connection_pool/reclaimer.ex +++ b/lib/pleroma/gun/connection_pool/reclaimer.ex @@ -3,7 +3,7 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do @registry Pleroma.Gun.ConnectionPool - def start_monitor() do + def start_monitor do pid = case :gen_server.start(__MODULE__, [], name: {:via, Registry, {@registry, "reclaimer"}}) do {:ok, pid} -> -- cgit v1.2.3 From ce1a42bd04bcf352ea1565b411444a98261b0a96 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 8 Jul 2020 15:12:09 +0200 Subject: Simplify TLS opts - `verify_fun` is not useful now - use `customize_check_hostname` (OTP 20+ so OK) - `partial_chain` is useless as of OTP 21.1 (wasn't there, but hackney/.. uses it) --- lib/pleroma/gun/conn.ex | 5 ++--- lib/pleroma/http/adapter_helper/gun.ex | 28 ---------------------------- 2 files changed, 2 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 9dc8880db..5c12e8153 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -28,9 +28,8 @@ defmodule Pleroma.Gun.Conn do cacertfile: CAStore.file_path(), depth: 20, reuse_sessions: false, - verify_fun: - {&:ssl_verify_hostname.verify_fun/3, - [check_hostname: Pleroma.HTTP.AdapterHelper.format_host(host)]} + log_level: :warning, + customize_hostname_check: [match_fun: :public_key.pkix_verify_hostname_match_fun(:https)] ] tls_opts = diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 74677ddb5..b4ff8306c 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -39,36 +39,8 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do defp add_scheme_opts(opts, %{scheme: "https"}) do opts |> Keyword.put(:certificates_verification, true) - |> Keyword.put(:tls_opts, - log_level: :warning, - customize_hostname_check: [match_fun: &ssl_match_fun/2] - ) end - # ssl_match_fun is adapted from [Mint](https://github.com/elixir-mint/mint) - # Copyright 2018 Eric Meadows-Jönsson and Andrea Leopardi - - # Wildcard domain handling for DNS ID entries in the subjectAltName X.509 - # extension. Note that this is a subset of the wildcard patterns implemented - # by OTP when matching against the subject CN attribute, but this is the only - # wildcard usage defined by the CA/Browser Forum's Baseline Requirements, and - # therefore the only pattern used in commercially issued certificates. - defp ssl_match_fun({:dns_id, reference}, {:dNSName, [?*, ?. | presented]}) do - case domain_without_host(reference) do - '' -> - :default - - domain -> - :string.casefold(domain) == :string.casefold(presented) - end - end - - defp ssl_match_fun(_reference, _presented), do: :default - - defp domain_without_host([]), do: [] - defp domain_without_host([?. | domain]), do: domain - defp domain_without_host([_ | more]), do: domain_without_host(more) - @spec get_conn(URI.t(), keyword()) :: {:ok, keyword()} | {:error, atom()} def get_conn(uri, opts) do case ConnectionPool.get_conn(uri, opts) do -- cgit v1.2.3 From afd378f84c4c1b784eba11b35c21e0b6ae3d7915 Mon Sep 17 00:00:00 2001 From: href Date: Wed, 8 Jul 2020 16:02:57 +0200 Subject: host is now useless --- lib/pleroma/gun/conn.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 5c12e8153..a3f75a4bb 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Gun.Conn do defp maybe_add_tls_opts(opts, %URI{scheme: "http"}), do: opts - defp maybe_add_tls_opts(opts, %URI{scheme: "https", host: host}) do + defp maybe_add_tls_opts(opts, %URI{scheme: "https"}) do tls_opts = [ verify: :verify_peer, cacertfile: CAStore.file_path(), -- cgit v1.2.3 From 6d583bcc3b23c0c16aefa3f34155e7e15b745b01 Mon Sep 17 00:00:00 2001 From: href Date: Mon, 13 Jul 2020 10:44:36 +0200 Subject: Set a default timeout for Gun adapter timeout --- lib/pleroma/http/adapter_helper.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 8ca433732..9ec3836b0 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -44,15 +44,17 @@ defmodule Pleroma.HTTP.AdapterHelper do @spec options(URI.t(), keyword()) :: keyword() def options(%URI{} = uri, opts \\ []) do @defaults - |> pool_timeout() + |> put_timeout() |> Keyword.merge(opts) |> adapter_helper().options(uri) end - defp pool_timeout(opts) do + # For Hackney, this is the time a connection can stay idle in the pool. + # For Gun, this is the timeout to receive a message from Gun. + defp put_timeout(opts) do {config_key, default} = if adapter() == Tesla.Adapter.Gun do - {:pools, Config.get([:pools, :default, :timeout])} + {:pools, Config.get([:pools, :default, :timeout], 5_000)} else {:hackney_pools, 10_000} end -- cgit v1.2.3 From 7115c5f82efe1ca1817da3152ba3cbc66e0da1a4 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 15 Jul 2020 15:58:08 +0300 Subject: ConnectionPool.Worker: do not stop with an error when there is a timeout This produced error log messages about GenServer termination every time the connection was not open due to a timeout. Instead we stop with `{:shutdown, }` since shutting down when the connection can't be established is normal behavior. --- lib/pleroma/gun/connection_pool.ex | 5 ++++- lib/pleroma/gun/connection_pool/worker.ex | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool.ex b/lib/pleroma/gun/connection_pool.ex index d3eead7d8..8b41a668c 100644 --- a/lib/pleroma/gun/connection_pool.ex +++ b/lib/pleroma/gun/connection_pool.ex @@ -53,7 +53,10 @@ defmodule Pleroma.Gun.ConnectionPool do {:ok, pid} {:DOWN, ^ref, :process, ^worker_pid, reason} -> - {:error, reason} + case reason do + {:shutdown, error} -> error + _ -> {:error, reason} + end end end diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index 16a508ad9..f33447cb6 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do :hibernate} else err -> - {:stop, err, nil} + {:stop, {:shutdown, err}, nil} end end -- cgit v1.2.3 From d29b8997f4a3601eac7f2e1e57de27a67df6699c Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Wed, 15 Jul 2020 15:25:33 +0200 Subject: MastoAPI: fix & test giving MRF reject reasons --- lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 12be530c9..9bb2ef117 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -172,6 +172,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do with_direct_conversation_id: true ) else + {:error, {:reject, message}} -> + conn + |> put_status(:unprocessable_entity) + |> json(%{error: message}) + {:error, message} -> conn |> put_status(:unprocessable_entity) -- cgit v1.2.3