From 718e8e1edb537aca984216be39b3be5c8af4e6da Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 16 May 2021 21:39:58 -0500 Subject: Create NsfwApiPolicy --- .../web/activity_pub/mrf/nsfw_api_policy.ex | 185 +++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex new file mode 100644 index 000000000..9ad175b1b --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -0,0 +1,185 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do + @moduledoc """ + Hide, delete, or mark sensitive NSFW content with artificial intelligence. + + Requires a NSFW API server, configured like so: + + config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF, + url: "http://127.0.0.1:5000/", + threshold: 0.8, + mark_sensitive: true, + unlist: false, + reject: false + + The NSFW API server must implement an HTTP endpoint like this: + + curl http://localhost:5000/?url=https://fedi.com/images/001.jpg + + Returning a response like this: + + {"score", 0.314} + + Where a score is 0-1, with `1` being definitely NSFW. + + A good API server is here: https://github.com/EugenCepoi/nsfw_api + You can run it with Docker with a one-liner: + + docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest + + Options: + + - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/" + - `threshold`: Lowest score to take action on. Default: `0.7` + - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true` + - `unlist`: Unlist all detected NSFW content? Default: `false` + - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` + """ + alias Pleroma.Config + alias Pleroma.Constants + alias Pleroma.HTTP + alias Pleroma.User + + require Logger + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF + @policy :mrf_nsfw_api + + defp build_request_url(url) do + Config.get([@policy, :url]) + |> URI.parse() + |> Map.put(:query, "url=#{url}") + |> URI.to_string() + end + + defp parse_url(url) do + request = build_request_url(url) + + with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do + Jason.decode(body) + else + error -> + Logger.warn(""" + [NsfwApiPolicy]: The API server failed. Skipping. + #{inspect(error)} + """) + + error + end + end + + defp check_url_nsfw(url) when is_binary(url) do + threshold = Config.get([@policy, :threshold]) + + case parse_url(url) do + {:ok, %{"score" => score}} when score >= threshold -> + {:nsfw, %{url: url, score: score, threshold: threshold}} + + _ -> + {:sfw, url} + end + end + + defp check_url_nsfw(%{"href" => url}) when is_binary(url) do + check_url_nsfw(url) + end + + defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do + if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do + {:sfw, attachment} + else + {:nsfw, attachment} + end + end + + defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do + if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do + {:sfw, object} + else + {:nsfw, object} + end + end + + defp check_object_nsfw(%{"object" => %{} = child_object} = object) do + case check_object_nsfw(child_object) do + {:sfw, _} -> {:sfw, object} + {:nsfw, _} -> {:nsfw, object} + end + end + + defp check_object_nsfw(object), do: {:sfw, object} + + @impl true + def filter(object) do + with {:sfw, object} <- check_object_nsfw(object) do + {:ok, object} + else + {:nsfw, _data} -> handle_nsfw(object) + _ -> {:reject, "NSFW: Attachment rejected"} + end + end + + defp handle_nsfw(object) do + if Config.get([@policy, :reject]) do + {:reject, object} + else + {:ok, + object + |> maybe_unlist() + |> maybe_mark_sensitive()} + end + end + + defp maybe_unlist(object) do + if Config.get([@policy, :unlist]) do + unlist(object) + else + object + end + end + + defp maybe_mark_sensitive(object) do + if Config.get([@policy, :mark_sensitive]) do + mark_sensitive(object) + else + object + end + end + + defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + with %User{} = user <- User.get_cached_by_ap_id(actor) do + to = + [user.follower_address | to] + |> List.delete(Constants.as_public()) + |> Enum.uniq() + + cc = + [Constants.as_public() | cc] + |> List.delete(user.follower_address) + |> Enum.uniq() + + object + |> Map.put("to", to) + |> Map.put("cc", cc) + end + end + + defp mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do + Map.put(object, "object", mark_sensitive(child_object)) + end + + defp mark_sensitive(object) when is_map(object) do + tags = (object["tag"] || []) ++ ["nsfw"] + + object + |> Map.put("tag", tags) + |> Map.put("sensitive", true) + end + + @impl true + def describe, do: {:ok, %{}} +end -- cgit v1.2.3 From f15d419062b5f9aba2a2e84257dc2379b44f92e8 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 16 Jun 2021 22:30:18 -0500 Subject: NsfwApiPolicy: raise if can't fetch user --- lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 9ad175b1b..63e6af0a0 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -165,6 +165,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do object |> Map.put("to", to) |> Map.put("cc", cc) + else + _ -> raise "[NsfwApiPolicy]: Could not fetch user #{actor}" end end -- cgit v1.2.3 From 2b3dfbb42f7ec0c5604876276a81d55a05955416 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 17 Jun 2021 14:36:51 -0500 Subject: NsfwApiPolicy: add tests --- .../web/activity_pub/mrf/nsfw_api_policy.ex | 45 +++++++++++++++------- 1 file changed, 32 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 63e6af0a0..9dcdf560e 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -49,14 +49,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do @behaviour Pleroma.Web.ActivityPub.MRF @policy :mrf_nsfw_api - defp build_request_url(url) do + def build_request_url(url) do Config.get([@policy, :url]) |> URI.parse() + |> fix_path() |> Map.put(:query, "url=#{url}") |> URI.to_string() end - defp parse_url(url) do + def parse_url(url) do request = build_request_url(url) with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do @@ -72,23 +73,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end - defp check_url_nsfw(url) when is_binary(url) do + def check_url_nsfw(url) when is_binary(url) do threshold = Config.get([@policy, :threshold]) case parse_url(url) do {:ok, %{"score" => score}} when score >= threshold -> {:nsfw, %{url: url, score: score, threshold: threshold}} + {:ok, %{"score" => score}} -> + {:sfw, %{url: url, score: score, threshold: threshold}} + _ -> - {:sfw, url} + {:sfw, %{url: url, score: nil, threshold: threshold}} end end - defp check_url_nsfw(%{"href" => url}) when is_binary(url) do + def check_url_nsfw(%{"href" => url}) when is_binary(url) do check_url_nsfw(url) end - defp check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do + def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do {:sfw, attachment} else @@ -96,7 +100,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end - defp check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do + def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do + case check_url_nsfw(url) do + {:sfw, _} -> {:sfw, attachment} + {:nsfw, _} -> {:nsfw, attachment} + end + end + + def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do {:sfw, object} else @@ -104,14 +115,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end - defp check_object_nsfw(%{"object" => %{} = child_object} = object) do + def check_object_nsfw(%{"object" => %{} = child_object} = object) do case check_object_nsfw(child_object) do {:sfw, _} -> {:sfw, object} {:nsfw, _} -> {:nsfw, object} end end - defp check_object_nsfw(object), do: {:sfw, object} + def check_object_nsfw(object), do: {:sfw, object} @impl true def filter(object) do @@ -150,7 +161,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end - defp unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do with %User{} = user <- User.get_cached_by_ap_id(actor) do to = [user.follower_address | to] @@ -166,15 +177,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do |> Map.put("to", to) |> Map.put("cc", cc) else - _ -> raise "[NsfwApiPolicy]: Could not fetch user #{actor}" + _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}" end end - defp mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do + def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do Map.put(object, "object", mark_sensitive(child_object)) end - defp mark_sensitive(object) when is_map(object) do + def mark_sensitive(object) when is_map(object) do tags = (object["tag"] || []) ++ ["nsfw"] object @@ -182,6 +193,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do |> Map.put("sensitive", true) end + # Hackney needs a trailing slash + defp fix_path(%URI{path: path} = uri) when is_binary(path) do + path = String.trim_trailing(path, "/") <> "/" + Map.put(uri, :path, path) + end + + defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/") + @impl true def describe, do: {:ok, %{}} end -- cgit v1.2.3 From b293c14a1b01398029dfa80aea306946efc2f284 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 17 Jun 2021 14:52:07 -0500 Subject: NsfwApiPolicy: add describe/0 and config_description/0 --- .../web/activity_pub/mrf/nsfw_api_policy.ex | 56 +++++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 9dcdf560e..a1560c584 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF, url: "http://127.0.0.1:5000/", - threshold: 0.8, + threshold: 0.7, mark_sensitive: true, unlist: false, reject: false @@ -202,5 +202,57 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/") @impl true - def describe, do: {:ok, %{}} + def describe do + options = %{ + threshold: Config.get([@policy, :threshold]), + mark_sensitive: Config.get([@policy, :mark_sensitive]), + unlist: Config.get([@policy, :unlist]), + reject: Config.get([@policy, :reject]) + } + + {:ok, %{@policy => options}} + end + + @impl true + def config_description do + %{ + key: @policy, + related_policy: to_string(__MODULE__), + label: "NSFW API Policy", + description: + "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.", + children: [ + %{ + key: :url, + type: :string, + description: "Base URL of the API server.", + suggestions: ["http://127.0.0.1:5000/"] + }, + %{ + key: :threshold, + type: :float, + description: "Lowest score to take action on. Between 0 and 1.", + suggestions: [0.7] + }, + %{ + key: :mark_sensitive, + type: :boolean, + description: "Mark sensitive all detected NSFW content?", + suggestions: [true] + }, + %{ + key: :unlist, + type: :boolean, + description: "Unlist sensitive all detected NSFW content?", + suggestions: [false] + }, + %{ + key: :reject, + type: :boolean, + description: "Reject sensitive all detected NSFW content (takes precedence)?", + suggestions: [false] + } + ] + } + end end -- cgit v1.2.3 From a704d5499c03cb5609ea38a5f2ef06095ced3ef3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 17 Jun 2021 15:32:42 -0500 Subject: NsfwApiPolicy: Fall back more generously when functions don't match --- lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index a1560c584..920821f38 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -92,6 +92,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do check_url_nsfw(url) end + def check_url_nsfw(url) do + threshold = Config.get([@policy, :threshold]) + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do {:sfw, attachment} @@ -107,6 +112,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end + def check_attachment_nsfw(attachment), do: {:sfw, attachment} + def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do {:sfw, object} -- cgit v1.2.3 From 9423052e9217aa1358950d37c5c96b11d554b37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 25 Apr 2022 12:39:36 +0200 Subject: Add "status" notification type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/notification.ex | 38 +++++++++++++++++++--- .../api_spec/operations/notification_operation.ex | 4 ++- .../controllers/notification_controller.ex | 1 + .../web/mastodon_api/views/notification_view.ex | 3 ++ lib/pleroma/web/push/impl.ex | 1 + 5 files changed, 41 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 52fd2656b..d142baa8b 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -73,6 +73,7 @@ defmodule Pleroma.Notification do pleroma:report reblog poll + status } def changeset(%Notification{} = notification, attrs) do @@ -397,11 +398,18 @@ defmodule Pleroma.Notification do {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) potential_receivers = enabled_receivers ++ disabled_receivers + {enabled_subscribers, disabled_subscribers} = get_notified_subscribers_from_activity(activity) + potential_subscribers = (enabled_subscribers ++ disabled_subscribers) -- potential_receivers + notifications = - Enum.map(potential_receivers, fn user -> - do_send = do_send && user in enabled_receivers - create_notification(activity, user, do_send: do_send) - end) + (Enum.map(potential_receivers, fn user -> + do_send = do_send && user in enabled_receivers + create_notification(activity, user, do_send: do_send) + end) ++ + Enum.map(potential_subscribers, fn user -> + do_send = do_send && user in enabled_subscribers + create_notification(activity, user, do_send: do_send, type: "status") + end)) |> Enum.reject(&is_nil/1) {:ok, notifications} @@ -533,6 +541,27 @@ defmodule Pleroma.Notification do def get_notified_from_activity(_, _local_only), do: {[], []} + def get_notified_subscribers_from_activity(activity, local_only \\ true) + + def get_notified_subscribers_from_activity( + %Activity{data: %{"type" => "Create"}} = activity, + local_only + ) do + notification_enabled_ap_ids = + [] + |> Utils.maybe_notify_subscribers(activity) + + potential_receivers = + User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only) + + notification_enabled_users = + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + + {notification_enabled_users, potential_receivers -- notification_enabled_users} + end + + def get_notified_subscribers_from_activity(_, _), do: {[], []} + # For some activities, only notify the author of the object def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}}) when type in ~w{Like Announce EmojiReact} do @@ -557,7 +586,6 @@ defmodule Pleroma.Notification do [] |> Utils.maybe_notify_to_recipients(activity) |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) |> Utils.maybe_notify_followers(activity) |> Enum.uniq() end diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 7f2336ff6..aa965fabb 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -196,7 +196,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "pleroma:report", "move", "follow_request", - "poll" + "poll", + "status" ], description: """ The type of event that resulted in the notification. @@ -210,6 +211,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do - `pleroma:emoji_reaction` - Someone reacted with emoji to your status - `pleroma:chat_mention` - Someone mentioned you in a chat message - `pleroma:report` - Someone was reported + - `status` - Someone you are subscribed to created a status """ } end diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index 932bc6423..9209e8ebd 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -51,6 +51,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do move pleroma:emoji_reaction poll + status } def index(%{assigns: %{user: user}} = conn, params) do params = diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 0dc7f3beb..b10b0893c 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -103,6 +103,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do "mention" -> put_status(response, activity, reading_user, status_render_opts) + "status" -> + put_status(response, activity, reading_user, status_render_opts) + "favourite" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts) diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index daf3eeb9e..77bc2941d 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -183,6 +183,7 @@ defmodule Pleroma.Web.Push.Impl do def format_title(%{type: type}, mastodon_type) do case mastodon_type || type do "mention" -> "New Mention" + "status" -> "New Status" "follow" -> "New Follower" "follow_request" -> "New Follow Request" "reblog" -> "New Repeat" -- cgit v1.2.3 From bd52e2aec7f07da3bc3609f72f7f1bf5969c9baf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Feb 2022 18:03:53 +0100 Subject: Instance rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/rule.ex | 56 ++++++++++ .../web/admin_api/controllers/rule_controller.ex | 64 ++++++++++++ lib/pleroma/web/admin_api/views/rule_view.ex | 21 ++++ .../api_spec/operations/admin/rule_operation.ex | 113 +++++++++++++++++++++ .../web/mastodon_api/views/instance_view.ex | 7 ++ lib/pleroma/web/router.ex | 5 + 6 files changed, 266 insertions(+) create mode 100644 lib/pleroma/rule.ex create mode 100644 lib/pleroma/web/admin_api/controllers/rule_controller.ex create mode 100644 lib/pleroma/web/admin_api/views/rule_view.ex create mode 100644 lib/pleroma/web/api_spec/operations/admin/rule_operation.ex (limited to 'lib') diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex new file mode 100644 index 000000000..d772a32bd --- /dev/null +++ b/lib/pleroma/rule.ex @@ -0,0 +1,56 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Rule do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.Repo + alias Pleroma.Rule + + schema "rules" do + field(:priority, :integer, default: 0) + field(:text, :string) + + timestamps() + end + + def changeset(%Rule{} = rule, params \\ %{}) do + rule + |> cast(params, [:priority, :text]) + |> validate_required([:text]) + end + + def query do + Rule + |> order_by(asc: :priority) + end + + def get(id), do: Repo.get(__MODULE__, id) + + def create(params) do + {:ok, rule} = + %Rule{} + |> changeset(params) + |> Repo.insert() + + rule + end + + def update(params, id) do + {:ok, rule} = + get(id) + |> changeset(params) + |> Repo.update() + + rule + end + + def delete(id) do + get(id) + |> Repo.delete() + end +end diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex new file mode 100644 index 000000000..2db88b6ba --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + alias Pleroma.Rule + alias Pleroma.Web.Plugs.OAuthScopesPlug + + import Pleroma.Web.ControllerHelper, + only: [ + json_response: 3 + ] + + require Logger + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write"]} + when action in [:create, :update, :delete] + ) + + plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index) + + action_fallback(AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation + + def index(conn, _) do + rules = + Rule.query() + |> Repo.all() + + render(conn, "index.json", rules: rules) + end + + def create(%{body_params: params} = conn, _) do + rule = + params + |> Rule.create() + + render(conn, "show.json", rule: rule) + end + + def update(%{body_params: params} = conn, %{id: id}) do + rule = + params + |> Rule.update(id) + + render(conn, "show.json", rule: rule) + end + + def delete(conn, %{id: id}) do + with {:ok, _} <- Rule.delete(id) do + json(conn, %{}) + else + _ -> json_response(conn, :bad_request, "") + end + end +end diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex new file mode 100644 index 000000000..f29145248 --- /dev/null +++ b/lib/pleroma/web/admin_api/views/rule_view.ex @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleView do + use Pleroma.Web, :view + + require Pleroma.Constants + + def render("index.json", %{rules: rules} = _opts) do + render_many(rules, __MODULE__, "show.json") + end + + def render("show.json", %{rule: rule} = _opts) do + %{ + id: rule.id, + priority: rule.priority, + text: rule.text + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex new file mode 100644 index 000000000..28f2be5e7 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -0,0 +1,113 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Retrieve a list of instance rules", + operationId: "AdminAPI.RuleController.index", + security: [%{"oAuth" => ["admin:read"]}], + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :array, + items: rule() + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Create new rule", + operationId: "AdminAPI.RuleController.create", + security: [%{"oAuth" => ["admin:write"]}], + parameters: admin_api_params(), + requestBody: request_body("Parameters", create_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def update_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Modify existing rule", + operationId: "AdminAPI.RuleController.update", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Delete rule", + operationId: "AdminAPI.RuleController.delete", + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + security: [%{"oAuth" => ["admin:write"]}], + responses: %{ + 200 => empty_object_response(), + 404 => Operation.response("Not Found", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + defp create_request do + %Schema{ + type: :object, + required: [:text], + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } + end + + defp update_request do + %Schema{ + type: :object, + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } + end + + defp rule do + %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :integer}, + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + created_at: %Schema{type: :string, format: :"date-time"} + } + } + end +end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index ee52475d5..9652da53a 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -40,6 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), + rules: rules(), pleroma: %{ metadata: %{ account_activation_required: Keyword.get(instance, :account_activation_required), @@ -137,4 +138,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do value_length: Config.get([:instance, :account_field_value_length]) } end + + def rules do + Pleroma.Rule.query() + |> Pleroma.Repo.all() + |> Enum.map(&%{id: &1.id, text: &1.text}) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index ceb6c3cfd..e8b7b17aa 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -229,6 +229,11 @@ defmodule Pleroma.Web.Router do post("/frontends/install", FrontendController, :install) post("/backups", AdminAPIController, :create_backup) + + get("/rules", RuleController, :index) + post("/rules", RuleController, :create) + patch("/rules/:id", RuleController, :update) + delete("/rules/:id", RuleController, :delete) end # AdminAPI: admins and mods (staff) can perform these actions (if enabled by config) -- cgit v1.2.3 From 432599311d3aab8829ed2cc7f795a1662e7a8f82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Feb 2022 19:13:30 +0100 Subject: Add GET /api/v1/instance/rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../api_spec/operations/admin/rule_operation.ex | 2 +- .../web/api_spec/operations/instance_operation.ex | 27 +++++++++++++++++++++- .../controllers/instance_controller.ex | 5 ++++ .../web/mastodon_api/views/instance_view.ex | 4 ++++ lib/pleroma/web/router.ex | 1 + 5 files changed, 37 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex index 28f2be5e7..ed0d9eaf6 100644 --- a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do def index_operation do %Operation{ tags: ["Instance rule managment"], - summary: "Retrieve a list of instance rules", + summary: "Retrieve list of instance rules", operationId: "AdminAPI.RuleController.index", security: [%{"oAuth" => ["admin:read"]}], responses: %{ diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 3c4b504fe..e66e5b7a3 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -34,6 +34,17 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + def rules_operation do + %Operation{ + tags: ["Instance"], + summary: "Retrieve list of instance rules", + operationId: "InstanceController.rules", + responses: %{ + 200 => Operation.response("Array of domains", "application/json", array_of_rules()) + } + } + end + defp instance do %Schema{ type: :object, @@ -160,7 +171,8 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do "urls" => %{ "streaming_api" => "wss://lain.com" }, - "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)" + "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)", + "rules" => array_of_rules() } } end @@ -172,4 +184,17 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do example: ["pleroma.site", "lain.com", "bikeshed.party"] } end + + defp array_of_rules do + %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } + } + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 6410e872c..d6aa89432 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -20,4 +20,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) end + + @doc "GET /api/v1/instance/rules" + def rules(conn, _params) do + render(conn, "rules.json") + end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 9652da53a..feb94b40b 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -58,6 +58,10 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do } end + def render("rules.json", _) do + rules() + end + def features do [ "pleroma_api", diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e8b7b17aa..1246989b0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -606,6 +606,7 @@ defmodule Pleroma.Web.Router do get("/instance", InstanceController, :show) get("/instance/peers", InstanceController, :peers) + get("/instance/rules", InstanceController, :rules) get("/statuses", StatusController, :index) get("/statuses/:id", StatusController, :show) -- cgit v1.2.3 From 384f8bfa786f51f3abec101e2ab78917f324a4fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 7 Feb 2022 23:41:41 +0100 Subject: Instance rules: Use render_many MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/admin_api/controllers/rule_controller.ex | 2 -- lib/pleroma/web/mastodon_api/views/instance_view.ex | 19 +++++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex index 2db88b6ba..43b2f209a 100644 --- a/lib/pleroma/web/admin_api/controllers/rule_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex @@ -14,8 +14,6 @@ defmodule Pleroma.Web.AdminAPI.RuleController do json_response: 3 ] - require Logger - plug(Pleroma.Web.ApiSpec.CastAndValidate) plug( diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index feb94b40b..8379731e4 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), - rules: rules(), + rules: render(__MODULE__, "rules.json"), pleroma: %{ metadata: %{ account_activation_required: Keyword.get(instance, :account_activation_required), @@ -59,7 +59,16 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do end def render("rules.json", _) do - rules() + Pleroma.Rule.query() + |> Pleroma.Repo.all() + |> render_many(__MODULE__, "rule.json", as: :rule) + end + + def render("rule.json", %{rule: rule}) do + %{ + id: rule.id, + text: rule.text + } end def features do @@ -142,10 +151,4 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do value_length: Config.get([:instance, :account_field_value_length]) } end - - def rules do - Pleroma.Rule.query() - |> Pleroma.Repo.all() - |> Enum.map(&%{id: &1.id, text: &1.text}) - end end -- cgit v1.2.3 From 574db5b988e3caca14d1729a729af83d2f23e214 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 20 Feb 2022 21:44:42 +0100 Subject: Allow submitting an array of rule_ids to /api/v1/reports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/constants.ex | 3 ++- lib/pleroma/rule.ex | 5 +++++ lib/pleroma/web/activity_pub/utils.ex | 8 ++++++-- lib/pleroma/web/admin_api/views/report_view.ex | 15 ++++++++++++++- .../web/api_spec/operations/admin/report_operation.ex | 10 ++++++++++ lib/pleroma/web/api_spec/operations/report_operation.ex | 9 ++++++++- lib/pleroma/web/common_api.ex | 17 +++++++++++++++-- 7 files changed, 60 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index a42c71d23..bf43becb3 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Constants do "context_id", "deleted_activity_id", "pleroma_internal", - "generator" + "generator", + "rules" ] ) diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index d772a32bd..486cff8cc 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -29,6 +29,11 @@ defmodule Pleroma.Rule do |> order_by(asc: :priority) end + def get(ids) when is_list(ids) do + from(r in __MODULE__, where: r.id in ^ids) + |> Repo.all() + end + def get(id), do: Repo.get(__MODULE__, id) def create(params) do diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 9cde7805c..72d17e2aa 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -692,14 +692,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do #### Flag-related helpers @spec make_flag_data(map(), map()) :: map() - def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do + def make_flag_data( + %{actor: actor, context: context, content: content} = params, + additional + ) do %{ "type" => "Flag", "actor" => actor.ap_id, "content" => content, "object" => build_flag_object(params), "context" => context, - "state" => "open" + "state" => "open", + "rules" => Map.get(params, :rules, nil) } |> Map.merge(additional) end diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index b761dbb22..ca70f4359 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -6,10 +6,12 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view alias Pleroma.HTML + alias Pleroma.Rule alias Pleroma.User alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report alias Pleroma.Web.CommonAPI.Utils + alias Pleroma.Web.MastodonAPI.InstanceView alias Pleroma.Web.MastodonAPI.StatusView defdelegate merge_account_views(user), to: AdminAPI.AccountView @@ -46,7 +48,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do as: :activity }), state: report.data["state"], - notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}) + notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}), + rules: rules(Map.get(report.data, "rules", nil)) } end @@ -71,4 +74,14 @@ defmodule Pleroma.Web.AdminAPI.ReportView do created_at: Utils.to_masto_date(inserted_at) } end + + defp rules(nil) do + [] + end + + defp rules(rule_ids) do + rule_ids + |> Rule.get() + |> render_many(InstanceView, "rule.json", as: :rule) + end end diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index 312e091a5..bb71abbd1 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -169,6 +169,16 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do inserted_at: %Schema{type: :string, format: :"date-time"} } } + }, + rules: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :integer}, + text: %Schema{type: :string} + } + } } } } diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex index c74ac7d5f..fd68f67a2 100644 --- a/lib/pleroma/web/api_spec/operations/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/report_operation.ex @@ -53,6 +53,12 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do default: false, description: "If the account is remote, should the report be forwarded to the remote admin?" + }, + rule_ids: %Schema{ + type: :array, + nullable: true, + items: %Schema{type: :number}, + description: "Array of rules" } }, required: [:account_id], @@ -60,7 +66,8 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do "account_id" => "123", "status_ids" => ["1337"], "comment" => "bad status!", - "forward" => "false" + "forward" => "false", + "rule_ids" => [3] } } end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 1b95ee89c..9f8d4def4 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Conversation.Participation alias Pleroma.Formatter alias Pleroma.Object + alias Pleroma.Rule alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.UserRelationship @@ -505,14 +506,16 @@ defmodule Pleroma.Web.CommonAPI do def report(user, data) do with {:ok, account} <- get_reported_account(data.account_id), {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]), - {:ok, statuses} <- get_report_statuses(account, data) do + {:ok, statuses} <- get_report_statuses(account, data), + rules <- get_report_rules(Map.get(data, :rule_ids, nil)) do ActivityPub.flag(%{ context: Utils.generate_context_id(), actor: user, account: account, statuses: statuses, content: content_html, - forward: Map.get(data, :forward, false) + forward: Map.get(data, :forward, false), + rules: rules }) end end @@ -524,6 +527,16 @@ defmodule Pleroma.Web.CommonAPI do end end + defp get_report_rules(nil) do + nil + end + + defp get_report_rules(rule_ids) do + rule_ids + |> Rule.get() + |> Enum.map(& &1.id) + end + def update_report_state(activity_ids, state) when is_list(activity_ids) do case Utils.update_report_state(activity_ids, state) do :ok -> {:ok, activity_ids} -- cgit v1.2.3 From 5c383ada8ad7491125a8264a8a03d85fe822f61e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 5 Mar 2022 21:45:34 +0100 Subject: Correctly order rules by id/creation date MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/rule.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index 486cff8cc..b1db1dc0c 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -27,6 +27,7 @@ defmodule Pleroma.Rule do def query do Rule |> order_by(asc: :priority) + |> order_by(asc: :id) end def get(ids) when is_list(ids) do -- cgit v1.2.3 From b354d70e85bbf0f685f3d56f7377fde2efce4187 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 30 May 2022 12:30:03 +0200 Subject: Apply, suggestions, use strings for actual Mastodon API compatibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/rule.ex | 2 ++ lib/pleroma/web/admin_api/views/report_view.ex | 10 ++++++---- lib/pleroma/web/admin_api/views/rule_view.ex | 2 +- lib/pleroma/web/api_spec/operations/admin/report_operation.ex | 2 +- lib/pleroma/web/api_spec/operations/admin/rule_operation.ex | 5 ++--- lib/pleroma/web/api_spec/operations/instance_operation.ex | 2 +- lib/pleroma/web/api_spec/operations/report_operation.ex | 4 ++-- lib/pleroma/web/common_api.ex | 3 +-- lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 +- 9 files changed, 17 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index b1db1dc0c..611e945b3 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -37,6 +37,8 @@ defmodule Pleroma.Rule do def get(id), do: Repo.get(__MODULE__, id) + def exists?(id), do: not is_nil(get(id)) + def create(params) do {:ok, rule} = %Rule{} diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index ca70f4359..b4b0be267 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -10,8 +10,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do alias Pleroma.User alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.AdminAPI.RuleView alias Pleroma.Web.CommonAPI.Utils - alias Pleroma.Web.MastodonAPI.InstanceView alias Pleroma.Web.MastodonAPI.StatusView defdelegate merge_account_views(user), to: AdminAPI.AccountView @@ -80,8 +80,10 @@ defmodule Pleroma.Web.AdminAPI.ReportView do end defp rules(rule_ids) do - rule_ids - |> Rule.get() - |> render_many(InstanceView, "rule.json", as: :rule) + rules = + rule_ids + |> Rule.get() + + render(RuleView, "index.json", rules: rules) end end diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex index f29145248..abfdd593f 100644 --- a/lib/pleroma/web/admin_api/views/rule_view.ex +++ b/lib/pleroma/web/admin_api/views/rule_view.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Web.AdminAPI.RuleView do def render("show.json", %{rule: rule} = _opts) do %{ - id: rule.id, + id: to_string(rule.id), priority: rule.priority, text: rule.text } diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index bb71abbd1..b90bbd592 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -175,7 +175,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do items: %Schema{ type: :object, properties: %{ - id: %Schema{type: :integer}, + id: %Schema{type: :string}, text: %Schema{type: :string} } } diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex index ed0d9eaf6..2360880e4 100644 --- a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -103,10 +103,9 @@ defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do %Schema{ type: :object, properties: %{ - id: %Schema{type: :integer}, + id: %Schema{type: :string}, priority: %Schema{type: :integer}, - text: %Schema{type: :string}, - created_at: %Schema{type: :string, format: :"date-time"} + text: %Schema{type: :string} } } end diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index e66e5b7a3..f3dba108e 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -191,7 +191,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do items: %Schema{ type: :object, properties: %{ - id: %Schema{type: :integer}, + id: %Schema{type: :string}, text: %Schema{type: :string} } } diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex index fd68f67a2..f5f88974c 100644 --- a/lib/pleroma/web/api_spec/operations/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/report_operation.ex @@ -57,7 +57,7 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do rule_ids: %Schema{ type: :array, nullable: true, - items: %Schema{type: :number}, + items: %Schema{type: :string}, description: "Array of rules" } }, @@ -67,7 +67,7 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do "status_ids" => ["1337"], "comment" => "bad status!", "forward" => "false", - "rule_ids" => [3] + "rule_ids" => ["3"] } } end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 9f8d4def4..6fd744ddc 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -533,8 +533,7 @@ defmodule Pleroma.Web.CommonAPI do defp get_report_rules(rule_ids) do rule_ids - |> Rule.get() - |> Enum.map(& &1.id) + |> Enum.filter(&Rule.exists?/1) end def update_report_state(activity_ids, state) when is_list(activity_ids) do diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 8379731e4..c7f5ff554 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def render("rule.json", %{rule: rule}) do %{ - id: rule.id, + id: to_string(rule.id), text: rule.text } end -- cgit v1.2.3 From 0ecd6ba35e868eac296b013f743d82a120dd68db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 30 May 2022 12:50:44 +0200 Subject: AdminAPI: Allow filtering reports by rule_id MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++++++++++ lib/pleroma/web/api_spec/operations/admin/report_operation.ex | 6 ++++++ 2 files changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 064f93b22..e54adf611 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1191,6 +1191,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_filtered(query, _), do: query + defp restrict_rule(query, %{rule_id: rule_id}) do + from( + activity in query, + where: fragment("(?)->'rules' \\? (?)", activity.data, ^rule_id) + ) + end + + defp restrict_rule(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1353,6 +1362,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_instance(opts) |> restrict_announce_object_actor(opts) |> restrict_filtered(opts) + |> restrict_rule(opts) |> Activity.restrict_deactivated_users() |> exclude_poll_votes(opts) |> exclude_chat_messages(opts) diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index b90bbd592..18386296f 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -30,6 +30,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do report_state(), "Filter by report state" ), + Operation.parameter( + :rule_id, + :query, + %Schema{type: :string}, + "Filter by selected rule id" + ), Operation.parameter( :limit, :query, -- cgit v1.2.3 From 5846e7d5f6b91ab63270f2104543d874589d39ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 1 Jun 2022 16:03:22 +0200 Subject: Use Repo.exists? MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/rule.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex index 611e945b3..c8e3470c7 100644 --- a/lib/pleroma/rule.ex +++ b/lib/pleroma/rule.ex @@ -37,7 +37,10 @@ defmodule Pleroma.Rule do def get(id), do: Repo.get(__MODULE__, id) - def exists?(id), do: not is_nil(get(id)) + def exists?(id) do + from(r in __MODULE__, where: r.id == ^id) + |> Repo.exists?() + end def create(params) do {:ok, rule} = -- cgit v1.2.3 From 6b74a5527410c5e98a6c4906a8e6f9c4d9f5e5d1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 1 Jul 2022 15:40:40 -0500 Subject: InstanceStatic should have reasonable caching While here fix the naming convention of the old module attribute restricting caching and add a new one for the default cache value All frontends should be shipped with versioned assets. There be dragons if you don't. --- lib/pleroma/web/endpoint.ex | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index d8d40cceb..17bd956d8 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Web.Plugs.HTTPSecurityPlug) plug(Pleroma.Web.Plugs.UploadedMedia) - @static_cache_control "public, no-cache" + @static_cache_control "public, max-age=1209600" + @static_cache_disabled "public, no-cache" # InstanceStatic needs to be before Plug.Static to be able to override shipped-static files # If you're adding new paths to `only:` you'll need to configure them in InstanceStatic as well @@ -30,22 +31,32 @@ defmodule Pleroma.Web.Endpoint do from: :pleroma, only: ["emoji", "images"], gzip: true, - cache_control_for_etags: "public, max-age=1209600", + cache_control_for_etags: @static_cache_control, headers: %{ - "cache-control" => "public, max-age=1209600" + "cache-control" => @static_cache_control } ) plug(Pleroma.Web.Plugs.InstanceStatic, at: "/", gzip: true, - cache_control_for_etags: @static_cache_control, + cache_control_for_etags: @static_cache_disabled, headers: %{ - "cache-control" => @static_cache_control + "cache-control" => @static_cache_disabled + } + ) + + plug(Pleroma.Web.Plugs.FrontendStatic, + at: "/", + frontend_type: :primary, + only: ["index.html"], + gzip: true, + cache_control_for_etags: @static_cache_disabled, + headers: %{ + "cache-control" => @static_cache_disabled } ) - # Careful! No `only` restriction here, as we don't know what frontends contain. plug(Pleroma.Web.Plugs.FrontendStatic, at: "/", frontend_type: :primary, @@ -62,9 +73,9 @@ defmodule Pleroma.Web.Endpoint do at: "/pleroma/admin", frontend_type: :admin, gzip: true, - cache_control_for_etags: @static_cache_control, + cache_control_for_etags: @static_cache_disabled, headers: %{ - "cache-control" => @static_cache_control + "cache-control" => @static_cache_disabled } ) @@ -79,9 +90,9 @@ defmodule Pleroma.Web.Endpoint do only: Pleroma.Constants.static_only_files(), # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength gzip: true, - cache_control_for_etags: @static_cache_control, + cache_control_for_etags: @static_cache_disabled, headers: %{ - "cache-control" => @static_cache_control + "cache-control" => @static_cache_disabled } ) -- cgit v1.2.3 From fa2a6d5d6b24657ddbda4ef11d2e6dbcb59545d3 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 7 Apr 2022 18:25:02 +0200 Subject: feat: simple, but not stupid, uploader for IPFS fix: format fix with credo --- lib/pleroma/upload.ex | 13 +++++++-- lib/pleroma/uploaders/ipfs.ex | 64 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/uploaders/ipfs.ex (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index db2909276..de39bcd6c 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,8 +235,14 @@ defmodule Pleroma.Upload do "" end - [base_url, path] - |> Path.join() + uploader = Config.get([Pleroma.Upload, :uploader]) + + if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do + String.replace(base_url, "{CID}", path) + else + [base_url, path] + |> Path.join() + end end defp url_from_spec(_upload, _base_url, {:url, url}), do: url @@ -273,6 +279,9 @@ defmodule Pleroma.Upload do Path.join([upload_base_url, bucket_with_namespace]) end + Pleroma.Uploaders.IPFS -> + Config.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + _ -> public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" end diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex new file mode 100644 index 000000000..b46e9322e --- /dev/null +++ b/lib/pleroma/uploaders/ipfs.ex @@ -0,0 +1,64 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFS do + @behaviour Pleroma.Uploaders.Uploader + require Logger + + alias Pleroma.Config + alias Tesla.Multipart + + @impl true + def get_file(file) do + b_url = Pleroma.Upload.base_url() + + if String.contains?(b_url, "{CID}") do + {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}} + else + {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"} + end + end + + @impl true + def put_file(%Pleroma.Upload{} = upload) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(upload.tempfile) + + final_url = Path.join([post_base_url, "/api/v0/add"]) + + case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do + {:ok, ret} -> + case Jason.decode(ret.body) do + {:ok, ret} -> + {:ok, {:file, ret["Hash"]}} + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "JSON decode failed"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "IPFS Gateway Upload failed"} + end + end + + @impl true + def delete_file(file) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + final_url = Path.join([post_base_url, "/api/v0/files/rm"]) + + case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do + {:ok, %{status_code: 204}} -> :ok + error -> {:error, inspect(error)} + end + end +end -- cgit v1.2.3 From 43dfa58ebda407a0813d398bee8d0ae3e5c9fd5b Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Mon, 11 Apr 2022 15:10:01 +0200 Subject: added tests for ipfs uploader. adapted changelog.md accordingly. improved ipfs uploader with external suggestions fix lint description.exs --- lib/pleroma/upload.ex | 6 ++---- lib/pleroma/uploaders/ipfs.ex | 14 +++++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index de39bcd6c..b51d23f9e 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,10 +235,8 @@ defmodule Pleroma.Upload do "" end - uploader = Config.get([Pleroma.Upload, :uploader]) - - if uploader == Pleroma.Uploaders.IPFS && String.contains?(base_url, "{CID}") do - String.replace(base_url, "{CID}", path) + if String.contains?(base_url, "<%= cid %>") do + EEx.eval_string(base_url, cid: path) else [base_url, path] |> Path.join() diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index b46e9322e..722c68fa1 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -13,10 +13,10 @@ defmodule Pleroma.Uploaders.IPFS do def get_file(file) do b_url = Pleroma.Upload.base_url() - if String.contains?(b_url, "{CID}") do - {:ok, {:url, String.replace(b_url, "{CID}", URI.decode(file))}} + if String.contains?(b_url, "<%= cid %>") do + {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}} else - {:error, "IPFS Get URL doesn't contain '{CID}' placeholder"} + {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} end end @@ -36,7 +36,11 @@ defmodule Pleroma.Uploaders.IPFS do {:ok, ret} -> case Jason.decode(ret.body) do {:ok, ret} -> - {:ok, {:file, ret["Hash"]}} + if Map.has_key?(ret, "Hash") do + {:ok, {:file, ret["Hash"]}} + else + {:error, "JSON doesn't contain Hash value"} + end error -> Logger.error("#{__MODULE__}: #{inspect(error)}") @@ -45,7 +49,7 @@ defmodule Pleroma.Uploaders.IPFS do error -> Logger.error("#{__MODULE__}: #{inspect(error)}") - {:error, "IPFS Gateway Upload failed"} + {:error, "IPFS Gateway upload failed"} end end -- cgit v1.2.3 From 44659ecd65fb2251f9130fcecf1732b8931104c1 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Sat, 16 Apr 2022 09:38:49 +0200 Subject: ipfs: revert to String.replace for cid placeholder ipfs: fix lint --- lib/pleroma/upload.ex | 4 ++-- lib/pleroma/uploaders/ipfs.ex | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index b51d23f9e..8a01cf613 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -235,8 +235,8 @@ defmodule Pleroma.Upload do "" end - if String.contains?(base_url, "<%= cid %>") do - EEx.eval_string(base_url, cid: path) + if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do + String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path) else [base_url, path] |> Path.join() diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 722c68fa1..dde520d8e 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -9,12 +9,15 @@ defmodule Pleroma.Uploaders.IPFS do alias Pleroma.Config alias Tesla.Multipart + @placeholder "{CID}" + def placeholder, do: @placeholder + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() - if String.contains?(b_url, "<%= cid %>") do - {:ok, {:url, EEx.eval_string(b_url, cid: URI.decode(file))}} + if String.contains?(b_url, @placeholder) do + {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}} else {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} end -- cgit v1.2.3 From 7c1af86f979ecebcd38995e5278fe2d59a36eda5 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Mon, 9 May 2022 12:15:40 +0200 Subject: ipfs: refactor final_url generation. add tests for final_url fix lint --- lib/pleroma/uploaders/ipfs.ex | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index dde520d8e..7a7481d81 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -12,6 +12,13 @@ defmodule Pleroma.Uploaders.IPFS do @placeholder "{CID}" def placeholder, do: @placeholder + def get_final_url(method) do + config = Config.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + Path.join([post_base_url, method]) + end + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() @@ -25,15 +32,12 @@ defmodule Pleroma.Uploaders.IPFS do @impl true def put_file(%Pleroma.Upload{} = upload) do - config = Config.get([__MODULE__]) - post_base_url = Keyword.get(config, :post_gateway_url) - mp = Multipart.new() |> Multipart.add_content_type_param("charset=utf-8") |> Multipart.add_file(upload.tempfile) - final_url = Path.join([post_base_url, "/api/v0/add"]) + final_url = get_final_url("/api/v0/add") case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do {:ok, ret} -> @@ -42,7 +46,7 @@ defmodule Pleroma.Uploaders.IPFS do if Map.has_key?(ret, "Hash") do {:ok, {:file, ret["Hash"]}} else - {:error, "JSON doesn't contain Hash value"} + {:error, "JSON doesn't contain Hash key"} end error -> @@ -58,10 +62,7 @@ defmodule Pleroma.Uploaders.IPFS do @impl true def delete_file(file) do - config = Config.get([__MODULE__]) - post_base_url = Keyword.get(config, :post_gateway_url) - - final_url = Path.join([post_base_url, "/api/v0/files/rm"]) + final_url = get_final_url("/api/v0/files/rm") case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do {:ok, %{status_code: 204}} -> :ok -- cgit v1.2.3 From 98f268e5ecc5bab98c98270a582f8b3f0e3be4e8 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 9 Jun 2022 19:24:13 +0200 Subject: ipfs: small refactor and more tests --- lib/pleroma/uploaders/ipfs.ex | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 7a7481d81..9f6f26e2e 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -9,16 +9,24 @@ defmodule Pleroma.Uploaders.IPFS do alias Pleroma.Config alias Tesla.Multipart - @placeholder "{CID}" - def placeholder, do: @placeholder - - def get_final_url(method) do + defp get_final_url(method) do config = Config.get([__MODULE__]) post_base_url = Keyword.get(config, :post_gateway_url) Path.join([post_base_url, method]) end + def put_file_endpoint() do + get_final_url("/api/v0/add") + end + + def delete_file_endpoint() do + get_final_url("/api/v0/files/rm") + end + + @placeholder "{CID}" + def placeholder, do: @placeholder + @impl true def get_file(file) do b_url = Pleroma.Upload.base_url() @@ -37,9 +45,7 @@ defmodule Pleroma.Uploaders.IPFS do |> Multipart.add_content_type_param("charset=utf-8") |> Multipart.add_file(upload.tempfile) - final_url = get_final_url("/api/v0/add") - - case Pleroma.HTTP.post(final_url, mp, [], params: ["cid-version": "1"]) do + case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do {:ok, ret} -> case Jason.decode(ret.body) do {:ok, ret} -> @@ -62,9 +68,7 @@ defmodule Pleroma.Uploaders.IPFS do @impl true def delete_file(file) do - final_url = get_final_url("/api/v0/files/rm") - - case Pleroma.HTTP.post(final_url, "", [], params: [arg: file]) do + case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do {:ok, %{status_code: 204}} -> :ok error -> {:error, inspect(error)} end -- cgit v1.2.3 From 254f2ea85400ebd692fc4a45f5ac22fedd49ec09 Mon Sep 17 00:00:00 2001 From: Claudio Maradonna Date: Thu, 9 Jun 2022 23:38:50 +0200 Subject: ipfs: remove unused alias fix analysis job --- lib/pleroma/uploaders/ipfs.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex index 9f6f26e2e..32e06c5cf 100644 --- a/lib/pleroma/uploaders/ipfs.ex +++ b/lib/pleroma/uploaders/ipfs.ex @@ -16,11 +16,11 @@ defmodule Pleroma.Uploaders.IPFS do Path.join([post_base_url, method]) end - def put_file_endpoint() do + def put_file_endpoint do get_final_url("/api/v0/add") end - def delete_file_endpoint() do + def delete_file_endpoint do get_final_url("/api/v0/files/rm") end -- cgit v1.2.3 From e154ebbf7933123e91d5b5c6f5070e78eb3e383b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 15 Aug 2021 21:53:04 +0300 Subject: Initial meilisearch implementation, doesn't delete posts yet --- lib/mix/tasks/pleroma/search/meilisearch.ex | 38 ++++++++++++++ lib/pleroma/activity.ex | 1 + lib/pleroma/activity/search.ex | 4 +- lib/pleroma/application.ex | 6 ++- lib/pleroma/search/meilisearch.ex | 60 ++++++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 6 +++ .../mastodon_api/controllers/search_controller.ex | 5 +- 7 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 lib/mix/tasks/pleroma/search/meilisearch.ex create mode 100644 lib/pleroma/search/meilisearch.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..2af8e5853 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Meilisearch do + import Mix.Pleroma + + import Ecto.Query + + def run(["index"]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + Pleroma.Repo.chunk_stream( + from(Pleroma.Object, + limit: 200, + where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") + ), + 100, + :batches + ) + |> Stream.map(fn objects -> + Enum.map(objects, fn object -> + data = object.data + %{id: object.id, source: data["source"], ap: data["id"]} + end) + end) + |> Stream.each(fn activities -> + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!(activities) + ) + end) + |> Stream.run() + end +end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index ebfd4ed45..9563136f9 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -367,6 +367,7 @@ defmodule Pleroma.Activity do end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + def add_to_index(_activity), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 0b9b24aa4..3dce9d355 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -136,7 +136,7 @@ defmodule Pleroma.Activity.Search do ) end - defp maybe_restrict_local(q, user) do + def maybe_restrict_local(q, user) do limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) case {limit, user} do @@ -149,7 +149,7 @@ defmodule Pleroma.Activity.Search do defp restrict_local(q), do: where(q, local: true) - defp maybe_fetch(activities, user, search_query) do + def maybe_fetch(activities, user, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 1c1db8c10..62d1b8b39 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -321,7 +321,11 @@ defmodule Pleroma.Application do def limiters_setup do config = Config.get(ConcurrentLimiter, []) - [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy] + [ + Pleroma.Web.RichMedia.Helpers, + Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, + Pleroma.Search + ] |> Enum.each(fn module -> mod_config = Keyword.get(config, module, []) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..92e0d3429 --- /dev/null +++ b/lib/pleroma/search/meilisearch.ex @@ -0,0 +1,60 @@ +defmodule Pleroma.Search.Meilisearch do + require Logger + + alias Pleroma.Activity + + import Pleroma.Activity.Search + import Ecto.Query + + def search(user, query, options \\ []) do + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/search", + Jason.encode!(%{q: query, offset: offset, limit: limit}) + ) + + hits = Jason.decode!(result.body)["hits"] |> Enum.map(& &1["ap"]) + + try do + hits + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object() + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> maybe_fetch(user, query) + |> order_by([activity], desc: activity.id) + |> Pleroma.Repo.all() + rescue + _ -> maybe_fetch([], user, query) + end + end + + def add_to_index(activity) do + object = activity.object + + if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" do + data = object.data + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!([%{id: object.id, source: data["source"], ap: data["id"]}]) + ) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + Logger.error("Failed to add activity #{activity.id} to index: #{result.body}") + end + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a5d7036d9..034c3b185 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -140,6 +140,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + {:ok, activity} else %Activity{} = activity -> diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 5e6e04734..99c33eba6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do use Pleroma.Web, :controller - alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ControllerHelper @@ -100,7 +99,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end) + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + statuses = with_fallback(fn -> search_module.search(options[:for_user], query, options) end) StatusView.render("index.json", activities: statuses, -- cgit v1.2.3 From 0318e9a59945d7a5625111157867f0f9ebaffd91 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 10:18:01 +0300 Subject: Add logging to milisiearch index and make it use desc(id) --- lib/mix/tasks/pleroma/search/meilisearch.ex | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2af8e5853..1fece96e5 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -3,8 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - import Mix.Pleroma + require Logger + import Mix.Pleroma import Ecto.Query def run(["index"]) do @@ -12,12 +13,25 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/settings/ranking-rules", + Jason.encode!([ + "desc(id)", + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ]) + ) + Pleroma.Repo.chunk_stream( from(Pleroma.Object, - limit: 200, where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") ), - 100, + 200, :batches ) |> Stream.map(fn objects -> @@ -26,12 +40,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do %{id: object.id, source: data["source"], ap: data["id"]} end) end) - |> Stream.each(fn activities -> + |> Stream.each(fn objects -> {:ok, _} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!(activities) + Jason.encode!(objects) ) + + IO.puts("Indexed #{Enum.count(objects)} entries") end) |> Stream.run() end -- cgit v1.2.3 From 365024abec905e427babb5403f0fccbde65f4bcd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 22:24:31 +0300 Subject: Ensure only indexing public posts and implement clearing and delete --- lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++++- lib/pleroma/activity.ex | 1 + lib/pleroma/search/meilisearch.ex | 17 ++++++++++++++++- lib/pleroma/web/common_api.ex | 7 +++++++ 4 files changed, 38 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 1fece96e5..0b86fdece 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do require Logger + require Pleroma.Constants import Mix.Pleroma import Ecto.Query @@ -29,7 +30,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.Repo.chunk_stream( from(Pleroma.Object, - where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") + # Only index public posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + fragment("LENGTH(data->>'source') > 0") and + fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) ), 200, :batches @@ -51,4 +56,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end) |> Stream.run() end + + def run(["clear"]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + end end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 9563136f9..2c168fd41 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -368,6 +368,7 @@ defmodule Pleroma.Activity do defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 92e0d3429..dbe6b2d67 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Search.Meilisearch do require Logger + require Pleroma.Constants alias Pleroma.Activity @@ -41,7 +42,8 @@ defmodule Pleroma.Search.Meilisearch do def add_to_index(activity) do object = activity.object - if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" do + if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" and + Pleroma.Constants.as_public() in object.data["to"] do data = object.data endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) @@ -57,4 +59,17 @@ defmodule Pleroma.Search.Meilisearch do end end end + + def remove_from_index(object) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = + Pleroma.HTTP.request( + :delete, + "#{endpoint}/indexes/objects/documents/#{object.id}", + "", + [], + [] + ) + end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 89f5dd606..54a8aa213 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -147,6 +147,13 @@ defmodule Pleroma.Web.CommonAPI do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + {:ok, delete} else {:find_activity, _} -> -- cgit v1.2.3 From ea6a6a128712e81c4f298b2bb2cedfadf2295cff Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 22:30:56 +0300 Subject: Make the indexing batch differently and more, show number indexed --- lib/mix/tasks/pleroma/search/meilisearch.ex | 63 +++++++++++++++++------------ 1 file changed, 38 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 0b86fdece..2a6438528 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,33 +28,46 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) - Pleroma.Repo.chunk_stream( - from(Pleroma.Object, - # Only index public posts which are notes and have some text - where: - fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'source') > 0") and - fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) - ), - 200, - :batches - ) - |> Stream.map(fn objects -> - Enum.map(objects, fn object -> - data = object.data - %{id: object.id, source: data["source"], ap: data["id"]} - end) - end) - |> Stream.each(fn objects -> - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!(objects) + chunk_size = 100_000 + + Pleroma.Repo.transaction( + fn -> + Pleroma.Repo.stream( + from(Pleroma.Object, + # Only index public posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + fragment("LENGTH(data->>'source') > 0") and + fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), + order_by: fragment("data->'published' DESC") + ), + timeout: :infinity ) + |> Stream.chunk_every(chunk_size) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) - IO.puts("Indexed #{Enum.count(objects)} entries") - end) - |> Stream.run() + IO.puts("Indexed #{new_acc} entries") + + {[objects], new_acc} + end) + |> Stream.map(fn objects -> + Enum.map(objects, fn object -> + data = object.data + %{id: object.id, source: data["source"], ap: data["id"]} + end) + end) + |> Stream.each(fn objects -> + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!(objects) + ) + end) + |> Stream.run() + end, + timeout: :infinity + ) end def run(["clear"]) do -- cgit v1.2.3 From 38996f551a4ec014e9f4cb4a691d31beecab43ba Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:06:32 +0300 Subject: Make meilisearch sort on publish date converted to unix time --- lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2a6438528..2dd9c0a62 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -18,7 +18,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.HTTP.post( "#{endpoint}/indexes/objects/settings/ranking-rules", Jason.encode!([ - "desc(id)", + "desc(published)", "typo", "words", "proximity", @@ -54,7 +54,15 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do |> Stream.map(fn objects -> Enum.map(objects, fn object -> data = object.data - %{id: object.id, source: data["source"], ap: data["id"]} + + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + source: data["source"], + ap: data["id"], + published: published |> DateTime.to_unix() + } end) end) |> Stream.each(fn objects -> -- cgit v1.2.3 From 9beaebd97e1746df010aecfcc01d9e2e9a4c60ac Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:30:14 +0300 Subject: Tweak search ordering to hopefully return newer results --- lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++--- lib/pleroma/search/meilisearch.ex | 13 +++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2dd9c0a62..dcecbd7cf 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -39,7 +39,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), - order_by: fragment("data->'published' DESC") + order_by: [desc: fragment("data->'published'")] ), timeout: :infinity ) @@ -66,11 +66,15 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end) end) |> Stream.each(fn objects -> - {:ok, _} = + {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", Jason.encode!(objects) ) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + IO.puts("Failed to index: #{result}") + end end) |> Stream.run() end, @@ -83,6 +87,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + {:ok, result} = + Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + IO.puts("Failed to clear: #{result}") + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index dbe6b2d67..9fdb0a07f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Search.Meilisearch do |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) |> maybe_fetch(user, query) - |> order_by([activity], desc: activity.id) + |> order_by([object: obj], desc: obj.data["published"]) |> Pleroma.Repo.all() rescue _ -> maybe_fetch([], user, query) @@ -48,10 +48,19 @@ defmodule Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!([%{id: object.id, source: data["source"], ap: data["id"]}]) + Jason.encode!([ + %{ + id: object.id, + source: data["source"], + ap: data["id"], + published: published |> DateTime.to_unix() + } + ]) ) if not Map.has_key?(Jason.decode!(result.body), "updateId") do -- cgit v1.2.3 From 00c48a33acf0bd59fa7e7b58a67b049e4f4adc31 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:57:53 +0300 Subject: Use content instead of source and scrub it --- lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++-------- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index dcecbd7cf..5270de255 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -37,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do # Only index public posts which are notes and have some text where: fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'source') > 0") and + fragment("LENGTH(data->>'content') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), order_by: [desc: fragment("data->'published'")] ), @@ -56,10 +56,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do data = object.data {:ok, published, _} = DateTime.from_iso8601(data["published"]) + {:ok, content} = FastSanitize.strip_tags(data["content"]) %{ id: object.id, - source: data["source"], + content: content, ap: data["id"], published: published |> DateTime.to_unix() } @@ -87,11 +88,6 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = - Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) - - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - IO.puts("Failed to clear: #{result}") - end + {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], []) end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 9fdb0a07f..87fdeaf5e 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Search.Meilisearch do Jason.encode!([ %{ id: object.id, - source: data["source"], + content: data["content"] |> Pleroma.HTML.filter_tags(), ap: data["id"], published: published |> DateTime.to_unix() } -- cgit v1.2.3 From e35d87ea54f70a39206f6103ef0e7334e2a428cc Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 01:37:43 +0300 Subject: Make the chunk size smaller --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 5270de255..44af25f3e 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) - chunk_size = 100_000 + chunk_size = 10_000 Pleroma.Repo.transaction( fn -> -- cgit v1.2.3 From 2b2e409ad72862967cabf06344874ae9bff9860f Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 16:37:52 +0300 Subject: Also index incoming federated posts --- lib/pleroma/search/search.ex | 18 ++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 7 ++----- lib/pleroma/web/activity_pub/side_effects.ex | 7 +++++++ lib/pleroma/web/common_api.ex | 8 ++------ 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/search/search.ex (limited to 'lib') diff --git a/lib/pleroma/search/search.ex b/lib/pleroma/search/search.ex new file mode 100644 index 000000000..e363abf19 --- /dev/null +++ b/lib/pleroma/search/search.ex @@ -0,0 +1,18 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 034c3b185..7178cf9eb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -140,11 +140,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + # Add local posts to search index + Pleroma.Search.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 5eefd2824..15e006b18 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -197,6 +197,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Increase replies count # - Set up ActivityExpiration # - Set up notifications + # - Index incoming posts for search (if needed) @impl true def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), @@ -226,6 +227,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + meta = meta |> add_notifications(notifications) @@ -286,6 +289,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Reduce the user note count # - Reduce the reply count # - Stream out the activity + # - Removes posts from search index (if needed) @impl true def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do deleted_object = @@ -325,6 +329,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) + + Pleroma.Search.remove_from_index(object) + {:ok, object, meta} else {:error, result} diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 54a8aa213..ba6c07975 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -147,12 +147,8 @@ defmodule Pleroma.Web.CommonAPI do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + # Remove from search index for local posts + Pleroma.Search.remove_from_index(object) {:ok, delete} else -- cgit v1.2.3 From 9f16ca80e0fe60b8b0e3e8ddb9b06ca0bec31002 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 18:47:41 +0300 Subject: Mark only content as searchable for meilisearch --- lib/mix/tasks/pleroma/search/meilisearch.ex | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 44af25f3e..ebd3cc81f 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,6 +28,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/settings/searchable-attributes", + Jason.encode!([ + "content" + ]) + ) + chunk_size = 10_000 Pleroma.Repo.transaction( @@ -55,8 +63,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Enum.map(objects, fn object -> data = object.data + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + {:ok, published, _} = DateTime.from_iso8601(data["published"]) - {:ok, content} = FastSanitize.strip_tags(data["content"]) + {:ok, content} = FastSanitize.strip_tags(content_str) %{ id: object.id, -- cgit v1.2.3 From 3dedadf192a3acd0c1dfc2b11eba5a247ae7f61c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 19:38:03 +0300 Subject: Adjust content indexing to skip more unneeded stuff --- lib/mix/tasks/pleroma/search/meilisearch.ex | 47 +++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index ebd3cc81f..3704e0bdc 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -52,13 +52,6 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do timeout: :infinity ) |> Stream.chunk_every(chunk_size) - |> Stream.transform(0, fn objects, acc -> - new_acc = acc + Enum.count(objects) - - IO.puts("Indexed #{new_acc} entries") - - {[objects], new_acc} - end) |> Stream.map(fn objects -> Enum.map(objects, fn object -> data = object.data @@ -70,15 +63,34 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end {:ok, published, _} = DateTime.from_iso8601(data["published"]) - {:ok, content} = FastSanitize.strip_tags(content_str) - - %{ - id: object.id, - content: content, - ap: data["id"], - published: published |> DateTime.to_unix() - } + + content = + with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + trimmed <- String.trim(scrubbed) do + trimmed + end + + # Only index if there is anything in the string. If there is a single symbol, + # it's probably a dot from mastodon posts with just the picture + if String.length(content) > 1 do + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + else + nil + end end) + |> Enum.filter(fn o -> not is_nil(o) end) + end) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) + + IO.puts("Indexed #{new_acc} entries") + + {[objects], new_acc} end) |> Stream.each(fn objects -> {:ok, result} = @@ -102,6 +114,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], []) + {:ok, _} = + Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], + timeout: :infinity + ) end end -- cgit v1.2.3 From 35e9192cedcbc56fb07c9933e2988bf900256b53 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 22:53:18 +0300 Subject: Rework task indexing to share code with the main module The code in the main module now scrubs new posts too --- lib/mix/tasks/pleroma/search/meilisearch.ex | 35 ++-------------------- lib/pleroma/search/meilisearch.ex | 46 ++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 47 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 3704e0bdc..b5a394e34 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -51,40 +51,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ), timeout: :infinity ) + |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) + |> Stream.filter(fn o -> not is_nil(o) end) |> Stream.chunk_every(chunk_size) - |> Stream.map(fn objects -> - Enum.map(objects, fn object -> - data = object.data - - content_str = - case data["content"] do - [nil | rest] -> to_string(rest) - str -> str - end - - {:ok, published, _} = DateTime.from_iso8601(data["published"]) - - content = - with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), - trimmed <- String.trim(scrubbed) do - trimmed - end - - # Only index if there is anything in the string. If there is a single symbol, - # it's probably a dot from mastodon posts with just the picture - if String.length(content) > 1 do - %{ - id: object.id, - content: content, - ap: data["id"], - published: published |> DateTime.to_unix() - } - else - nil - end - end) - |> Enum.filter(fn o -> not is_nil(o) end) - end) |> Stream.transform(0, fn objects, acc -> new_acc = acc + Enum.count(objects) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 87fdeaf5e..10468e36c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -39,28 +39,46 @@ defmodule Pleroma.Search.Meilisearch do end end - def add_to_index(activity) do - object = activity.object - - if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" and + def object_to_search_data(object) do + if not is_nil(object) and object.data["type"] == "Note" and Pleroma.Constants.as_public() in object.data["to"] do data = object.data - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + + content = + with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + trimmed <- String.trim(scrubbed) do + trimmed + end + + if String.length(content) > 1 do + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + end + end + end - {:ok, published, _} = DateTime.from_iso8601(data["published"]) + def add_to_index(activity) do + maybe_search_data = object_to_search_data(activity) + + if activity.data["type"] == "Create" and maybe_search_data do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!([ - %{ - id: object.id, - content: data["content"] |> Pleroma.HTML.filter_tags(), - ap: data["id"], - published: published |> DateTime.to_unix() - } - ]) + Jason.encode!([maybe_search_data]) ) if not Map.has_key?(Jason.decode!(result.body), "updateId") do -- cgit v1.2.3 From 410c8cb765bbec1014cb2bbdbcc44d3a25f834e1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 23:47:43 +0300 Subject: Make indexing logs rewrite themselves --- lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index b5a394e34..2485a441d 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -57,7 +57,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do |> Stream.transform(0, fn objects, acc -> new_acc = acc + Enum.count(objects) - IO.puts("Indexed #{new_acc} entries") + # Reset to the beginning of the line and rewrite it + IO.write("\r") + IO.write("Indexed #{new_acc} entries") {[objects], new_acc} end) @@ -76,6 +78,8 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end, timeout: :infinity ) + + IO.write("\n") end def run(["clear"]) do -- cgit v1.2.3 From 2c7d973af7797ae860829c1764ade521a17e7263 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 19:35:21 +0300 Subject: Implement meilisearch auth --- lib/mix/tasks/pleroma/search/meilisearch.ex | 75 +++++++++++++++++------------ lib/pleroma/search/meilisearch.ex | 69 ++++++++++++++++---------- 2 files changed, 88 insertions(+), 56 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2485a441d..230be5aa1 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,32 +9,30 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Mix.Pleroma import Ecto.Query + import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1] + def run(["index"]) do start_pleroma() - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/settings/ranking-rules", - Jason.encode!([ - "desc(published)", - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ]) - ) + meili_post!( + "/indexes/objects/settings/ranking-rules", + [ + "desc(published)", + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ] + ) - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/settings/searchable-attributes", - Jason.encode!([ - "content" - ]) - ) + meili_post!( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) chunk_size = 10_000 @@ -64,14 +62,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> - {:ok, result} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!(objects) + result = + meili_post!( + "/indexes/objects/documents", + objects ) - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - IO.puts("Failed to index: #{result}") + if not Map.has_key?(result, "updateId") do + IO.puts("Failed to index: #{inspect(result)}") end end) |> Stream.run() @@ -85,11 +83,26 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["clear"]) do start_pleroma() + meili_delete!("/indexes/objects/documents") + end + + def run(["show-private-key", master_key]) do + start_pleroma() + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = - Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], - timeout: :infinity + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, "/keys"), + [{"X-Meili-API-Key", master_key}] ) + + decoded = Jason.decode!(result.body) + + if decoded["private"] do + IO.puts(decoded["private"]) + else + IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 10468e36c..8745d539d 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,20 +7,50 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Activity.Search import Ecto.Query - def search(user, query, options \\ []) do - limit = Enum.min([Keyword.get(options, :limit), 40]) - offset = Keyword.get(options, :offset, 0) - author = Keyword.get(options, :author) + defp meili_headers() do + private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) + if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + end + + def meili_post!(path, params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) {:ok, result} = Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/search", - Jason.encode!(%{q: query, offset: offset, limit: limit}) + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers() + ) + + Jason.decode!(result.body) + end + + def meili_delete!(path) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + timeout: :infinity + ) + end + + def search(user, query, options \\ []) do + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + result = + meili_post!( + "/indexes/objects/search", + %{q: query, offset: offset, limit: limit} ) - hits = Jason.decode!(result.body)["hits"] |> Enum.map(& &1["ap"]) + hits = result["hits"] |> Enum.map(& &1["ap"]) try do hits @@ -73,30 +103,19 @@ defmodule Pleroma.Search.Meilisearch do maybe_search_data = object_to_search_data(activity) if activity.data["type"] == "Create" and maybe_search_data do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, result} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!([maybe_search_data]) + result = + meili_post!( + "/indexes/objects/documents", + [maybe_search_data] ) - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - Logger.error("Failed to add activity #{activity.id} to index: #{result.body}") + if not Map.has_key?(result, "updateId") do + Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") end end end def remove_from_index(object) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, _} = - Pleroma.HTTP.request( - :delete, - "#{endpoint}/indexes/objects/documents/#{object.id}", - "", - [], - [] - ) + meili_delete!("/indexes/objects/documents/#{object.id}") end end -- cgit v1.2.3 From a67f9da5cc46b4e184aa1afe3dd1bd1df31de15b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:02:34 +0300 Subject: Add a message with a count of posts to index --- lib/mix/tasks/pleroma/search/meilisearch.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 230be5aa1..557b06182 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -38,7 +38,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.Repo.transaction( fn -> - Pleroma.Repo.stream( + query = from(Pleroma.Object, # Only index public posts which are notes and have some text where: @@ -46,7 +46,13 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fragment("LENGTH(data->>'content') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), order_by: [desc: fragment("data->'published'")] - ), + ) + + count = query |> Pleroma.Repo.aggregate(:count, :data) + IO.puts("Entries to index: #{count}") + + Pleroma.Repo.stream( + query, timeout: :infinity ) |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) -- cgit v1.2.3 From 09a1ae1b6eca4efbb935aa1c0da950009d110fb2 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:21:46 +0300 Subject: Add the meilisearch.stats command --- lib/mix/tasks/pleroma/search/meilisearch.ex | 10 +++++++++- lib/pleroma/search/meilisearch.ex | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 557b06182..f2d9fe312 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Mix.Pleroma import Ecto.Query - import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1] + import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] def run(["index"]) do start_pleroma() @@ -111,4 +111,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") end end + + def run(["stats"]) do + start_pleroma() + + result = meili_get!("/indexes/objects/stats") + IO.puts("Number of entries: #{result["numberOfDocuments"]}") + IO.puts("Indexing? #{result["isIndexing"]}") + end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 8745d539d..1ad17bf9f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -13,6 +13,18 @@ defmodule Pleroma.Search.Meilisearch do if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end + def meili_get!(path) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, path), + meili_headers() + ) + + Jason.decode!(result.body) + end + def meili_post!(path, params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) -- cgit v1.2.3 From d9ef7e075880ba39dd4ca8e21566c680070faa42 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 21:15:15 +0300 Subject: Fix activity being passed to objec_to_search_data --- lib/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 1ad17bf9f..212bdd473 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -112,7 +112,7 @@ defmodule Pleroma.Search.Meilisearch do end def add_to_index(activity) do - maybe_search_data = object_to_search_data(activity) + maybe_search_data = object_to_search_data(activity.object) if activity.data["type"] == "Create" and maybe_search_data do result = -- cgit v1.2.3 From 40280cc273ad7f2b355846e2f41b9873a8d5ff2c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 28 Aug 2021 15:59:13 +0300 Subject: Reorder ranking rules for (maybe) better results --- lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index f2d9fe312..cdf9ab0bd 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -18,12 +18,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do "/indexes/objects/settings/ranking-rules", [ "desc(published)", - "typo", "words", + "exactness", "proximity", - "attribute", "wordsPosition", - "exactness" + "typo", + "attribute" ] ) -- cgit v1.2.3 From 6beef2d1179ab9a377e87872b7fbe2997bbbbebd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 8 Oct 2021 12:24:37 +0300 Subject: Move add_to_index / remove_from_index to Pleroma.Actitivy.Search --- lib/pleroma/activity.ex | 2 -- lib/pleroma/activity/search.ex | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 2c168fd41..ebfd4ed45 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -367,8 +367,6 @@ defmodule Pleroma.Activity do end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search - def add_to_index(_activity), do: nil - def remove_from_index(_object), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 3dce9d355..47ab5208c 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -45,6 +45,9 @@ defmodule Pleroma.Activity.Search do end end + def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end -- cgit v1.2.3 From 95cb2bb694e3f8857895b21331b02b9277d65d9b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 12 Oct 2021 19:17:37 +0300 Subject: Don't try removing from index again in common_api It's already removed in the side effects of the pipeline --- lib/pleroma/web/common_api.ex | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index ba6c07975..89f5dd606 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -147,9 +147,6 @@ defmodule Pleroma.Web.CommonAPI do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do - # Remove from search index for local posts - Pleroma.Search.remove_from_index(object) - {:ok, delete} else {:find_activity, _} -> -- cgit v1.2.3 From cf558208c202d5188954e26077d35bcc1ae02fce Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 12 Oct 2021 19:34:57 +0300 Subject: Use proper deleted object for removing from index --- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 15e006b18..4762b5ac6 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(object) + Pleroma.Search.remove_from_index(deleted_object) {:ok, object, meta} else -- cgit v1.2.3 From e4b7a3f51f270f468c15cc4ce850c847633c030b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 00:38:00 +0300 Subject: Modify some meilisearch variables --- lib/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 212bdd473..b8248e40c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -47,7 +47,7 @@ defmodule Pleroma.Search.Meilisearch do Path.join(endpoint, path), "", meili_headers(), - timeout: :infinity + [] ) end -- cgit v1.2.3 From 0b4fd0d342e3ced073e82355b380cbfee5478c60 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 13:58:24 +0300 Subject: Set content-type to application/json --- lib/pleroma/search/meilisearch.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index b8248e40c..d94ab8b64 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -10,7 +10,8 @@ defmodule Pleroma.Search.Meilisearch do defp meili_headers() do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) - if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + [{"Content-Type", "application/json"}] ++ + if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end def meili_get!(path) do -- cgit v1.2.3 From 4445421297f4a4375ce9df4857a66ad08e984507 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 21:04:59 +0300 Subject: Only add local posts to index in activity_pub Remote ones are already added in another place --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7178cf9eb..cdc70aacf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) # Add local posts to search index - Pleroma.Search.add_to_index(activity) + if local, do: Pleroma.Search.add_to_index(activity) {:ok, activity} else -- cgit v1.2.3 From e928e307f34542b0a0af8b615c986aeac478b637 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 10 Nov 2021 21:25:12 +0300 Subject: Add a reindex option Signed-off-by: Ekaterina Vaartis --- lib/mix/tasks/pleroma/search/meilisearch.ex | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index cdf9ab0bd..2a3c3a8b9 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -11,9 +11,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] - def run(["index"]) do + def run(["index" | args]) do start_pleroma() + is_reindex = "--reindex" in args + meili_post!( "/indexes/objects/settings/ranking-rules", [ @@ -68,6 +70,19 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> + objects = + objects + |> Enum.filter(fn o -> + if is_reindex do + result = meili_get!("/indexes/objects/documents/#{o.id}") + + # Filter out the already indexed documents. This is true when the document does not exist + result["errorCode"] == "document_not_found" + else + true + end + end) + result = meili_post!( "/indexes/objects/documents", -- cgit v1.2.3 From 9c1a9307079c8d007ae7cbf3e089d2bc5ea6b733 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 13 Nov 2021 15:07:51 +0300 Subject: Support reindexing meilisearch >=0.24.0 It has has a different error code key --- lib/mix/tasks/pleroma/search/meilisearch.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2a3c3a8b9..3b134ad3f 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -76,8 +76,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do if is_reindex do result = meili_get!("/indexes/objects/documents/#{o.id}") + # With >= 0.24.0 the name for "errorCode" is just "code" + error_code_key = + if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"), + do: "code", + else: "errorCode" + # Filter out the already indexed documents. This is true when the document does not exist - result["errorCode"] == "document_not_found" + result[error_code_key] == "document_not_found" else true end -- cgit v1.2.3 From 7009ef5672ad20f92374d218cd614a38cd70515e Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 14 Nov 2021 20:24:05 +0300 Subject: Move the search.ex file so credo doesn't complain --- lib/pleroma/search.ex | 18 ++++++++++++++++++ lib/pleroma/search/search.ex | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 lib/pleroma/search.ex delete mode 100644 lib/pleroma/search/search.ex (limited to 'lib') diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex new file mode 100644 index 000000000..e363abf19 --- /dev/null +++ b/lib/pleroma/search.ex @@ -0,0 +1,18 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end +end diff --git a/lib/pleroma/search/search.ex b/lib/pleroma/search/search.ex deleted file mode 100644 index e363abf19..000000000 --- a/lib/pleroma/search/search.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule Pleroma.Search do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) - end - - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) - end -end -- cgit v1.2.3 From 39e596a5b51c0c86b6d6bd5f23177a1e6a64cf0b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 14 Nov 2021 21:42:18 +0300 Subject: Style fixes --- lib/mix/tasks/pleroma/search/meilisearch.ex | 3 ++- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 3b134ad3f..62ace7e39 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -82,7 +82,8 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do do: "code", else: "errorCode" - # Filter out the already indexed documents. This is true when the document does not exist + # Filter out the already indexed documents. + # This is true when the document does not exist result[error_code_key] == "document_not_found" else true diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index d94ab8b64..41f99ad9f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Activity.Search import Ecto.Query - defp meili_headers() do + defp meili_headers do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) [{"Content-Type", "application/json"}] ++ -- cgit v1.2.3 From 0fae71f88d142f64ec18a49ff4292db816dacdc8 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 16 Nov 2021 21:54:26 +0300 Subject: Rename search.ex to database_search.ex and add search/2 --- lib/pleroma/search.ex | 18 ---------------- lib/pleroma/search/database_search.ex | 24 ++++++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/side_effects.ex | 4 ++-- .../mastodon_api/controllers/search_controller.ex | 4 +--- 5 files changed, 28 insertions(+), 24 deletions(-) delete mode 100644 lib/pleroma/search.ex create mode 100644 lib/pleroma/search/database_search.ex (limited to 'lib') diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex deleted file mode 100644 index e363abf19..000000000 --- a/lib/pleroma/search.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule Pleroma.Search do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) - end - - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) - end -end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex new file mode 100644 index 000000000..be0e19be0 --- /dev/null +++ b/lib/pleroma/search/database_search.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Search.DatabaseSearch do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cdc70aacf..7e3444676 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) # Add local posts to search index - if local, do: Pleroma.Search.add_to_index(activity) + if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 4762b5ac6..fa57eab69 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -227,7 +227,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object)) meta = meta @@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(deleted_object) + Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object) {:ok, object, meta} else diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 99c33eba6..10f1aa532 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -99,9 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - - statuses = with_fallback(fn -> search_module.search(options[:for_user], query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end) StatusView.render("index.json", activities: statuses, -- cgit v1.2.3 From a6946048fbe049aa223d094d36eb767739ab5ff2 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 17 Nov 2021 22:29:49 +0300 Subject: Rename Activity.Search to Search.DatabaseSearch --- lib/pleroma/activity.ex | 2 +- lib/pleroma/activity/search.ex | 165 --------------------- lib/pleroma/search.ex | 24 +++ lib/pleroma/search/database_search.ex | 157 ++++++++++++++++++-- lib/pleroma/search/meilisearch.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/side_effects.ex | 4 +- .../mastodon_api/controllers/search_controller.ex | 2 +- 8 files changed, 173 insertions(+), 185 deletions(-) delete mode 100644 lib/pleroma/activity/search.ex create mode 100644 lib/pleroma/search.ex (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index ebfd4ed45..389c80691 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -366,7 +366,7 @@ defmodule Pleroma.Activity do from(activity in query, where: activity.actor not in subquery(deactivated_users_query)) end - defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex deleted file mode 100644 index 47ab5208c..000000000 --- a/lib/pleroma/activity/search.ex +++ /dev/null @@ -1,165 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Activity.Search do - alias Pleroma.Activity - alias Pleroma.Object.Fetcher - alias Pleroma.Pagination - alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility - - require Pleroma.Constants - - import Ecto.Query - - def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin - limit = Enum.min([Keyword.get(options, :limit), 40]) - offset = Keyword.get(options, :offset, 0) - author = Keyword.get(options, :author) - - search_function = - if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do - :websearch - else - :plain - end - - try do - Activity - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> restrict_public(user) - |> query_with(index_type, search_query, search_function) - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> Pagination.fetch_paginated( - %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, - :offset - ) - |> maybe_fetch(user, search_query) - rescue - _ -> maybe_fetch([], user, search_query) - end - end - - def add_to_index(_activity), do: nil - def remove_from_index(_object), do: nil - - def maybe_restrict_author(query, %User{} = author) do - Activity.Queries.by_author(query, author) - end - - def maybe_restrict_author(query, _), do: query - - def maybe_restrict_blocked(query, %User{} = user) do - Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) - end - - def maybe_restrict_blocked(query, _), do: query - - defp restrict_public(q, user) when not is_nil(user) do - intended_recipients = [ - Pleroma.Constants.as_public(), - Pleroma.Web.ActivityPub.Utils.as_local_public() - ] - - from([a, o] in q, - where: fragment("?->>'type' = 'Create'", a.data), - where: fragment("? && ?", ^intended_recipients, a.recipients) - ) - end - - defp restrict_public(q, _user) do - from([a, o] in q, - where: fragment("?->>'type' = 'Create'", a.data), - where: ^Pleroma.Constants.as_public() in a.recipients - ) - end - - defp query_with(q, :gin, search_query, :plain) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :gin, search_query, :websearch) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :rum, search_query, :plain) do - from([a, o] in q, - where: - fragment( - "? @@ plainto_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - defp query_with(q, :rum, search_query, :websearch) do - from([a, o] in q, - where: - fragment( - "? @@ websearch_to_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - def maybe_restrict_local(q, user) do - limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) - - case {limit, user} do - {:all, _} -> restrict_local(q) - {:unauthenticated, %User{}} -> q - {:unauthenticated, _} -> restrict_local(q) - {false, _} -> q - end - end - - defp restrict_local(q), do: where(q, local: true) - - def maybe_fetch(activities, user, search_query) do - with true <- Regex.match?(~r/https?:/, search_query), - {:ok, object} <- Fetcher.fetch_object_from_id(search_query), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), - true <- Visibility.visible_for_user?(activity, user) do - [activity | activities] - else - _ -> activities - end - end -end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex new file mode 100644 index 000000000..ae0b28c54 --- /dev/null +++ b/lib/pleroma/search.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end +end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index be0e19be0..5a8b8ca67 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -1,24 +1,153 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Search.DatabaseSearch do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + alias Pleroma.Activity + alias Pleroma.Object.Fetcher + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility + + require Pleroma.Constants + + import Ecto.Query + + def search(user, search_query, options \\ []) do + index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + search_function = + if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do + :websearch + else + :plain + end + + try do + Activity + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> restrict_public() + |> query_with(index_type, search_query, search_function) + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> Pagination.fetch_paginated( + %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, + :offset + ) + |> maybe_fetch(user, search_query) + rescue + _ -> maybe_fetch([], user, search_query) + end + end + + def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil + + def maybe_restrict_author(query, %User{} = author) do + Activity.Queries.by_author(query, author) + end + + def maybe_restrict_author(query, _), do: query + + def maybe_restrict_blocked(query, %User{} = user) do + Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) + end + + def maybe_restrict_blocked(query, _), do: query + + def restrict_public(q) do + from([a, o] in q, + where: fragment("?->>'type' = 'Create'", a.data), + where: ^Pleroma.Constants.as_public() in a.recipients + ) + end + + defp query_with(q, :gin, search_query, :plain) do + %{rows: [[tsc]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select current_setting('default_text_search_config')::regconfig::oid;" + ) + + from([a, o] in q, + where: + fragment( + "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", + ^tsc, + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :gin, search_query, :websearch) do + %{rows: [[tsc]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select current_setting('default_text_search_config')::regconfig::oid;" + ) + + from([a, o] in q, + where: + fragment( + "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)", + ^tsc, + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :rum, search_query, :plain) do + from([a, o] in q, + where: + fragment( + "? @@ plainto_tsquery(?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + defp query_with(q, :rum, search_query, :websearch) do + from([a, o] in q, + where: + fragment( + "? @@ websearch_to_tsquery(?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) end - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + def maybe_restrict_local(q, user) do + limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + case {limit, user} do + {:all, _} -> restrict_local(q) + {:unauthenticated, %User{}} -> q + {:unauthenticated, _} -> restrict_local(q) + {false, _} -> q + end end - def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + defp restrict_local(q), do: where(q, local: true) - search_module.search(options[:for_user], query, options) + def maybe_fetch(activities, user, search_query) do + with true <- Regex.match?(~r/https?:/, search_query), + {:ok, object} <- Fetcher.fetch_object_from_id(search_query), + %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + true <- Visibility.visible_for_user?(activity, user) do + [activity | activities] + else + _ -> activities + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 41f99ad9f..fa9e27b03 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Search.Meilisearch do alias Pleroma.Activity - import Pleroma.Activity.Search + import Pleroma.Search.DatabaseSearch import Ecto.Query defp meili_headers do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7e3444676..cdc70aacf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) # Add local posts to search index - if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity) + if local, do: Pleroma.Search.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index fa57eab69..4762b5ac6 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -227,7 +227,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object)) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) meta = meta @@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object) + Pleroma.Search.remove_from_index(deleted_object) {:ok, object, meta} else diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 10f1aa532..e4acba226 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -99,7 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end) StatusView.render("index.json", activities: statuses, -- cgit v1.2.3 From a12f63bc81481e3f852934e8cc1269e16a57cf0a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 22 Nov 2021 21:39:54 +0300 Subject: Implement suggestions from the Meilisearch MR - Index unlisted posts - Move version check outside of the streaming and only do it once - Use a PUT request instead of checking manually if there is need to insert - Add error handling, sort of --- lib/mix/tasks/pleroma/search/meilisearch.ex | 86 ++++++++++++---------------- lib/pleroma/search/meilisearch.ex | 87 +++++++++++++++++++---------- 2 files changed, 95 insertions(+), 78 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 62ace7e39..6730a99a9 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -3,38 +3,40 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - require Logger require Pleroma.Constants import Mix.Pleroma import Ecto.Query - import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] + import Pleroma.Search.Meilisearch, + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] - def run(["index" | args]) do + def run(["index"]) do start_pleroma() - is_reindex = "--reindex" in args - - meili_post!( - "/indexes/objects/settings/ranking-rules", - [ - "desc(published)", - "words", - "exactness", - "proximity", - "wordsPosition", - "typo", - "attribute" - ] - ) + {:ok, _} = + meili_post( + "/indexes/objects/settings/ranking-rules", + [ + "desc(published)", + "words", + "exactness", + "proximity", + "wordsPosition", + "typo", + "attribute" + ] + ) - meili_post!( - "/indexes/objects/settings/searchable-attributes", - [ - "content" - ] - ) + {:ok, _} = + meili_post( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) + + IO.puts("Created indices. Starting to insert posts.") chunk_size = 10_000 @@ -42,11 +44,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fn -> query = from(Pleroma.Object, - # Only index public posts which are notes and have some text + # Only index public and unlisted posts which are notes and have some text where: fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'content') > 0") and - fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), + (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or + fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())), order_by: [desc: fragment("data->'published'")] ) @@ -70,34 +72,18 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> - objects = - objects - |> Enum.filter(fn o -> - if is_reindex do - result = meili_get!("/indexes/objects/documents/#{o.id}") - - # With >= 0.24.0 the name for "errorCode" is just "code" - error_code_key = - if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"), - do: "code", - else: "errorCode" - - # Filter out the already indexed documents. - # This is true when the document does not exist - result[error_code_key] == "document_not_found" - else - true - end - end) - result = - meili_post!( + meili_put( "/indexes/objects/documents", objects ) - if not Map.has_key?(result, "updateId") do - IO.puts("Failed to index: #{inspect(result)}") + with {:ok, res} <- result do + if not Map.has_key?(res, "updateId") do + IO.puts("\nFailed to index: #{inspect(result)}") + end + else + e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}") end end) |> Stream.run() @@ -137,7 +123,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["stats"]) do start_pleroma() - result = meili_get!("/indexes/objects/stats") + {:ok, result} = meili_get("/indexes/objects/stats") IO.puts("Number of entries: #{result["numberOfDocuments"]}") IO.puts("Indexing? #{result["isIndexing"]}") end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index fa9e27b03..21b44de86 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -14,29 +14,50 @@ defmodule Pleroma.Search.Meilisearch do if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end - def meili_get!(path) do + def meili_get(path) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = + result = Pleroma.HTTP.get( Path.join(endpoint, path), meili_headers() ) - Jason.decode!(result.body) + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end end - def meili_post!(path, params) do + def meili_post(path, params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = + result = Pleroma.HTTP.post( Path.join(endpoint, path), Jason.encode!(params), meili_headers() ) - Jason.decode!(result.body) + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_put(path, params) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.request( + :put, + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers(), + [] + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end end def meili_delete!(path) do @@ -57,34 +78,40 @@ defmodule Pleroma.Search.Meilisearch do offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) - result = - meili_post!( + res = + meili_post( "/indexes/objects/search", %{q: query, offset: offset, limit: limit} ) - hits = result["hits"] |> Enum.map(& &1["ap"]) - - try do - hits - |> Activity.create_by_object_ap_id() - |> Activity.with_preloaded_object() - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> maybe_fetch(user, query) - |> order_by([object: obj], desc: obj.data["published"]) - |> Pleroma.Repo.all() - rescue - _ -> maybe_fetch([], user, query) + with {:ok, result} <- res do + hits = result["hits"] |> Enum.map(& &1["ap"]) + + try do + hits + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object() + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> maybe_fetch(user, query) + |> order_by([object: obj], desc: obj.data["published"]) + |> Pleroma.Repo.all() + rescue + _ -> maybe_fetch([], user, query) + end end end def object_to_search_data(object) do + # Only index public or unlisted Notes if not is_nil(object) and object.data["type"] == "Note" and - Pleroma.Constants.as_public() in object.data["to"] do + not is_nil(object.data["content"]) and + (Pleroma.Constants.as_public() in object.data["to"] or + Pleroma.Constants.as_public() in object.data["cc"]) and + String.length(object.data["content"]) > 1 do data = object.data content_str = @@ -117,13 +144,17 @@ defmodule Pleroma.Search.Meilisearch do if activity.data["type"] == "Create" and maybe_search_data do result = - meili_post!( + meili_put( "/indexes/objects/documents", [maybe_search_data] ) - if not Map.has_key?(result, "updateId") do - Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") + with {:ok, res} <- result, + true <- Map.has_key?(res, "updateId") do + # Do nothing + else + _ -> + Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") end end end -- cgit v1.2.3 From 3179ed0921197a8a8f32a519c7d41dc09011024d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 18:48:52 +0300 Subject: Make chunk size configurable --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 6730a99a9..021552f7b 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -38,7 +38,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do IO.puts("Created indices. Starting to insert posts.") - chunk_size = 10_000 + chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size]) Pleroma.Repo.transaction( fn -> -- cgit v1.2.3 From 571533ae2618478f26db312e52265e143356debd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 19:05:59 +0300 Subject: Don't support meilisearch < 0.24.0, since it breaks things --- lib/mix/tasks/pleroma/search/meilisearch.ex | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 021552f7b..5098668ad 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -14,17 +14,29 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["index"]) do start_pleroma() + meili_version = + ( + {:ok, result} = meili_get("/version") + + result["pkgVersion"] + ) + + # The ranking rule syntax was changed but nothing about that is mentioned in the changelog + if not Version.match?(meili_version, ">= 0.24.0") do + raise "Meilisearch <0.24.0 not supported" + end + {:ok, _} = meili_post( "/indexes/objects/settings/ranking-rules", [ - "desc(published)", + "published:desc", "words", "exactness", "proximity", - "wordsPosition", "typo", - "attribute" + "attribute", + "sort" ] ) -- cgit v1.2.3 From 6f2f457751ea09507045e6dd5d5869a14befd3d1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 22:38:50 +0300 Subject: Add a search backend behaviour --- lib/pleroma/search/database_search.ex | 5 +++++ lib/pleroma/search/meilisearch.ex | 4 ++++ lib/pleroma/search/search_backend.ex | 17 +++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 lib/pleroma/search/search_backend.ex (limited to 'lib') diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 5a8b8ca67..3735a5fab 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -13,6 +13,8 @@ defmodule Pleroma.Search.DatabaseSearch do import Ecto.Query + @behaviour Pleroma.Search.SearchBackend + def search(user, search_query, options \\ []) do index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) @@ -45,7 +47,10 @@ defmodule Pleroma.Search.DatabaseSearch do end end + @impl true def add_to_index(_activity), do: nil + + @impl true def remove_from_index(_object), do: nil def maybe_restrict_author(query, %User{} = author) do diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 21b44de86..33bbf8392 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Search.DatabaseSearch import Ecto.Query + @behaviour Pleroma.Search.SearchBackend + defp meili_headers do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) @@ -139,6 +141,7 @@ defmodule Pleroma.Search.Meilisearch do end end + @impl true def add_to_index(activity) do maybe_search_data = object_to_search_data(activity.object) @@ -159,6 +162,7 @@ defmodule Pleroma.Search.Meilisearch do end end + @impl true def remove_from_index(object) do meili_delete!("/indexes/objects/documents/#{object.id}") end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex new file mode 100644 index 000000000..ed6bfd329 --- /dev/null +++ b/lib/pleroma/search/search_backend.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.Search.SearchBackend do + @doc """ + Add the object associated with the activity to the search index. + + The whole activity is passed, to allow filtering on things such as scope. + """ + @callback add_to_index(activity :: Pleroma.Activity.t()) :: nil + + @doc """ + Remove the object from the index. + + Just the object, as opposed to the whole activity, is passed, since the object + is what contains the actual content and there is no need for fitlering when removing + from index. + """ + @callback remove_from_index(object :: Pleroma.Object.t()) :: nil +end -- cgit v1.2.3 From 2bc21c6f1884bae3226f760ed1da39dd9c5f2958 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 15:23:11 +0300 Subject: Use oban for search indexing --- lib/pleroma/search.ex | 15 ++++----------- lib/pleroma/workers/search_indexing_worker.ex | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/workers/search_indexing_worker.ex (limited to 'lib') diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index ae0b28c54..af858fc46 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -1,19 +1,12 @@ defmodule Pleroma.Search do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + alias Pleroma.Workers.SearchIndexingWorker - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + def add_to_index(activity) do + SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id}) end def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id}) end def search(query, options) do diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex new file mode 100644 index 000000000..43b7bad1e --- /dev/null +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -0,0 +1,21 @@ +defmodule Pleroma.Workers.SearchIndexingWorker do + use Pleroma.Workers.WorkerHelper, queue: "search_indexing" + + @impl Oban.Worker + + def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do + activity = Pleroma.Activity.get_by_id_with_object(activity_id) + + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + search_module.add_to_index(activity) + end + + def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do + object = Pleroma.Object.get_by_id(object_id) + + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + search_module.remove_from_index(object) + end +end -- cgit v1.2.3 From d89dc5518b5c0eb232e7ac85ddd538f89c32606d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 16:31:32 +0300 Subject: Fix meilisearch tests and jobs for oban --- lib/pleroma/workers/search_indexing_worker.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 43b7bad1e..70a8d42d0 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -9,6 +9,8 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.add_to_index(activity) + + :ok end def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do @@ -17,5 +19,7 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.remove_from_index(object) + + :ok end end -- cgit v1.2.3 From 3387935e8354e32171fe6e28a8f96f49154acbb3 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 16:52:06 +0300 Subject: Don't try removing deleted users and such from index as posts --- lib/pleroma/search.ex | 8 ++++---- lib/pleroma/web/activity_pub/side_effects.ex | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index af858fc46..3b266e59b 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -1,12 +1,12 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker - def add_to_index(activity) do - SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id}) + def add_to_index(%Pleroma.Activity{id: activity_id}) do + SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id}) end - def remove_from_index(object) do - SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id}) + def remove_from_index(%Pleroma.Object{id: object_id}) do + SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id}) end def search(query, options) do diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 4762b5ac6..644e62630 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -330,7 +330,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(deleted_object) + # Only remove from index when deleting actual objects, not users or anything else + with %Pleroma.Object{} <- deleted_object do + Pleroma.Search.remove_from_index(deleted_object) + end {:ok, object, meta} else -- cgit v1.2.3 From 1e23f527e3e22108b402552a0766e488048ed3f4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 22 Mar 2022 20:29:17 +0300 Subject: Change the meilisearch key auth to conform to 0.25.0 --- lib/mix/tasks/pleroma/search/meilisearch.ex | 14 ++++++++------ lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 5098668ad..db56876fa 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ) # The ranking rule syntax was changed but nothing about that is mentioned in the changelog - if not Version.match?(meili_version, ">= 0.24.0") do + if not Version.match?(meili_version, ">= 0.25.0") do raise "Meilisearch <0.24.0 not supported" end @@ -112,7 +112,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do meili_delete!("/indexes/objects/documents") end - def run(["show-private-key", master_key]) do + def run(["show-keys", master_key]) do start_pleroma() endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) @@ -120,15 +120,17 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {:ok, result} = Pleroma.HTTP.get( Path.join(endpoint, "/keys"), - [{"X-Meili-API-Key", master_key}] + [{"Authorization", "Bearer #{master_key}"}] ) decoded = Jason.decode!(result.body) - if decoded["private"] do - IO.puts(decoded["private"]) + if decoded["results"] do + Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} -> + IO.puts("#{desc}: #{key}") + end) else - IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") + IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}") end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 33bbf8392..0f9182ffc 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Search.Meilisearch do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) [{"Content-Type", "application/json"}] ++ - if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}] end def meili_get(path) do -- cgit v1.2.3 From 84608be87e2c5961a4deb9030307c978bf1168e5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 22 Mar 2022 20:45:49 +0300 Subject: Change updateId to uid because apparently that's the new name --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index db56876fa..d4a83c3cd 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -91,7 +91,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ) with {:ok, res} <- result do - if not Map.has_key?(res, "updateId") do + if not Map.has_key?(res, "uid") do IO.puts("\nFailed to index: #{inspect(result)}") end else diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 0f9182ffc..3db65f261 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -153,7 +153,7 @@ defmodule Pleroma.Search.Meilisearch do ) with {:ok, res} <- result, - true <- Map.has_key?(res, "updateId") do + true <- Map.has_key?(res, "uid") do # Do nothing else _ -> -- cgit v1.2.3 From e20f74c71b078d706bc93632773f9b590d2fb018 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 26 Aug 2022 23:39:58 +0300 Subject: Remove duplicate function call --- lib/pleroma/search/meilisearch.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 3db65f261..53f8a2544 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -93,7 +93,6 @@ defmodule Pleroma.Search.Meilisearch do hits |> Activity.create_by_object_ap_id() |> Activity.with_preloaded_object() - |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() |> maybe_restrict_local(user) |> maybe_restrict_author(author) -- cgit v1.2.3 From 119b2b847b76c7300bd71699d9f2e5676bdb0bb4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 00:09:37 +0300 Subject: Instead of checking string length, explicitly check for "" and "." --- lib/pleroma/search/meilisearch.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 53f8a2544..2e13b8407 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -112,7 +112,7 @@ defmodule Pleroma.Search.Meilisearch do not is_nil(object.data["content"]) and (Pleroma.Constants.as_public() in object.data["to"] or Pleroma.Constants.as_public() in object.data["cc"]) and - String.length(object.data["content"]) > 1 do + object.data["content"] not in ["", "."] do data = object.data content_str = @@ -127,7 +127,8 @@ defmodule Pleroma.Search.Meilisearch do trimmed end - if String.length(content) > 1 do + # Make sure we have a non-empty string + if content != "" do {:ok, published, _} = DateTime.from_iso8601(data["published"]) %{ -- cgit v1.2.3 From 102ebb42bdba1673da39a8fa8ed1662bc8565aa4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 00:19:08 +0300 Subject: Make search a callback --- lib/pleroma/search/database_search.ex | 1 + lib/pleroma/search/meilisearch.ex | 1 + lib/pleroma/search/search_backend.ex | 11 +++++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 3735a5fab..9a340abf1 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Search.DatabaseSearch do @behaviour Pleroma.Search.SearchBackend + @impl true def search(user, search_query, options \\ []) do index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2e13b8407..4e88169d2 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -75,6 +75,7 @@ defmodule Pleroma.Search.Meilisearch do ) end + @impl true def search(user, query, options \\ []) do limit = Enum.min([Keyword.get(options, :limit), 40]) offset = Keyword.get(options, :offset, 0) diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index ed6bfd329..a42e2f5f6 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -1,10 +1,17 @@ defmodule Pleroma.Search.SearchBackend do + @doc """ + Search statuses with a query, restricting to only those the user should have access to. + """ + @callback search(user :: Pleroma.User.t(), query :: String.t(), options :: [any()]) :: [ + Pleroma.Activity.t() + ] + @doc """ Add the object associated with the activity to the search index. The whole activity is passed, to allow filtering on things such as scope. """ - @callback add_to_index(activity :: Pleroma.Activity.t()) :: nil + @callback add_to_index(activity :: Pleroma.Activity.t()) :: :ok | {:error, any()} @doc """ Remove the object from the index. @@ -13,5 +20,5 @@ defmodule Pleroma.Search.SearchBackend do is what contains the actual content and there is no need for fitlering when removing from index. """ - @callback remove_from_index(object :: Pleroma.Object.t()) :: nil + @callback remove_from_index(object :: Pleroma.Object.t()) :: {:ok, any()} | {:error, any()} end -- cgit v1.2.3 From 5ac67632384bfb284ac51f2a450d41cf3913378a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 00:31:36 +0300 Subject: Make add_to_index and remove_from_index report errors --- lib/mix/tasks/pleroma/search/meilisearch.ex | 4 ++-- lib/pleroma/search/meilisearch.ex | 27 ++++++++++++++++----------- lib/pleroma/workers/search_indexing_worker.ex | 4 ---- 3 files changed, 18 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index d4a83c3cd..72a558228 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Ecto.Query import Pleroma.Search.Meilisearch, - only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1] def run(["index"]) do start_pleroma() @@ -109,7 +109,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["clear"]) do start_pleroma() - meili_delete!("/indexes/objects/documents") + meili_delete("/indexes/objects/documents") end def run(["show-keys", master_key]) do diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 4e88169d2..24789b00c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -62,17 +62,16 @@ defmodule Pleroma.Search.Meilisearch do end end - def meili_delete!(path) do + def meili_delete(path) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = - Pleroma.HTTP.request( - :delete, - Path.join(endpoint, path), - "", - meili_headers(), - [] - ) + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + [] + ) end @impl true @@ -155,16 +154,22 @@ defmodule Pleroma.Search.Meilisearch do with {:ok, res} <- result, true <- Map.has_key?(res, "uid") do - # Do nothing + # Added successfully + :ok else _ -> + # There was an error, report it Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") + {:error, result} end + else + # The post isn't something we can search, that's ok + :ok end end @impl true def remove_from_index(object) do - meili_delete!("/indexes/objects/documents/#{object.id}") + meili_delete("/indexes/objects/documents/#{object.id}") end end diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 70a8d42d0..43b7bad1e 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.add_to_index(activity) - - :ok end def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do @@ -19,7 +17,5 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.remove_from_index(object) - - :ok end end -- cgit v1.2.3 From 6256822afd368e5f6b410d47c5ff9b584e50a461 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 01:11:50 +0300 Subject: Check for updateId, not uid --- lib/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 24789b00c..0b90971b1 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -153,7 +153,7 @@ defmodule Pleroma.Search.Meilisearch do ) with {:ok, res} <- result, - true <- Map.has_key?(res, "uid") do + true <- Map.has_key?(res, "updateId") do # Added successfully :ok else -- cgit v1.2.3 From 5a39866388c411f2bcee9848352f8c420513f34f Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 01:43:59 +0300 Subject: Specifically strip mentions for search indexing --- lib/mix/tasks/pleroma/search/meilisearch.ex | 1 + lib/pleroma/search/meilisearch.ex | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 72a558228..8379a0c25 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -13,6 +13,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["index"]) do start_pleroma() + Pleroma.HTML.compile_scrubbers() meili_version = ( diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 0b90971b1..7af7f460a 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -122,7 +122,8 @@ defmodule Pleroma.Search.Meilisearch do end content = - with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + with {:ok, scrubbed} <- + FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing), trimmed <- String.trim(scrubbed) do trimmed end -- cgit v1.2.3 From 675639225a905f5b0b2650cd3f20a4758fc3f868 Mon Sep 17 00:00:00 2001 From: HJ <30-hj@users.noreply.git.pleroma.social> Date: Fri, 28 Apr 2023 11:13:42 +0000 Subject: allow https: so that flash works across instances without need for media proxy --- lib/pleroma/web/plugs/http_security_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 34895c8d5..045384e08 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -104,7 +104,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do {[img_src, " https:"], [media_src, " https:"]} end - connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] + connect_src = ["connect-src 'self' blob: https: ", static_url, ?\s, websocket_url] connect_src = if Config.get(:env) == :dev do -- cgit v1.2.3 From c0d11da2d8edc57ef88163c06a19aad3e28d14db Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Sun, 7 May 2023 15:16:30 +0300 Subject: conditionally set csp depnding on media-proxy state --- lib/pleroma/web/plugs/http_security_plug.ex | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 045384e08..df46cfa0c 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -93,18 +93,26 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" + connect_src = ["connect-src 'self' blob:", static_url, ?\s, websocket_url] # Strict multimedia CSP enforcement only when MediaProxy is enabled - {img_src, media_src} = + {img_src, media_src, connect_src} = if Config.get([:media_proxy, :enabled]) && !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do sources = build_csp_multimedia_source_list() - {[img_src, sources], [media_src, sources]} + { + [img_src, sources], + [media_src, sources], + [connect_src, sources] + } else - {[img_src, " https:"], [media_src, " https:"]} + { + [img_src, " https:"], + [media_src, " https:"], + [connect_src, " https:"] + } end - connect_src = ["connect-src 'self' blob: https: ", static_url, ?\s, websocket_url] connect_src = if Config.get(:env) == :dev do -- cgit v1.2.3 From f8ef4924ecab5ba6851eee82845624bc15f868de Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Sun, 7 May 2023 15:24:09 +0300 Subject: fix whitespace --- lib/pleroma/web/plugs/http_security_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index df46cfa0c..a3166bc96 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -93,7 +93,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" - connect_src = ["connect-src 'self' blob:", static_url, ?\s, websocket_url] + connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = -- cgit v1.2.3 From f50fd9278fd36e6bd3ae36bb7f5033d9fd8a84ac Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Sun, 7 May 2023 15:29:19 +0300 Subject: reduce redundant reduntancy reduction --- lib/pleroma/web/plugs/http_security_plug.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index a3166bc96..b189d5bfd 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -93,7 +93,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" - connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] + connect_src = "connect-src 'self' blob:" # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = @@ -103,7 +103,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do { [img_src, sources], [media_src, sources], - [connect_src, sources] + [connect_src, sources, ?\s, websocket_url] } else { -- cgit v1.2.3 From 2a07411b0cb14ea26966659605d95074b02a8538 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Sun, 7 May 2023 15:34:17 +0300 Subject: keep the websocket url for all modes --- lib/pleroma/web/plugs/http_security_plug.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index b189d5bfd..b3dc8a3a6 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -93,7 +93,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" - connect_src = "connect-src 'self' blob:" + connect_src = ["connect-src 'self' blob: ", ?\s, websocket_url] # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = @@ -103,7 +103,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do { [img_src, sources], [media_src, sources], - [connect_src, sources, ?\s, websocket_url] + [connect_src, sources] } else { -- cgit v1.2.3 From e3110cb34e350bc9353d082328778daae47ba9e8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 13:36:21 -0400 Subject: Fix deprecated calls to get_flash/2 --- lib/pleroma/web.ex | 2 +- lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex | 8 ++++---- lib/pleroma/web/templates/o_auth/mfa/totp.html.eex | 8 ++++---- lib/pleroma/web/templates/o_auth/o_auth/register.html.eex | 8 ++++---- lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 8 ++++---- 5 files changed, 17 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index aee41b0fe..7a8b176cd 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -136,7 +136,7 @@ defmodule Pleroma.Web do namespace: Pleroma.Web # Import convenience functions from controllers - import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1] + import Phoenix.Controller, only: [get_csrf_token: 0, view_module: 1] import Pleroma.Web.ErrorHelpers import Pleroma.Web.Gettext diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index e45d13bdf..e3639aae7 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - +<%= if Phoenix.Flash.get(@flash, :info) do %> + <% end %> -<%= if get_flash(@conn, :error) do %> - +<%= if Phoenix.Flash.get(@flash, :error) do %> + <% end %>

<%= Gettext.dpgettext("static_pages", "mfa recover page title", "Two-factor recovery") %>

diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index 50e6c04b6..f995b8805 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - +<%= if Phoenix.Flash.get(@flash, :info) do %> + <% end %> -<%= if get_flash(@conn, :error) do %> - +<%= if Phoenix.Flash.get(@flash, :error) do %> + <% end %>

<%= Gettext.dpgettext("static_pages", "mfa auth page title", "Two-factor authentication") %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index 1f661efb2..e7f65266f 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - +<%= if Phoenix.Flash.get(@flash, :info) do %> + <% end %> -<%= if get_flash(@conn, :error) do %> - +<%= if Phoenix.Flash.get(@flash, :error) do %> + <% end %>

<%= Gettext.dpgettext("static_pages", "oauth register page title", "Registration Details") %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index b3654f3eb..5b38f7142 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - +<%= if Phoenix.Flash.get(@flash, :info) do %> + <% end %> -<%= if get_flash(@conn, :error) do %> - +<%= if Phoenix.Flash.get(@flash, :error) do %> + <% end %> <%= form_for @conn, Routes.o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> -- cgit v1.2.3 From ffee478ed0298f2cf29d4d51ed28105119552496 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 15:30:58 -0400 Subject: Move websocket config for Shoutbox to the Endpoint This is the modern way of configuring it --- lib/pleroma/web/endpoint.ex | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index d8d40cceb..9e0340cd2 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,7 +9,20 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config - socket("/socket", Pleroma.Web.UserSocket) + socket("/socket", Pleroma.Web.UserSocket, + websocket: [ + path: "/websocket", + serializer: [ + {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"}, + {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"} + ], + timeout: 60_000, + transport_log: false, + compress: false + ], + longpoll: false + ) + socket("/live", Phoenix.LiveView.Socket) plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) -- cgit v1.2.3 From 62322f71e2f2e607ee66a91a99e35eecbf89914d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 16:22:40 -0400 Subject: Clean up Plug.Parsers.MULTIPART deprecation warnings There is no need to the length setting to :multipart. The length setting is global for all of the parsers. --- lib/pleroma/web/endpoint.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 9e0340cd2..201a92653 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -116,7 +116,7 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Parsers, parsers: [ :urlencoded, - {:multipart, length: {Config, :get, [[:instance, :upload_limit]]}}, + :multipart, :json ], pass: ["*/*"], -- cgit v1.2.3 From 2b8bbb288c26a95970e2d9e988f5eb303e052644 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 22:11:17 +0000 Subject: Phoenix.Socket.Transport.force_ssl/4 no longer exists --- lib/phoenix/transports/web_socket/raw.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/phoenix/transports/web_socket/raw.ex b/lib/phoenix/transports/web_socket/raw.ex index 8cf9c32a2..cf4fda79f 100644 --- a/lib/phoenix/transports/web_socket/raw.ex +++ b/lib/phoenix/transports/web_socket/raw.ex @@ -26,7 +26,6 @@ defmodule Phoenix.Transports.WebSocket.Raw do conn |> fetch_query_params |> Transport.transport_log(opts[:transport_log]) - |> Transport.force_ssl(handler, endpoint, opts) |> Transport.check_origin(handler, endpoint, opts) case conn do -- cgit v1.2.3 From f0e5f0e8375bb55e13b0f054fcf09871d2967d24 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 22:14:55 +0000 Subject: Fix compile warning warning: doing a prefix match with globs is deprecated, invalid segment "pleroma*path" --- lib/pleroma/web/router.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c1a690e28..6b9e158a3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -996,8 +996,8 @@ defmodule Pleroma.Web.Router do scope "/", Pleroma.Web.Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) - match(:*, "/api/pleroma*path", LegacyPleromaApiRerouterPlug, []) - get("/api*path", RedirectController, :api_not_implemented) + match(:*, "/api/pleroma/*path", LegacyPleromaApiRerouterPlug, []) + get("/api/*path", RedirectController, :api_not_implemented) get("/*path", RedirectController, :redirector_with_preload) options("/*path", RedirectController, :empty) -- cgit v1.2.3 From 63ef1dcedca7fb315f611388e477d3847d1acaa1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 3 Jun 2023 14:17:49 -0400 Subject: Phoenix.Router.routes/1 is the public function we are meant to be using here --- lib/pleroma/web/router.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..a6e180c4c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -1003,9 +1003,8 @@ defmodule Pleroma.Web.Router do options("/*path", RedirectController, :empty) end - # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+ def get_api_routes do - __MODULE__.__routes__() + Phoenix.Router.routes(__MODULE__) |> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end) |> Enum.map(fn r -> r.path -- cgit v1.2.3 From ea4225a646b355150fb8e5e8c77d7fdc58b5e7ef Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 18 Jul 2023 18:39:59 -0400 Subject: Restrict attachments to only uploaded files only --- lib/pleroma/constants.ex | 2 ++ lib/pleroma/web/common_api/utils.ex | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 7b4fd03b6..6befc6897 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -81,4 +81,6 @@ defmodule Pleroma.Constants do const(mime_regex, do: ~r/^[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+\/[^[:cntrl:] ()<>@,;:\\"\/\[\]?=]+(; .*)?$/ ) + + const(upload_object_types, do: ["Document", "Image"]) end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index a93c97e1e..b9fe0224c 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -59,7 +59,12 @@ defmodule Pleroma.Web.CommonAPI.Utils do end defp get_attachment(media_id) do - Repo.get(Object, media_id) + with %Object{data: data} = object <- Repo.get(Object, media_id), + %{"type" => type} when type in Pleroma.Constants.upload_object_types() <- data do + object + else + _ -> nil + end end @spec get_to_and_cc(ActivityDraft.t()) :: {list(String.t()), list(String.t())} -- cgit v1.2.3 From dc4de79d43ba941d8aa16f7c14887faae9f65e9f Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Wed, 7 Dec 2022 22:37:50 +0500 Subject: status context: perform visibility check on activities around a status issue #2927 --- lib/pleroma/web/activity_pub/activity_pub.ex | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c93288b79..f1a12d22d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -455,6 +455,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) |> maybe_set_thread_muted_field(opts) + |> restrict_unauthenticated(opts[:user]) |> restrict_blocked(opts) |> restrict_blockers_visibility(opts) |> restrict_recipients(recipients, opts[:user]) @@ -1215,6 +1216,28 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_filtered(query, _), do: query + defp restrict_unauthenticated(query, nil) do + local = Config.restrict_unauthenticated_access?(:activities, :local) + remote = Config.restrict_unauthenticated_access?(:activities, :remote) + + cond do + local and remote -> + # is there a better way to handle this? + from(activity in query, where: 1 == 0) + + local -> + from(activity in query, where: activity.local == false) + + remote -> + from(activity in query, where: activity.local == true) + + true -> + query + end + end + + defp restrict_unauthenticated(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do -- cgit v1.2.3 From e5e76ec44559af62eb56043add0489c61e6c1ee5 Mon Sep 17 00:00:00 2001 From: Faried Nawaz Date: Fri, 10 Feb 2023 01:32:32 +0500 Subject: cleaner ecto query to handle restrict_unauthenticated for activities This fix is for this case: config :pleroma, :restrict_unauthenticated, activities: %{local: true, remote: true} --- lib/pleroma/web/activity_pub/activity_pub.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f1a12d22d..3979d418e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1222,8 +1222,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do cond do local and remote -> - # is there a better way to handle this? - from(activity in query, where: 1 == 0) + from(activity in query, where: false) local -> from(activity in query, where: activity.local == false) -- cgit v1.2.3 From 2c795094535537a8607cc0d3b7f076a609636f40 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 3 Aug 2023 13:08:37 -0400 Subject: Resolve information disclosure vulnerability through emoji pack archive download endpoint The pack name has been sanitized so an attacker cannot upload a media file called pack.json with their own handcrafted list of emoji files as arbitrary files on the filesystem and then call the emoji pack archive download endpoint with a pack name crafted to the location of the media file they uploaded which tricks Pleroma into generating a zip file of the target files the attacker wants to download. The attack only works if the Pleroma instance does not have the AnonymizeFilename upload filter enabled, which is currently the default. Reported by: graf@poast.org --- lib/pleroma/emoji/pack.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index a361ea200..6e58f8898 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -285,6 +285,7 @@ defmodule Pleroma.Emoji.Pack do @spec load_pack(String.t()) :: {:ok, t()} | {:error, :file.posix()} def load_pack(name) do + name = Path.basename(name) pack_file = Path.join([emoji_path(), name, "pack.json"]) with {:ok, _} <- File.stat(pack_file), -- cgit v1.2.3 From 8cc8100120abdbf26cfe4cdac2c0a012d7919e05 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 22 Jun 2023 00:46:52 +0200 Subject: Config: Restrict permissions of OTP config file --- lib/pleroma/config/release_runtime_provider.ex | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/config/release_runtime_provider.ex b/lib/pleroma/config/release_runtime_provider.ex index 91e5f1a54..9ec0f975e 100644 --- a/lib/pleroma/config/release_runtime_provider.ex +++ b/lib/pleroma/config/release_runtime_provider.ex @@ -20,6 +20,20 @@ defmodule Pleroma.Config.ReleaseRuntimeProvider do with_runtime_config = if File.exists?(config_path) do + # + %File.Stat{mode: mode} = File.lstat!(config_path) + + if Bitwise.band(mode, 0o007) > 0 do + raise "Configuration at #{config_path} has world-permissions, execute the following: chmod o= #{config_path}" + end + + if Bitwise.band(mode, 0o020) > 0 do + raise "Configuration at #{config_path} has group-wise write permissions, execute the following: chmod g-w #{config_path}" + end + + # Note: Elixir doesn't provides a getuid(2) + # so cannot forbid group-read only when config is owned by us + runtime_config = Config.Reader.read!(config_path) with_defaults -- cgit v1.2.3 From 69caedc591bd61029f897f37ef7ecddd470cf935 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 22 Jun 2023 00:58:05 +0200 Subject: instance gen: Reduce permissions of pleroma directories and config files --- lib/mix/tasks/pleroma/instance.ex | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 5c93f19ff..5d8b254a2 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -266,12 +266,20 @@ defmodule Mix.Tasks.Pleroma.Instance do config_dir = Path.dirname(config_path) psql_dir = Path.dirname(psql_path) + # Note: Distros requiring group read (0o750) on those directories should + # pre-create the directories. [config_dir, psql_dir, static_dir, uploads_dir] |> Enum.reject(&File.exists?/1) - |> Enum.map(&File.mkdir_p!/1) + |> Enum.each(fn dir -> + File.mkdir_p!(dir) + File.chmod!(dir, 0o700) + end) shell_info("Writing config to #{config_path}.") + # Sadly no fchmod(2) equivalent in Elixir… + File.touch!(config_path) + File.chmod!(config_path, 0o640) File.write(config_path, result_config) shell_info("Writing the postgres script to #{psql_path}.") File.write(psql_path, result_psql) @@ -290,8 +298,7 @@ defmodule Mix.Tasks.Pleroma.Instance do else shell_error( "The task would have overwritten the following files:\n" <> - (Enum.map(will_overwrite, &"- #{&1}\n") |> Enum.join("")) <> - "Rerun with `--force` to overwrite them." + Enum.map_join(will_overwrite, &"- #{&1}\n") <> "Rerun with `--force` to overwrite them." ) end end -- cgit v1.2.3 From ca0859b90f0f3cb9bb369d38d29868de59796c2c Mon Sep 17 00:00:00 2001 From: Mae Date: Fri, 4 Aug 2023 22:24:17 +0100 Subject: Prevent XML parser from loading external entities --- lib/pleroma/web/xml.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/xml.ex b/lib/pleroma/web/xml.ex index b699446b0..380a80ab8 100644 --- a/lib/pleroma/web/xml.ex +++ b/lib/pleroma/web/xml.ex @@ -29,7 +29,10 @@ defmodule Pleroma.Web.XML do {doc, _rest} = text |> :binary.bin_to_list() - |> :xmerl_scan.string(quiet: true) + |> :xmerl_scan.string( + quiet: true, + fetch_fun: fn _, _ -> raise "Resolving external entities not supported" end + ) {:ok, doc} rescue -- cgit v1.2.3 From 48b1e9bdc7382ec6ef33e95f2bd8674ae92f17b2 Mon Sep 17 00:00:00 2001 From: mae Date: Sat, 5 Aug 2023 14:13:49 +0200 Subject: Completely disable xml entity resolution --- lib/pleroma/web/xml.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/xml.ex b/lib/pleroma/web/xml.ex index 380a80ab8..64329e4ba 100644 --- a/lib/pleroma/web/xml.ex +++ b/lib/pleroma/web/xml.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.XML do |> :binary.bin_to_list() |> :xmerl_scan.string( quiet: true, - fetch_fun: fn _, _ -> raise "Resolving external entities not supported" end + allow_entities: false ) {:ok, doc} -- cgit v1.2.3 From 9effa24f308917f70276c41f91fb204b7684d942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 10 Aug 2023 22:52:38 +0200 Subject: Implement api/v2/instance route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/api_spec/operations/instance_operation.ex | 172 +++++++++++++++++++++ .../controllers/instance_controller.ex | 5 + .../web/mastodon_api/views/instance_view.ex | 120 +++++++++++--- lib/pleroma/web/router.ex | 3 + lib/pleroma/web/web_finger.ex | 2 +- 5 files changed, 279 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index a07be7e40..8e395bde8 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -23,6 +23,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + def show2_operation do + %Operation{ + tags: ["Instance misc"], + summary: "Retrieve instance information", + description: "Information about the server", + operationId: "InstanceController.show2", + responses: %{ + 200 => Operation.response("Instance", "application/json", instance2()) + } + } + end + def peers_operation do %Operation{ tags: ["Instance misc"], @@ -165,6 +177,166 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + defp instance2 do + %Schema{ + type: :object, + properties: %{ + domain: %Schema{type: :string, description: "The domain name of the instance"}, + title: %Schema{type: :string, description: "The title of the website"}, + version: %Schema{ + type: :string, + description: "The version of Pleroma installed on the instance" + }, + source_url: %Schema{ + type: :string, + description: "The version of Pleroma installed on the instance" + }, + description: %Schema{ + type: :string, + description: "Admin-defined description of the Pleroma site" + }, + usage: %Schema{ + type: :object, + description: "Instance usage statistics", + properties: %{ + users: %Schema{ + type: :object, + description: "User count statistics", + properties: %{ + active_month: %Schema{ + type: :integer, + description: "Monthly active users" + } + } + } + } + }, + email: %Schema{ + type: :string, + description: "An email that may be contacted for any inquiries", + format: :email + }, + urls: %Schema{ + type: :object, + description: "URLs of interest for clients apps", + properties: %{} + }, + stats: %Schema{ + type: :object, + description: "Statistics about how much information the instance contains", + properties: %{ + user_count: %Schema{ + type: :integer, + description: "Users registered on this instance" + }, + status_count: %Schema{ + type: :integer, + description: "Statuses authored by users on instance" + }, + domain_count: %Schema{ + type: :integer, + description: "Domains federated with this instance" + } + } + }, + thumbnail: %Schema{ + type: :object, + properties: %{ + url: %Schema{ + type: :string, + description: "Banner image for the website", + nullable: true + } + } + }, + languages: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: "Primary langauges of the website and its staff" + }, + registrations: %Schema{ + type: :object, + description: "Registrations-related configuration", + properties: %{ + enabled: %Schema{ + type: :boolean, + description: "Whether registrations are enabled" + }, + approval_required: %Schema{ + type: :boolean, + description: "Whether users need to be manually approved by admin" + } + } + }, + configuration: %Schema{ + type: :object, + description: "Instance configuration", + properties: %{ + urls: %Schema{ + type: :object, + properties: %{ + streaming: %Schema{ + type: :string, + description: "Websockets address for push streaming" + } + } + }, + statuses: %Schema{ + type: :object, + description: "A map with poll limits for local statuses", + properties: %{ + max_characters: %Schema{ + type: :integer, + description: "Posts character limit (CW/Subject included in the counter)" + }, + max_media_attachments: %Schema{ + type: :integer, + description: "Media attachment limit" + } + } + }, + media_attachments: %Schema{ + type: :object, + description: "A map with poll limits for media attachments", + properties: %{ + image_size_limit: %Schema{ + type: :integer, + description: "File size limit of uploaded images" + }, + video_size_limit: %Schema{ + type: :integer, + description: "File size limit of uploaded videos" + } + } + }, + polls: %Schema{ + type: :object, + description: "A map with poll limits for local polls", + properties: %{ + max_options: %Schema{ + type: :integer, + description: "Maximum number of options." + }, + max_characters_per_option: %Schema{ + type: :integer, + description: "Maximum number of characters per option." + }, + min_expiration: %Schema{ + type: :integer, + description: "Minimum expiration time (in seconds)." + }, + max_expiration: %Schema{ + type: :integer, + description: "Maximum expiration time (in seconds)." + } + } + } + } + } + } + } + end + defp array_of_domains do %Schema{ type: :array, diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 6410e872c..3757a850a 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -16,6 +16,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do render(conn, "show.json") end + @doc "GET /api/v2/instance" + def show2(conn, _params) do + render(conn, "show2.json") + end + @doc "GET /api/v1/instance/peers" def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index efd2a0af6..c987cafd6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do instance = Config.get(:instance) %{ - uri: Pleroma.Web.Endpoint.url(), + uri: Pleroma.Web.WebFinger.domain(), title: Keyword.get(instance, :name), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), @@ -30,6 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do languages: Keyword.get(instance, :languages, ["en"]), registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), + configuration: configuration(), # Extra (not present in Mastodon): max_toot_chars: Keyword.get(instance, :limit), max_media_attachments: Keyword.get(instance, :max_media_attachments), @@ -41,19 +42,38 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), - pleroma: %{ - metadata: %{ - account_activation_required: Keyword.get(instance, :account_activation_required), - features: features(), - federation: federation(), - fields_limits: fields_limits(), - post_formats: Config.get([:instance, :allowed_post_formats]), - birthday_required: Config.get([:instance, :birthday_required]), - birthday_min_age: Config.get([:instance, :birthday_min_age]) - }, - stats: %{mau: Pleroma.User.active_user_count()}, - vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) - } + pleroma: pleroma_configuration(instance) + } + end + + def render("show2.json", _) do + instance = Config.get(:instance) + + %{ + domain: Pleroma.Web.WebFinger.domain(), + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", + source_url: Pleroma.Application.repository(), + description: Keyword.get(instance, :short_description), + usage: %{users: %{active_month: Pleroma.User.active_user_count()}}, + thumbnail: %{ + url: + URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) + |> to_string + }, + languages: Keyword.get(instance, :languages, ["en"]), + configuration: configuration2(), + registrations: %{ + enabled: Keyword.get(instance, :registrations_open), + approval_required: Keyword.get(instance, :account_approval_required), + message: nil + }, + contact: %{ + email: Keyword.get(instance, :email), + account: nil + }, + # Extra (not present in Mastodon): + pleroma: pleroma_configuration2(instance) } end @@ -68,6 +88,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "shareable_emoji_packs", "multifetch", "pleroma:api/v1/notifications:include_types_filter", + "quote_posting", "editing", if Config.get([:activitypub, :blockers_visible]) do "blockers_visible" @@ -78,13 +99,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do if Config.get([:gopher, :enabled]) do "gopher" end, - # backwards compat - if Config.get([:shout, :enabled]) do - "chat" - end, - if Config.get([:shout, :enabled]) do - "shout" - end, if Config.get([:instance, :allow_relay]) do "relay" end, @@ -94,6 +108,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "pleroma_emoji_reactions", "pleroma_custom_emoji_reactions", "pleroma_chat_messages", + "email_list", if Config.get([:instance, :show_reactions]) do "exposable_reactions" end, @@ -132,7 +147,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do |> Map.put(:enabled, Config.get([:instance, :federating])) end - def fields_limits do + defp fields_limits do %{ max_fields: Config.get([:instance, :max_account_fields]), max_remote_fields: Config.get([:instance, :max_remote_account_fields]), @@ -140,4 +155,65 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do value_length: Config.get([:instance, :account_field_value_length]) } end + + defp configuration do + %{ + statuses: %{ + max_characters: Config.get([:instance, :limit]), + max_media_attachments: Config.get([:instance, :max_media_attachments]) + }, + media_attachments: %{ + image_size_limit: Config.get([:instance, :upload_limit]), + video_size_limit: Config.get([:instance, :upload_limit]) + }, + polls: %{ + max_options: Config.get([:instance, :poll_limits, :max_options]), + max_characters_per_option: Config.get([:instance, :poll_limits, :max_option_chars]), + min_expiration: Config.get([:instance, :poll_limits, :min_expiration]), + max_expiration: Config.get([:instance, :poll_limits, :max_expiration]) + } + } + end + + defp configuration2 do + configuration() + |> Map.merge(%{ + urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()} + }) + end + + defp pleroma_configuration(instance) do + %{ + metadata: %{ + account_activation_required: Keyword.get(instance, :account_activation_required), + features: features(), + federation: federation(), + fields_limits: fields_limits(), + post_formats: Config.get([:instance, :allowed_post_formats]), + birthday_required: Config.get([:instance, :birthday_required]), + birthday_min_age: Config.get([:instance, :birthday_min_age]) + }, + stats: %{mau: Pleroma.User.active_user_count()}, + vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } + end + + defp pleroma_configuration2(instance) do + configuration = pleroma_configuration(instance) + + configuration + |> Map.merge(%{ + metadata: + configuration.metadata + |> Map.merge(%{ + avatar_upload_limit: Keyword.get(instance, :avatar_upload_limit), + background_upload_limit: Keyword.get(instance, :background_upload_limit), + banner_upload_limit: Keyword.get(instance, :banner_upload_limit), + background_image: + Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), + description_limit: Keyword.get(instance, :description_limit), + shout_limit: Config.get([:shout, :limit]) + }) + }) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..5362100cc 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -774,11 +774,14 @@ defmodule Pleroma.Web.Router do scope "/api/v2", Pleroma.Web.MastodonAPI do pipe_through(:api) + get("/search", SearchController, :search2) post("/media", MediaController, :create2) get("/suggestions", SuggestionController, :index2) + + get("/instance", InstanceController, :show2) end scope "/api", Pleroma.Web do diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index f95dc2458..b28fad8d1 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -96,7 +96,7 @@ defmodule Pleroma.Web.WebFinger do |> XmlBuilder.to_doc() end - defp domain do + def domain do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end -- cgit v1.2.3 From 79e46ce73f782a83654986adc9fd0b256be6a2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 11 Aug 2023 13:57:22 +0200 Subject: InstanceView: Add common_information function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/mastodon_api/views/instance_view.ex | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index c987cafd6..e7a2feb7f 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -13,12 +13,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def render("show.json", _) do instance = Config.get(:instance) - %{ + common_information(instance) + |> Map.merge(%{ uri: Pleroma.Web.WebFinger.domain(), - title: Keyword.get(instance, :name), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", email: Keyword.get(instance, :email), urls: %{ streaming_api: Pleroma.Web.Endpoint.websocket_url() @@ -27,7 +26,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do thumbnail: URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) |> to_string, - languages: Keyword.get(instance, :languages, ["en"]), registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), configuration: configuration(), @@ -43,16 +41,15 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), pleroma: pleroma_configuration(instance) - } + }) end def render("show2.json", _) do instance = Config.get(:instance) - %{ + common_information(instance) + |> Map.merge(%{ domain: Pleroma.Web.WebFinger.domain(), - title: Keyword.get(instance, :name), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", source_url: Pleroma.Application.repository(), description: Keyword.get(instance, :short_description), usage: %{users: %{active_month: Pleroma.User.active_user_count()}}, @@ -61,7 +58,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) |> to_string }, - languages: Keyword.get(instance, :languages, ["en"]), configuration: configuration2(), registrations: %{ enabled: Keyword.get(instance, :registrations_open), @@ -74,6 +70,14 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }, # Extra (not present in Mastodon): pleroma: pleroma_configuration2(instance) + }) + end + + defp common_information(instance) do + %{ + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", + languages: Keyword.get(instance, :languages, ["en"]) } end -- cgit v1.2.3 From d838d1990bf23d452c1cc830629e42e51dbd7047 Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Wed, 16 Aug 2023 13:34:32 +0000 Subject: Apply lanodan's suggestion(s) to 1 file(s) --- lib/pleroma/web/plugs/http_security_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index b3dc8a3a6..a3166bc96 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -93,7 +93,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" - connect_src = ["connect-src 'self' blob: ", ?\s, websocket_url] + connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = -- cgit v1.2.3 From 3d09bc320efcc68beb9b57fba23b6b9f3dc17905 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 30 Aug 2023 20:34:16 -0400 Subject: Make lint happy --- lib/pleroma/web/plugs/http_security_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index a3166bc96..5093414c4 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -100,6 +100,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do if Config.get([:media_proxy, :enabled]) && !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do sources = build_csp_multimedia_source_list() + { [img_src, sources], [media_src, sources], @@ -113,7 +114,6 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do } end - connect_src = if Config.get(:env) == :dev do [connect_src, " http://localhost:3035/"] -- cgit v1.2.3 From 1afde067b12ad0062c1820091ea9b0a680819281 Mon Sep 17 00:00:00 2001 From: Mint Date: Sat, 2 Sep 2023 01:43:25 +0300 Subject: CommonAPI: Prevent users from accessing media of other users --- lib/pleroma/scheduled_activity.ex | 6 +++++- lib/pleroma/web/common_api.ex | 12 ++++++++++++ lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 27 ++++++++++++++------------- 4 files changed, 32 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index a7be58512..0ed51ad07 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -40,7 +40,11 @@ defmodule Pleroma.ScheduledActivity do %{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset ) when is_list(media_ids) do - media_attachments = Utils.attachments_from_ids(%{media_ids: media_ids}) + media_attachments = + Utils.attachments_from_ids( + %{media_ids: media_ids}, + User.get_cached_by_id(changeset.data.user_id) + ) params = params diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 77b3fa5d2..82c7f70d2 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -33,6 +33,7 @@ defmodule Pleroma.Web.CommonAPI do def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]), + :ok <- validate_chat_attachment_attribution(maybe_attachment, user), :ok <- validate_chat_content_length(content, !!maybe_attachment), {_, {:ok, chat_message_data, _meta}} <- {:build_object, @@ -71,6 +72,17 @@ defmodule Pleroma.Web.CommonAPI do text end + defp validate_chat_attachment_attribution(nil, _), do: :ok + + defp validate_chat_attachment_attribution(attachment, user) do + with :ok <- Object.authorize_access(attachment, user) do + :ok + else + e -> + e + end + end + defp validate_chat_content_length(_, true), do: :ok defp validate_chat_content_length(nil, false), do: {:error, :no_content} diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 9af635da8..63ed48a27 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -111,7 +111,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end defp attachments(%{params: params} = draft) do - attachments = Utils.attachments_from_ids(params) + attachments = Utils.attachments_from_ids(params, draft.user) draft = %__MODULE__{draft | attachments: attachments} case Utils.validate_attachments_count(attachments) do diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b9fe0224c..0f394e951 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -23,21 +23,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do require Logger require Pleroma.Constants - def attachments_from_ids(%{media_ids: ids, descriptions: desc}) do - attachments_from_ids_descs(ids, desc) + def attachments_from_ids(%{media_ids: ids, descriptions: desc}, user) do + attachments_from_ids_descs(ids, desc, user) end - def attachments_from_ids(%{media_ids: ids}) do - attachments_from_ids_no_descs(ids) + def attachments_from_ids(%{media_ids: ids}, user) do + attachments_from_ids_no_descs(ids, user) end - def attachments_from_ids(_), do: [] + def attachments_from_ids(_, _), do: [] - def attachments_from_ids_no_descs([]), do: [] + def attachments_from_ids_no_descs([], _), do: [] - def attachments_from_ids_no_descs(ids) do + def attachments_from_ids_no_descs(ids, user) do Enum.map(ids, fn media_id -> - case get_attachment(media_id) do + case get_attachment(media_id, user) do %Object{data: data} -> data _ -> nil end @@ -45,22 +45,23 @@ defmodule Pleroma.Web.CommonAPI.Utils do |> Enum.reject(&is_nil/1) end - def attachments_from_ids_descs([], _), do: [] + def attachments_from_ids_descs([], _, _), do: [] - def attachments_from_ids_descs(ids, descs_str) do + def attachments_from_ids_descs(ids, descs_str, user) do {_, descs} = Jason.decode(descs_str) Enum.map(ids, fn media_id -> - with %Object{data: data} <- get_attachment(media_id) do + with %Object{data: data} <- get_attachment(media_id, user) do Map.put(data, "name", descs[media_id]) end end) |> Enum.reject(&is_nil/1) end - defp get_attachment(media_id) do + defp get_attachment(media_id, user) do with %Object{data: data} = object <- Repo.get(Object, media_id), - %{"type" => type} when type in Pleroma.Constants.upload_object_types() <- data do + %{"type" => type} when type in Pleroma.Constants.upload_object_types() <- data, + :ok <- Object.authorize_access(object, user) do object else _ -> nil -- cgit v1.2.3 From 28ef5ebd3c7c2fc083424dbf4434392fcd2a7aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 7 Sep 2023 15:00:24 +0200 Subject: Update InstanceView.features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index e7a2feb7f..c1cecf6e6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -92,7 +92,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "shareable_emoji_packs", "multifetch", "pleroma:api/v1/notifications:include_types_filter", - "quote_posting", "editing", if Config.get([:activitypub, :blockers_visible]) do "blockers_visible" @@ -103,6 +102,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do if Config.get([:gopher, :enabled]) do "gopher" end, + # backwards compat + if Config.get([:shout, :enabled]) do + "chat" + end, + if Config.get([:shout, :enabled]) do + "shout" + end, if Config.get([:instance, :allow_relay]) do "relay" end, @@ -112,7 +118,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "pleroma_emoji_reactions", "pleroma_custom_emoji_reactions", "pleroma_chat_messages", - "email_list", if Config.get([:instance, :show_reactions]) do "exposable_reactions" end, -- cgit v1.2.3 From 31eb3dc24587ad3715da9fe00886867a6c0bf0c4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 16:41:30 -0600 Subject: ObjectValidators: accept "quoteUrl" field --- lib/pleroma/web/activity_pub/object_validators/common_fields.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index d580208df..835ed97b7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do end end - # All objects except Answer and CHatMessage + # All objects except Answer and ChatMessage defmacro object_fields do quote bind_quoted: binding() do field(:content, :string) @@ -58,6 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) field(:inReplyTo, ObjectValidators.ObjectID) + field(:quoteUrl, ObjectValidators.ObjectID) field(:url, ObjectValidators.BareUri) field(:likes, {:array, ObjectValidators.ObjectID}, default: []) -- cgit v1.2.3 From 795736af16dca77929725e7dd55f5de04a796fdb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 18:03:22 -0600 Subject: ObjectValidators: improve quoteUrl compatibility --- .../object_validators/article_note_page_validator.ex | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 2670e3f17..40bb67934 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -76,6 +76,21 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do def fix_attachments(data), do: data + defp fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data + + # Fix for Fedibird + # https://github.com/fedibird/mastodon/issues/9 + defp fix_quote_url(%{"quoteURL" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + # Misskey fallback + defp fix_quote_url(%{"_misskey_quote" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + defp fix_quote_url(data), do: data + defp fix(data) do data |> CommonFixes.fix_actor() @@ -84,6 +99,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do |> fix_tag() |> fix_replies() |> fix_attachments() + |> fix_quote_url() |> Transmogrifier.fix_emoji() |> Transmogrifier.fix_content_map() end -- cgit v1.2.3 From b022d6635dad4b2769fbf1fd4b97f77a4cc646b4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 18:46:58 -0600 Subject: Transmogrifier: fetch quoted post --- lib/pleroma/web/activity_pub/transmogrifier.ex | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 0e6c429f9..c466271ca 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -166,6 +166,22 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(object, _options), do: object + def fix_quote(object, options \\ []) + + def fix_quote(%{"quoteUrl" => quote_url} = object, options) + when not is_nil(quote_url) do + with {:ok, quoted_object} <- get_obj_helper(quote_url, options), + %Activity{} <- Activity.get_create_by_object_ap_id(quoted_object.data["id"]) do + Map.put(object, "quoteUrl", quoted_object.data["id"]) + else + e -> + Logger.warn("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") + object + end + end + + def fix_quote(object, _options), do: object + defp prepare_in_reply_to(in_reply_to) do cond do is_bitstring(in_reply_to) -> @@ -454,6 +470,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> strip_internal_fields() |> fix_type(fetch_options) |> fix_in_reply_to(fetch_options) + |> fix_quote(fetch_options) data = Map.put(data, "object", object) options = Keyword.put(options, :local, false) -- cgit v1.2.3 From cc4badaf60462fdb8bb57225437e3dd360ee0dfb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 19:14:39 -0600 Subject: Transmogrifier: fix quoteUrl here too --- lib/pleroma/web/activity_pub/transmogrifier.ex | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index c466271ca..f5771e75e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -166,9 +166,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(object, _options), do: object - def fix_quote(object, options \\ []) + def fix_quote_url(object, options \\ []) - def fix_quote(%{"quoteUrl" => quote_url} = object, options) + def fix_quote_url(%{"quoteUrl" => quote_url} = object, options) when not is_nil(quote_url) do with {:ok, quoted_object} <- get_obj_helper(quote_url, options), %Activity{} <- Activity.get_create_by_object_ap_id(quoted_object.data["id"]) do @@ -180,7 +180,22 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - def fix_quote(object, _options), do: object + # Fix for Fedibird + # https://github.com/fedibird/mastodon/issues/9 + def fix_quote_url(%{"quoteURL" => quote_url} = object, options) do + object + |> Map.put("quoteUrl", quote_url) + |> fix_quote_url(options) + end + + # Misskey fallback + def fix_quote_url(%{"_misskey_quote" => quote_url} = object, options) do + object + |> Map.put("quoteUrl", quote_url) + |> fix_quote_url(options) + end + + def fix_quote_url(object, _options), do: object defp prepare_in_reply_to(in_reply_to) do cond do @@ -470,7 +485,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> strip_internal_fields() |> fix_type(fetch_options) |> fix_in_reply_to(fetch_options) - |> fix_quote(fetch_options) + |> fix_quote_url(fetch_options) data = Map.put(data, "object", object) options = Keyword.put(options, :local, false) -- cgit v1.2.3 From ce5eb3172321f0ef2ae85d7819b44cc8241a5581 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 19:47:08 -0600 Subject: StatusView: show quoted posts through the API, probably --- lib/pleroma/web/mastodon_api/views/status_view.ex | 42 ++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index dea22f9c2..b966a84d0 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -57,6 +57,27 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end) end + defp get_quoted_activities([]), do: %{} + + defp get_quoted_activities(activities) do + activities + |> Enum.map(fn + %{data: %{"type" => "Create"}} = activity -> + object = Object.normalize(activity, fetch: false) + object && object.data["quoteUrl"] != "" && object.data["quoteUrl"] + + _ -> + nil + end) + |> Enum.filter(& &1) + |> Activity.create_by_object_ap_id_with_object() + |> Repo.all() + |> Enum.reduce(%{}, fn activity, acc -> + object = Object.normalize(activity, fetch: false) + if object, do: Map.put(acc, object.data["id"], activity), else: acc + end) + end + # DEPRECATED This field seems to be a left-over from the StatusNet era. # If your application uses `pleroma.conversation_id`: this field is deprecated. # It is currently stubbed instead by doing a CRC32 of the context, and @@ -97,6 +118,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # length(activities_with_links) * timeout fetch_rich_media_for_activities(activities) replied_to_activities = get_replied_to_activities(activities) + quoted_activities = get_quoted_activities(activities) parent_activities = activities @@ -129,6 +151,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do opts = opts |> Map.put(:replied_to_activities, replied_to_activities) + |> Map.put(:quoted_activities, quoted_activities) |> Map.put(:parent_activities, parent_activities) |> Map.put(:relationships, relationships_opt) @@ -277,7 +300,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end reply_to = get_reply_to(activity, opts) - reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) history_len = @@ -290,6 +312,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # Here the implicit index of the current content is 0 chrono_order = history_len - 1 + quote_activity = get_quote(activity, opts) + content = object |> render_content() @@ -398,6 +422,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do conversation_id: get_context_id(activity), context: object.data["context"], in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, + quote_id: quote_activity && to_string(quote_activity.id), content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary}, expires_at: expires_at, @@ -633,6 +658,21 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end + def get_quote(activity, %{quoted_activities: quoted_activities}) do + object = Object.normalize(activity, fetch: false) + quoted_activities[object.data["quoteUrl"]] + end + + def get_quote(%{data: %{"object" => _object}} = activity, _) do + object = Object.normalize(activity, fetch: false) + + if object.data["quoteUrl"] && object.data["quoteUrl"] != "" do + Activity.get_create_by_object_ap_id(object.data["quoteUrl"]) + else + nil + end + end + def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do url = object.data["url"] || object.data["id"] -- cgit v1.2.3 From 0d9c443e51c85d9ded3e20954c9620f7a9d2361e Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 20:05:58 -0600 Subject: StatusView: render the whole quoted status --- lib/pleroma/web/api_spec/schemas/status.ex | 5 +++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 10 +++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index bc29cf4a6..39241aa39 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -193,6 +193,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do nullable: true, description: "The `acct` property of User entity for replied user (if any)" }, + quote: %Schema{ + allOf: [%OpenApiSpex.Reference{"$ref": "#/components/schemas/Status"}], + nullable: true, + description: "Quoted status (if any)" + }, local: %Schema{ type: :boolean, description: "`true` if the post was made on the local instance" diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index b966a84d0..5bde1ce04 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -314,6 +314,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do quote_activity = get_quote(activity, opts) + quote_post = + if quote_activity do + quote_rendering_opts = Map.put(opts, :activity, quote_activity) + render("show.json", quote_rendering_opts) + else + nil + end + content = object |> render_content() @@ -422,7 +430,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do conversation_id: get_context_id(activity), context: object.data["context"], in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, - quote_id: quote_activity && to_string(quote_activity.id), + quote: quote_post, content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary}, expires_at: expires_at, -- cgit v1.2.3 From 6ac19c3999c543e5a26bbf04932a6a7aaa447b99 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 21:27:05 -0600 Subject: ActivityDraft: create quote posts --- lib/pleroma/web/common_api/activity_draft.ex | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 63ed48a27..375aabc91 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -22,6 +22,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do attachments: [], in_reply_to: nil, in_reply_to_conversation: nil, + quote_post: nil, visibility: nil, expires_at: nil, extra: nil, @@ -53,6 +54,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do |> poll() |> with_valid(&in_reply_to/1) |> with_valid(&in_reply_to_conversation/1) + |> with_valid("e_post/1) |> with_valid(&visibility/1) |> content() |> with_valid(&to_and_cc/1) @@ -132,6 +134,18 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(draft), do: draft + defp quote_post(%{params: %{quote_id: ""}} = draft), do: draft + + defp quote_post(%{params: %{quote_id: id}} = draft) when is_binary(id) do + %__MODULE__{draft | quote_post: Activity.get_by_id(id)} + end + + defp quote_post(%{params: %{quote_id: %Activity{} = quote_post}} = draft) do + %__MODULE__{draft | quote_post: quote_post} + end + + defp quote_post(draft), do: draft + defp in_reply_to_conversation(draft) do in_reply_to_conversation = Participation.get(draft.params[:in_reply_to_conversation_id]) %__MODULE__{draft | in_reply_to_conversation: in_reply_to_conversation} -- cgit v1.2.3 From d4fea8b5595e9e6cd37bdb1cee21285f905693f1 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 22:15:54 -0600 Subject: ActivityDraft: allow quoting --- lib/pleroma/web/activity_pub/builder.ex | 11 +++++++++++ lib/pleroma/web/api_spec/operations/status_operation.ex | 7 ++++++- 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 8eab3a241..eb0bb0e33 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -217,6 +217,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do "tag" => Keyword.values(draft.tags) |> Enum.uniq() } |> add_in_reply_to(draft.in_reply_to) + |> add_quote(draft.quote_post) |> Map.merge(draft.extra) {:ok, data, []} @@ -232,6 +233,16 @@ defmodule Pleroma.Web.ActivityPub.Builder do end end + defp add_quote(object, nil), do: object + + defp add_quote(object, quote_post) do + with %Object{} = quote_object <- Object.normalize(quote_post, fetch: false) do + Map.put(object, "quoteUrl", quote_object.data["id"]) + else + _ -> object + end + end + def chat_message(actor, recipient, content, opts \\ []) do basic = %{ "id" => Utils.generate_object_id(), diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 5d6e82f3c..8fa3b0890 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -581,7 +581,12 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do type: :string, description: "Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`." - } + }, + quote_id: %Schema{ + nullable: true, + allOf: [FlakeID], + description: "ID of the status being quoted, if any" + }, }, example: %{ "status" => "What time is it?", -- cgit v1.2.3 From 96009739173e5e48a636bb964855eb7aea11c828 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 22:57:42 -0600 Subject: mix format --- lib/pleroma/web/api_spec/operations/status_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 8fa3b0890..c133a3aac 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -586,7 +586,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do nullable: true, allOf: [FlakeID], description: "ID of the status being quoted, if any" - }, + } }, example: %{ "status" => "What time is it?", -- cgit v1.2.3 From 5716f88a1d8424cf7c62a0491b3bf9607dc9aa3f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sat, 22 Jan 2022 23:09:33 -0600 Subject: InstanceView: add "quote_posting" feature --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index efd2a0af6..1b01d7371 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -69,6 +69,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "multifetch", "pleroma:api/v1/notifications:include_types_filter", "editing", + "quote_posting", if Config.get([:activitypub, :blockers_visible]) do "blockers_visible" end, -- cgit v1.2.3 From 80ab2572a4d5590d738cc763a87156b3f79362fb Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Jan 2022 13:55:25 -0600 Subject: Return quote_url through the API, don't render quotes more than 1 level deep --- lib/pleroma/web/api_spec/schemas/status.ex | 6 ++++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 39241aa39..f4ee9b38c 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -198,6 +198,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do nullable: true, description: "Quoted status (if any)" }, + quote_url: %Schema{ + type: :string, + format: :uri, + nullable: true, + description: "URL of the quoted status" + }, local: %Schema{ type: :boolean, description: "`true` if the post was made on the local instance" diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 5bde1ce04..06adfb221 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -316,7 +316,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do quote_post = if quote_activity do - quote_rendering_opts = Map.put(opts, :activity, quote_activity) + quote_rendering_opts = Map.merge(opts, %{activity: quote_activity, show_quote: false}) render("show.json", quote_rendering_opts) else nil @@ -431,6 +431,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do context: object.data["context"], in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, quote: quote_post, + quote_url: object.data["quoteUrl"], content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary}, expires_at: expires_at, @@ -666,6 +667,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end + def get_quote(_activity, %{show_quote: false}) do + nil + end + def get_quote(activity, %{quoted_activities: quoted_activities}) do object = Object.normalize(activity, fetch: false) quoted_activities[object.data["quoteUrl"]] -- cgit v1.2.3 From 54a989793878c63900d2c6de7b4ffc8fbd8fe8c6 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Jan 2022 15:46:44 -0600 Subject: ActivityDraft: mention the OP of a quoted post --- lib/pleroma/web/common_api/activity_draft.ex | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 375aabc91..588aba55e 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -137,11 +137,11 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp quote_post(%{params: %{quote_id: ""}} = draft), do: draft defp quote_post(%{params: %{quote_id: id}} = draft) when is_binary(id) do - %__MODULE__{draft | quote_post: Activity.get_by_id(id)} - end - - defp quote_post(%{params: %{quote_id: %Activity{} = quote_post}} = draft) do - %__MODULE__{draft | quote_post: quote_post} + with %Activity{actor: actor_ap_id} = activity <- Activity.get_by_id(id) do + %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]} + else + _ -> draft + end end defp quote_post(draft), do: draft @@ -178,12 +178,15 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end - defp content(draft) do + defp content(%{mentions: mentions} = draft) do {content_html, mentioned_users, tags} = Utils.make_content_html(draft) + mentioned_ap_ids = + Enum.map(mentioned_users, fn {_, mentioned_user} -> mentioned_user.ap_id end) + mentions = - mentioned_users - |> Enum.map(fn {_, mentioned_user} -> mentioned_user.ap_id end) + mentions + |> Kernel.++(mentioned_ap_ids) |> Utils.get_addressed_users(draft.params[:to]) %__MODULE__{draft | content_html: content_html, mentions: mentions, tags: tags} -- cgit v1.2.3 From 1f19dd76f66ca657ddfe79a51e59b8997a4c6321 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 23 Jan 2022 16:03:46 -0600 Subject: ActivityDraft: mix format, defensive actor ID --- lib/pleroma/web/common_api/activity_draft.ex | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 588aba55e..95534f3cb 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do alias Pleroma.Web.CommonAPI.Utils import Pleroma.Web.Gettext + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] defstruct valid?: true, errors: [], @@ -134,13 +135,16 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(draft), do: draft - defp quote_post(%{params: %{quote_id: ""}} = draft), do: draft + defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do + case Activity.get_by_id(id) do + %Activity{actor: actor_ap_id} = activity when not_empty_string(actor_ap_id) -> + %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]} - defp quote_post(%{params: %{quote_id: id}} = draft) when is_binary(id) do - with %Activity{actor: actor_ap_id} = activity <- Activity.get_by_id(id) do - %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]} - else - _ -> draft + %Activity{} = activity -> + %__MODULE__{draft | quote_post: activity} + + _ -> + draft end end -- cgit v1.2.3 From 57ef1d121101d785c043ef6aaf2d33bb9be3ec3b Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Mon, 24 Jan 2022 16:44:35 -0600 Subject: Add InlineQuotePolicy to force quote URLs inline --- .../web/activity_pub/mrf/inline_quote_policy.ex | 53 ++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex new file mode 100644 index 000000000..0f1dc9f42 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -0,0 +1,53 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do + @moduledoc "Force a quote line into the message content." + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + defp build_inline_quote(prefix, url) do + "

#{prefix}: #{url}" + end + + defp filter_object(%{"quoteUrl" => quote_url} = object) do + content = object["content"] || "" + + if content =~ quote_url do + object + else + prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix]) + content = content <> build_inline_quote(prefix, quote_url) + Map.put(object, "content", content) + end + end + + @impl true + def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do + {:ok, Map.put(activity, "object", filter_object(object))} + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe, do: {:ok, %{}} + + @impl true + def config_description do + %{ + key: :mrf_inline_quote, + related_policy: "Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy", + label: "MRF Inline Quote", + description: "Force quote post URLs inline", + children: [ + %{ + key: :prefix, + type: :string, + description: "Prefix before the link", + suggestions: ["RT", "QT", "RE", "RN"] + } + ] + } + end +end -- cgit v1.2.3 From 59326247aa754991add9170e204257a8bf94c40f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 26 Jan 2022 11:21:49 -0600 Subject: CommonAPI: disallow quoting private posts through the API --- lib/pleroma/web/common_api/activity_draft.ex | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 95534f3cb..d4875765c 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do alias Pleroma.Conversation.Participation alias Pleroma.Object alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils @@ -57,6 +58,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do |> with_valid(&in_reply_to_conversation/1) |> with_valid("e_post/1) |> with_valid(&visibility/1) + |> with_valid("ing_visibility/1) |> content() |> with_valid(&to_and_cc/1) |> with_valid(&context/1) @@ -136,7 +138,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(draft), do: draft defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do - case Activity.get_by_id(id) do + case Activity.get_by_id_with_object(id) do %Activity{actor: actor_ap_id} = activity when not_empty_string(actor_ap_id) -> %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]} @@ -165,6 +167,17 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end + defp quoting_visibility(%{quote_post: %Activity{}} = draft) do + with %Object{} = object <- Object.normalize(draft.quote_post, fetch: false), + visibility when visibility in ~w(public unlisted) <- Visibility.get_visibility(object) do + draft + else + _ -> add_error(draft, dgettext("errors", "Cannot quote private message")) + end + end + + defp quoting_visibility(draft), do: draft + defp expires_at(draft) do case CommonAPI.check_expiry_date(draft.params[:expires_in]) do {:ok, expires_at} -> %__MODULE__{draft | expires_at: expires_at} -- cgit v1.2.3 From 6f11f11519f9c735f6b059c250f4bf01e09b305f Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 26 Jan 2022 11:49:31 -0600 Subject: StatusView: fix quote visibility --- lib/pleroma/web/mastodon_api/views/status_view.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 06adfb221..7360d1093 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -315,7 +315,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do quote_activity = get_quote(activity, opts) quote_post = - if quote_activity do + if visible_for_user?(quote_activity, opts[:for]) do quote_rendering_opts = Map.merge(opts, %{activity: quote_activity, show_quote: false}) render("show.json", quote_rendering_opts) else -- cgit v1.2.3 From 74e0a4555f583a6962ad116bf6e54f06e42fe465 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Wed, 26 Jan 2022 11:52:50 -0600 Subject: StatusView: add `quote_visible` param --- lib/pleroma/web/api_spec/schemas/status.ex | 4 ++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 1 + 2 files changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index f4ee9b38c..5d0eedb08 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -204,6 +204,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do nullable: true, description: "URL of the quoted status" }, + quote_visible: %Schema{ + type: :boolean, + description: "`true` if the quoted post is visible to the user" + }, local: %Schema{ type: :boolean, description: "`true` if the post was made on the local instance" diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 7360d1093..2aa44b0f6 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -432,6 +432,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, quote: quote_post, quote_url: object.data["quoteUrl"], + quote_visible: visible_for_user?(quote_activity, opts[:for]), content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary}, expires_at: expires_at, -- cgit v1.2.3 From bee7e419597615ac6852942fe563166feba3fe73 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Thu, 27 Jan 2022 14:28:06 -0600 Subject: InlineQuotePolicy: don't add line breaks to markdown posts --- lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index 0f1dc9f42..46013fc5e 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do @moduledoc "Force a quote line into the message content." @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp build_inline_quote(prefix, url) do - "

#{prefix}: #{url}" + defp build_inline_quote(prefix, url, br) do + "#{String.duplicate("
", br)}#{prefix}: #{url}" end defp filter_object(%{"quoteUrl" => quote_url} = object) do @@ -17,7 +17,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do object else prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix]) - content = content <> build_inline_quote(prefix, quote_url) + + inline_quote = + if String.ends_with?(content, "

"), + do: build_inline_quote(prefix, quote_url, 0), + else: build_inline_quote(prefix, quote_url, 2) + + content = content <> inline_quote Map.put(object, "content", content) end end -- cgit v1.2.3 From cf8e4258830e3f362ab1e54238d622f6b2056502 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 28 Jan 2022 12:33:07 -0600 Subject: StatusView: return quote post inside a reblog --- lib/pleroma/web/mastodon_api/views/status_view.ex | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2aa44b0f6..ba4a8f3eb 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -668,13 +668,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end - def get_quote(_activity, %{show_quote: false}) do - nil - end + def get_quote(_activity, %{show_quote: false}), do: nil def get_quote(activity, %{quoted_activities: quoted_activities}) do object = Object.normalize(activity, fetch: false) - quoted_activities[object.data["quoteUrl"]] + + with nil <- quoted_activities[object.data["quoteUrl"]] do + # For when a quote post is inside an Announce + Activity.get_create_by_object_ap_id_with_object(object.data["quoteUrl"]) + end end def get_quote(%{data: %{"object" => _object}} = activity, _) do -- cgit v1.2.3 From 3c8319fe9f7a7c793f8fdc347be2015190981e33 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 28 Jan 2022 14:06:32 -0600 Subject: Transmogrifier: federate quotes with _misskey_quote field --- lib/pleroma/web/activity_pub/transmogrifier.ex | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index f5771e75e..163ae54fa 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -660,6 +660,14 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def set_reply_to_uri(obj), do: obj + # Misskey quotes + # Despite being underscored, it's potentially more reliable for interop. + def set_quote_url(%{"quoteUrl" => quote_url} = object) when is_binary(quote_url) do + Map.put(object, "_misskey_quote", quote_url) + end + + def set_quote_url(obj), do: obj + @doc """ Serialized Mastodon-compatible `replies` collection containing _self-replies_. Based on Mastodon's ActivityPub::NoteSerializer#replies. @@ -714,6 +722,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> prepare_attachments |> set_conversation |> set_reply_to_uri + |> set_quote_url |> set_replies |> strip_internal_fields |> strip_internal_tags -- cgit v1.2.3 From 817e308c0d9e42dfc39742c554e4c7a8da6f1c50 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 28 Jan 2022 15:55:52 -0600 Subject: Handle Fedibird's new quoteUri field --- .../object_validators/article_note_page_validator.ex | 8 +++++++- lib/pleroma/web/activity_pub/transmogrifier.ex | 19 +++++++++++++++---- 2 files changed, 22 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 40bb67934..0b435b251 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -78,7 +78,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do defp fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data - # Fix for Fedibird + # Fedibird + # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac + defp fix_quote_url(%{"quoteUri" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + # Old Fedibird (bug) # https://github.com/fedibird/mastodon/issues/9 defp fix_quote_url(%{"quoteURL" => quote_url} = data) do Map.put(data, "quoteUrl", quote_url) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 163ae54fa..01e135fc1 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -180,7 +180,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end end - # Fix for Fedibird + # Fedibird + # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac + def fix_quote_url(%{"quoteUri" => quote_url} = object, options) do + object + |> Map.put("quoteUrl", quote_url) + |> fix_quote_url(options) + end + + # Old Fedibird (bug) # https://github.com/fedibird/mastodon/issues/9 def fix_quote_url(%{"quoteURL" => quote_url} = object, options) do object @@ -660,10 +668,13 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def set_reply_to_uri(obj), do: obj - # Misskey quotes - # Despite being underscored, it's potentially more reliable for interop. def set_quote_url(%{"quoteUrl" => quote_url} = object) when is_binary(quote_url) do - Map.put(object, "_misskey_quote", quote_url) + Map.merge(object, %{ + # Fedibird quote + "quoteUri" => quote_url, + # Misskey quote + "_misskey_quote" => quote_url + }) end def set_quote_url(obj), do: obj -- cgit v1.2.3 From 4075eecca0033e4487fa9d5d5bb75384597c3e79 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 28 Jan 2022 16:07:17 -0600 Subject: InlineQuotePolicy: improve the way Markdown quotes are displayed by other software --- lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index 46013fc5e..7de4935f2 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do @moduledoc "Force a quote line into the message content." @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp build_inline_quote(prefix, url, br) do - "#{String.duplicate("
", br)}#{prefix}: #{url}" + defp build_inline_quote(prefix, url) do + "

#{prefix}: #{url}" end defp filter_object(%{"quoteUrl" => quote_url} = object) do @@ -18,12 +18,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do else prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix]) - inline_quote = + content = if String.ends_with?(content, "

"), - do: build_inline_quote(prefix, quote_url, 0), - else: build_inline_quote(prefix, quote_url, 2) + do: + String.trim_trailing(content, "

") <> + build_inline_quote(prefix, quote_url) <> "

", + else: content <> build_inline_quote(prefix, quote_url) - content = content <> inline_quote Map.put(object, "content", content) end end -- cgit v1.2.3 From 79fca39faf6d084eabb6be44a2263431943b8dd4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Fri, 28 Jan 2022 17:53:19 -0600 Subject: Actually, don't send _misskey_quote anymore --- lib/pleroma/web/activity_pub/transmogrifier.ex | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 01e135fc1..6c6cd712b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -668,13 +668,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def set_reply_to_uri(obj), do: obj + @doc """ + Fedibird compatibility + https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac + """ def set_quote_url(%{"quoteUrl" => quote_url} = object) when is_binary(quote_url) do - Map.merge(object, %{ - # Fedibird quote - "quoteUri" => quote_url, - # Misskey quote - "_misskey_quote" => quote_url - }) + Map.put(object, "quoteUri", quote_url) end def set_quote_url(obj), do: obj -- cgit v1.2.3 From f9697e68c2b75a77575b9b7c89d08a5687bfd7b4 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Sun, 30 Jan 2022 10:57:29 -0600 Subject: InlineQuotePolicy: skip objects which already have an .inline-quote span --- lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index 7de4935f2..c78675caf 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -10,10 +10,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do "

#{prefix}: #{url}" end + defp has_inline_quote?(content, quote_url) do + cond do + # Does the quote URL exist in the content? + content =~ quote_url -> true + # Does the content already have a .quote-inline span? + content =~ "" -> true + # No inline quote found + true -> false + end + end + defp filter_object(%{"quoteUrl" => quote_url} = object) do content = object["content"] || "" - if content =~ quote_url do + if has_inline_quote?(content, quote_url) do object else prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix]) -- cgit v1.2.3 From b0a7e795e799d3c8d750ab909657ec7b3d0bfd58 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 10 Jul 2023 17:57:09 -0400 Subject: Unify logic for normalizing quoteUri --- .../article_note_page_validator.ex | 23 +----------- .../activity_pub/object_validators/common_fixes.ex | 21 +++++++++++ lib/pleroma/web/activity_pub/transmogrifier.ex | 42 +++++++--------------- 3 files changed, 34 insertions(+), 52 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 0b435b251..1b5b2e8fb 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -76,27 +76,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do def fix_attachments(data), do: data - defp fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data - - # Fedibird - # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac - defp fix_quote_url(%{"quoteUri" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) - end - - # Old Fedibird (bug) - # https://github.com/fedibird/mastodon/issues/9 - defp fix_quote_url(%{"quoteURL" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) - end - - # Misskey fallback - defp fix_quote_url(%{"_misskey_quote" => quote_url} = data) do - Map.put(data, "quoteUrl", quote_url) - end - - defp fix_quote_url(data), do: data - defp fix(data) do data |> CommonFixes.fix_actor() @@ -105,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do |> fix_tag() |> fix_replies() |> fix_attachments() - |> fix_quote_url() + |> CommonFixes.fix_quote_url() |> Transmogrifier.fix_emoji() |> Transmogrifier.fix_content_map() end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index add46d561..cc2ad9116 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -76,4 +76,25 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do Map.put(data, "to", to) end + + def fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data + + # Fedibird + # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac + def fix_quote_url(%{"quoteUri" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + # Old Fedibird (bug) + # https://github.com/fedibird/mastodon/issues/9 + def fix_quote_url(%{"quoteURL" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + # Misskey fallback + def fix_quote_url(%{"_misskey_quote" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + def fix_quote_url(data), do: data end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 6c6cd712b..86d3ac60f 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -166,45 +166,27 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(object, _options), do: object - def fix_quote_url(object, options \\ []) + def fix_quote_url_and_maybe_fetch(object, options \\ []) do + quote_url = + case Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes.fix_quote_url(object) do + %{"quoteUrl" => quote_url} -> quote_url + _ -> nil + end - def fix_quote_url(%{"quoteUrl" => quote_url} = object, options) - when not is_nil(quote_url) do - with {:ok, quoted_object} <- get_obj_helper(quote_url, options), + with {:quoting?, true} <- {:quoting?, not is_nil(quote_url)}, + {:ok, quoted_object} <- get_obj_helper(quote_url, options), %Activity{} <- Activity.get_create_by_object_ap_id(quoted_object.data["id"]) do Map.put(object, "quoteUrl", quoted_object.data["id"]) else + {:quoting?, _} -> + object + e -> Logger.warn("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") object end end - # Fedibird - # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac - def fix_quote_url(%{"quoteUri" => quote_url} = object, options) do - object - |> Map.put("quoteUrl", quote_url) - |> fix_quote_url(options) - end - - # Old Fedibird (bug) - # https://github.com/fedibird/mastodon/issues/9 - def fix_quote_url(%{"quoteURL" => quote_url} = object, options) do - object - |> Map.put("quoteUrl", quote_url) - |> fix_quote_url(options) - end - - # Misskey fallback - def fix_quote_url(%{"_misskey_quote" => quote_url} = object, options) do - object - |> Map.put("quoteUrl", quote_url) - |> fix_quote_url(options) - end - - def fix_quote_url(object, _options), do: object - defp prepare_in_reply_to(in_reply_to) do cond do is_bitstring(in_reply_to) -> @@ -493,7 +475,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> strip_internal_fields() |> fix_type(fetch_options) |> fix_in_reply_to(fetch_options) - |> fix_quote_url(fetch_options) + |> fix_quote_url_and_maybe_fetch(fetch_options) data = Map.put(data, "object", object) options = Keyword.put(options, :local, false) -- cgit v1.2.3 From 9bcec87aba5ce4de6b61b5a95d6832da9dfa0fd8 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 10 Jul 2023 18:27:23 -0400 Subject: Allow local quote and private self-quote --- lib/pleroma/web/common_api/activity_draft.ex | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index d4875765c..32921aa5a 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -167,9 +167,21 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end + defp can_quote?(_draft, _object, visibility) when visibility in ~w(public unlisted local) do + true + end + + defp can_quote?(draft, object, "private") do + draft.user.ap_id == object.data["actor"] + end + + defp can_quote?(_, _, _) do + false + end + defp quoting_visibility(%{quote_post: %Activity{}} = draft) do with %Object{} = object <- Object.normalize(draft.quote_post, fetch: false), - visibility when visibility in ~w(public unlisted) <- Visibility.get_visibility(object) do + true <- can_quote?(draft, object, Visibility.get_visibility(object)) do draft else _ -> add_error(draft, dgettext("errors", "Cannot quote private message")) -- cgit v1.2.3 From 163e5637335f9454688d3cc83530f82fc640a5b9 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 09:30:43 -0400 Subject: Allow more flexibility in InlineQuotePolicy --- lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index c78675caf..a0eefefc0 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -6,8 +6,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do @moduledoc "Force a quote line into the message content." @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp build_inline_quote(prefix, url) do - "

#{prefix}: #{url}" + defp build_inline_quote(template, url) do + quote_line = String.replace(template, "{url}", "#{url}") + + "

#{quote_line}" end defp has_inline_quote?(content, quote_url) do @@ -27,14 +29,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do if has_inline_quote?(content, quote_url) do object else - prefix = Pleroma.Config.get([:mrf_inline_quote, :prefix]) + template = Pleroma.Config.get([:mrf_inline_quote, :template]) content = if String.ends_with?(content, "

"), do: String.trim_trailing(content, "

") <> - build_inline_quote(prefix, quote_url) <> "

", - else: content <> build_inline_quote(prefix, quote_url) + build_inline_quote(template, quote_url) <> "

", + else: content <> build_inline_quote(template, quote_url) Map.put(object, "content", content) end -- cgit v1.2.3 From e9cd004ba1b904d92b1c07446bbf03dc070cce6a Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 11:09:10 -0400 Subject: Parse object link as quoteUrl --- lib/pleroma/constants.ex | 7 ++++++ .../audio_image_video_validator.ex | 1 + .../activity_pub/object_validators/common_fixes.ex | 27 ++++++++++++++++++++++ .../object_validators/question_validator.ex | 1 + 4 files changed, 36 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 6befc6897..9d764ec25 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -83,4 +83,11 @@ defmodule Pleroma.Constants do ) const(upload_object_types, do: ["Document", "Image"]) + + const(activity_json_mime_types, + do: [ + "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", + "application/activity+json" + ] + ) end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex index 79ff76104..65ac6bb93 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex @@ -99,6 +99,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioImageVideoValidator do data |> CommonFixes.fix_actor() |> CommonFixes.fix_object_defaults() + |> CommonFixes.fix_quote_url() |> Transmogrifier.fix_emoji() |> fix_url() |> fix_content() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index cc2ad9116..65b8d9a2c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils + require Pleroma.Constants + def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback) @@ -96,5 +98,30 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do Map.put(data, "quoteUrl", quote_url) end + def fix_quote_url(%{"tag" => [_ | _] = tags} = data) do + tag = Enum.find(tags, &is_object_link_tag/1) + + if not is_nil(tag) do + data + |> Map.put("quoteUrl", tag["href"]) + else + data + end + end + def fix_quote_url(data), do: data + + # https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md + defp is_object_link_tag( + %{ + "type" => "Link", + "mediaType" => media_type, + "href" => href + } = tag + ) + when media_type in Pleroma.Constants.activity_json_mime_types() and is_binary(href) do + true + end + + defp is_object_link_tag(_), do: false end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index ce3305142..621085e6c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -62,6 +62,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do data |> CommonFixes.fix_actor() |> CommonFixes.fix_object_defaults() + |> CommonFixes.fix_quote_url() |> Transmogrifier.fix_emoji() |> fix_closed() end -- cgit v1.2.3 From 479a6f11dbbba0c945c08883956ffab198f91688 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 14:08:24 -0400 Subject: Keep incoming Link tag --- .../web/activity_pub/object_validators/tag_validator.ex | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index cfd510c19..47cf7b415 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -9,15 +9,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do import Ecto.Changeset + require Pleroma.Constants + @primary_key false embedded_schema do # Common field(:type, :string) field(:name, :string) - # Mention, Hashtag + # Mention, Hashtag, Link field(:href, ObjectValidators.Uri) + # Link + field(:mediaType, :string) + # Emoji embeds_one :icon, IconObjectValidator, primary_key: false do field(:type, :string) @@ -68,6 +73,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :name, :icon]) end + def changeset(struct, %{"type" => "Link"} = data) do + struct + |> cast(data, [:type, :name, :mediaType, :href]) + |> validate_inclusion(:mediaType, Pleroma.Constants.activity_json_mime_types()) + |> validate_required([:type, :href, :mediaType]) + end + def changeset(struct, %{"type" => _} = data) do struct |> cast(data, []) -- cgit v1.2.3 From e349e92a441840bbbdbf13cacd307e65f85a38ff Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 14:27:29 -0400 Subject: Add mrf to force link tag of quoting posts --- lib/pleroma/constants.ex | 4 ++ .../activity_pub/mrf/quote_to_link_tag_policy.ex | 49 ++++++++++++++++++++++ .../activity_pub/object_validators/common_fixes.ex | 16 ++++--- 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex (limited to 'lib') diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 9d764ec25..ed60bcc37 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -84,6 +84,10 @@ defmodule Pleroma.Constants do const(upload_object_types, do: ["Document", "Image"]) + const(activity_json_canonical_mime_type, + do: "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + ) + const(activity_json_mime_types, do: [ "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex new file mode 100644 index 000000000..f1c573d1b --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do + @moduledoc "Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions)" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + + require Pleroma.Constants + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do + {:ok, Map.put(activity, "object", filter_object(object))} + end + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def filter(object), do: {:ok, object} + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def describe, do: {:ok, %{}} + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def history_awareness, do: :auto + + defp filter_object(%{"quoteUrl" => quote_url} = object) do + tags = object["tag"] || [] + + if Enum.any?(tags, fn tag -> + CommonFixes.is_object_link_tag(tag) and tag["href"] == quote_url + end) do + object + else + object + |> Map.put( + "tag", + tags ++ + [ + %{ + "type" => "Link", + "mediaType" => Pleroma.Constants.activity_json_canonical_mime_type(), + "href" => quote_url + } + ] + ) + end + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 65b8d9a2c..4d9be0bdd 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -112,16 +112,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def fix_quote_url(data), do: data # https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md - defp is_object_link_tag( - %{ - "type" => "Link", - "mediaType" => media_type, - "href" => href - } = tag - ) - when media_type in Pleroma.Constants.activity_json_mime_types() and is_binary(href) do + def is_object_link_tag(%{ + "type" => "Link", + "mediaType" => media_type, + "href" => href + }) + when media_type in Pleroma.Constants.activity_json_mime_types() and is_binary(href) do true end - defp is_object_link_tag(_), do: false + def is_object_link_tag(_), do: false end -- cgit v1.2.3 From 8b98a98dfb4ab3db9b4f2347713d86ed9c87b61b Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 14:37:12 -0400 Subject: Make InlineQuotePolicy history aware --- lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index a0eefefc0..aaa209aa1 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -53,6 +53,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do @impl true def describe, do: {:ok, %{}} + @impl Pleroma.Web.ActivityPub.MRF.Policy + def history_awareness, do: :auto + @impl true def config_description do %{ -- cgit v1.2.3 From 87353e5ad12799d12507253fe9a0363fd9f0c817 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 22:07:16 -0400 Subject: Fix config descriptions for mrf inline quote --- lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index aaa209aa1..171b22c5e 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -61,14 +61,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do %{ key: :mrf_inline_quote, related_policy: "Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy", - label: "MRF Inline Quote", - description: "Force quote post URLs inline", + label: "MRF Inline Quote Policy", + type: :group, + description: "Force quote url to appear in post content.", children: [ %{ - key: :prefix, + key: :template, type: :string, - description: "Prefix before the link", - suggestions: ["RT", "QT", "RE", "RN"] + description: + "The template to append to the post. `{url}` will be replaced with the actual link to the quoted post.", + suggestions: ["RT: {url}"] } ] } -- cgit v1.2.3 From 875b46d97d910ffd2c33ac26ed8dfe38f7672f52 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 23:29:23 -0400 Subject: Do not mention original poster when quoting --- lib/pleroma/web/common_api/activity_draft.ex | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 32921aa5a..ca1329284 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -139,9 +139,6 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do case Activity.get_by_id_with_object(id) do - %Activity{actor: actor_ap_id} = activity when not_empty_string(actor_ap_id) -> - %__MODULE__{draft | quote_post: activity, mentions: [actor_ap_id]} - %Activity{} = activity -> %__MODULE__{draft | quote_post: activity} -- cgit v1.2.3 From a8b2f9205d16465a3b11d3802c966db3da908c5d Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 23:47:31 -0400 Subject: Expose quote_id parameter on the api --- lib/pleroma/web/api_spec/schemas/status.ex | 5 +++++ lib/pleroma/web/mastodon_api/views/status_view.ex | 10 ++++++++++ 2 files changed, 15 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 5d0eedb08..07f03134a 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -198,6 +198,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do nullable: true, description: "Quoted status (if any)" }, + quote_id: %Schema{ + nullable: true, + allOf: [FlakeID], + description: "ID of the status being quoted, if any" + }, quote_url: %Schema{ type: :string, format: :uri, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index ba4a8f3eb..3d3039751 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -312,6 +312,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # Here the implicit index of the current content is 0 chrono_order = history_len - 1 + quote_id = get_quote_id(activity) + quote_activity = get_quote(activity, opts) quote_post = @@ -431,6 +433,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do context: object.data["context"], in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, quote: quote_post, + quote_id: quote_id, quote_url: object.data["quoteUrl"], quote_visible: visible_for_user?(quote_activity, opts[:for]), content: %{"text/plain" => content_plaintext}, @@ -689,6 +692,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end + defp get_quote_id(activity) do + case get_quote(activity, %{}) do + %Activity{id: id} -> id + _ -> nil + end + end + def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do url = object.data["url"] || object.data["id"] -- cgit v1.2.3 From 08608afca5566f712acdc14b7c43976d6d071106 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 12 Jul 2023 23:56:54 -0400 Subject: Fix quote_visible attribute --- lib/pleroma/web/mastodon_api/views/status_view.ex | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 3d3039751..d070262cc 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -312,12 +312,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # Here the implicit index of the current content is 0 chrono_order = history_len - 1 - quote_id = get_quote_id(activity) - quote_activity = get_quote(activity, opts) + quote_id = + case quote_activity do + %Activity{id: id} -> id + _ -> nil + end + quote_post = - if visible_for_user?(quote_activity, opts[:for]) do + if visible_for_user?(quote_activity, opts[:for]) and opts[:show_quote] != false do quote_rendering_opts = Map.merge(opts, %{activity: quote_activity, show_quote: false}) render("show.json", quote_rendering_opts) else @@ -671,8 +675,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end - def get_quote(_activity, %{show_quote: false}), do: nil - def get_quote(activity, %{quoted_activities: quoted_activities}) do object = Object.normalize(activity, fetch: false) @@ -692,13 +694,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end - defp get_quote_id(activity) do - case get_quote(activity, %{}) do - %Activity{id: id} -> id - _ -> nil - end - end - def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do url = object.data["url"] || object.data["id"] -- cgit v1.2.3 From 2f6fc6a7ab0e757abfe9ec535842a12b887d2fe6 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 24 Sep 2023 22:52:41 +0200 Subject: TwitterAPI: Return proper error when healthcheck is disabled --- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index d5a24ae6c..ca8a98960 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -345,13 +345,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def healthcheck(conn, _params) do - with true <- Config.get([:instance, :healthcheck]), + with {:cfg, true} <- {:cfg, Config.get([:instance, :healthcheck])}, %{healthy: true} = info <- Healthcheck.system_info() do json(conn, info) else %{healthy: false} = info -> service_unavailable(conn, info) + {:cfg, false} -> + service_unavailable(conn, %{"error" => "Healthcheck disabled"}) + _ -> service_unavailable(conn, %{}) end -- cgit v1.2.3 From 2b5636bf127b1987736c281d346a5771c8168c09 Mon Sep 17 00:00:00 2001 From: tusooa Date: Fri, 31 Mar 2023 21:47:37 -0400 Subject: Allow unified streaming endpoint --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 17 ++++++++++++----- lib/pleroma/web/streamer.ex | 8 ++++++-- 2 files changed, 18 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 88444106d..97a1f1401 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -32,8 +32,15 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do req end + topics = + if topic do + [topic] + else + [] + end + {:cowboy_websocket, req, - %{user: user, topic: topic, oauth_token: oauth_token, count: 0, timer: nil}, + %{user: user, topics: topics, oauth_token: oauth_token, count: 0, timer: nil}, %{idle_timeout: @timeout}} else {:error, :bad_topic} -> @@ -50,10 +57,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def websocket_init(state) do Logger.debug( - "#{__MODULE__} accepted websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topic #{state.topic}" + "#{__MODULE__} accepted websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics}" ) - Streamer.add_socket(state.topic, state.oauth_token) + Enum.each(state.topics, fn topic -> Streamer.add_socket(topic, state.oauth_token) end) {:ok, %{state | timer: timer()}} end @@ -109,10 +116,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def terminate(reason, _req, state) do Logger.debug( - "#{__MODULE__} terminating websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topic #{state.topic || "?"}: #{inspect(reason)}" + "#{__MODULE__} terminating websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics || "?"}: #{inspect(reason)}" ) - Streamer.remove_socket(state.topic) + Enum.each(state.topics, fn topic -> Streamer.remove_socket(topic) end) :ok end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index b9a04cc76..6b5fdbf0c 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -59,10 +59,14 @@ defmodule Pleroma.Web.Streamer do end @doc "Expand and authorizes a stream" - @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) :: - {:ok, topic :: String.t()} | {:error, :bad_topic} + @spec get_topic(stream :: String.t() | nil, User.t() | nil, Token.t() | nil, Map.t()) :: + {:ok, topic :: String.t() | nil} | {:error, :bad_topic} def get_topic(stream, user, oauth_token, params \\ %{}) + def get_topic(nil = _stream, _user, _oauth_token, _params) do + {:ok, nil} + end + # Allow all public steams if the instance allows unauthenticated access. # Otherwise, only allow users with valid oauth tokens. def get_topic(stream, user, oauth_token, _params) when stream in @public_streams do -- cgit v1.2.3 From 273cda63ad79b61f4d37e4b7603694908e894e4f Mon Sep 17 00:00:00 2001 From: tusooa Date: Fri, 31 Mar 2023 22:55:52 -0400 Subject: Allow subscribing to streams --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 74 +++++++++++++++++++++++ lib/pleroma/web/views/streamer_view.ex | 18 ++++++ 2 files changed, 92 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 97a1f1401..a42a9a63c 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.User alias Pleroma.Web.OAuth.Token alias Pleroma.Web.Streamer + alias Pleroma.Web.StreamerView @behaviour :cowboy_websocket @@ -73,6 +74,16 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do # We only receive pings for now def websocket_handle(:ping, state), do: {:ok, state} + def websocket_handle({:text, text}, state) do + with {:ok, %{} = event} <- Jason.decode(text) do + handle_client_event(event, state) + else + _ -> + Logger.error("#{__MODULE__} received non-JSON event: #{inspect(text)}") + {:ok, state} + end + end + def websocket_handle(frame, state) do Logger.error("#{__MODULE__} received frame: #{inspect(frame)}") {:ok, state} @@ -144,4 +155,67 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do defp timer do Process.send_after(self(), :tick, @tick) end + + defp handle_client_event(%{"type" => "subscribe", "stream" => _topic} = params, state) do + with {_, {:ok, topic}} <- + {:topic, Streamer.get_topic(params["stream"], state.user, state.oauth_token, params)}, + {_, false} <- {:subscribed, topic in state.topics} do + Streamer.add_socket(topic, state.oauth_token) + + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "success"})} + ], %{state | topics: [topic | state.topics]}} + else + {:subscribed, true} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "ignored"})} + ], state} + + {:topic, {:error, error}} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "subscribe", + result: "error", + error: error + })} + ], state} + end + end + + defp handle_client_event(%{"type" => "unsubscribe", "stream" => _topic} = params, state) do + with {_, {:ok, topic}} <- + {:topic, Streamer.get_topic(params["stream"], state.user, state.oauth_token, params)}, + {_, true} <- {:subscribed, topic in state.topics} do + Streamer.remove_socket(topic) + + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "success"})} + ], %{state | topics: List.delete(state.topics, topic)}} + else + {:subscribed, false} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "ignored"})} + ], state} + + {:topic, {:error, error}} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "unsubscribe", + result: "error", + error: error + })} + ], state} + end + end + + defp handle_client_event(params, state) do + Logger.error("#{__MODULE__} received unknown event: #{inspect(params)}") + {[], state} + end end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 6a55242b0..19f098783 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -135,4 +135,22 @@ defmodule Pleroma.Web.StreamerView do } |> Jason.encode!() end + + def render("pleroma_respond.json", %{type: type, result: result} = params) do + %{ + event: "pleroma.respond", + payload: + %{ + result: result, + type: type + } + |> Map.merge(maybe_error(params)) + |> Jason.encode!() + } + |> Jason.encode!() + end + + defp maybe_error(%{error: :bad_topic}), do: %{error: "bad_topic"} + defp maybe_error(%{error: :unauthorized}), do: %{error: "unauthorized"} + defp maybe_error(_), do: %{} end -- cgit v1.2.3 From 21395aa5090f2a53bdbe0ef5fac46693d16025ed Mon Sep 17 00:00:00 2001 From: tusooa Date: Fri, 31 Mar 2023 23:19:57 -0400 Subject: Allow authenticating via client-sent events --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 36 +++++++++++++++++++++++ lib/pleroma/web/views/streamer_view.ex | 1 + 2 files changed, 37 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index a42a9a63c..6233c3340 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -214,6 +214,42 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end end + defp handle_client_event( + %{"type" => "pleroma.authenticate", "token" => access_token} = _params, + state + ) do + with {:auth, nil, nil} <- {:auth, state.user, state.oauth_token}, + {:ok, user, oauth_token} <- authenticate_request(access_token, nil) do + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma.authenticate", + result: "success" + })} + ], %{state | user: user, oauth_token: oauth_token}} + else + {:auth, _, _} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma.authenticate", + result: "error", + error: :already_authenticated + })} + ], state} + + _ -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma.authenticate", + result: "error", + error: :unauthorized + })} + ], state} + end + end + defp handle_client_event(params, state) do Logger.error("#{__MODULE__} received unknown event: #{inspect(params)}") {[], state} diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 19f098783..0cdcb1918 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -152,5 +152,6 @@ defmodule Pleroma.Web.StreamerView do defp maybe_error(%{error: :bad_topic}), do: %{error: "bad_topic"} defp maybe_error(%{error: :unauthorized}), do: %{error: "unauthorized"} + defp maybe_error(%{error: :already_authenticated}), do: %{error: "already_authenticated"} defp maybe_error(_), do: %{} end -- cgit v1.2.3 From 7d005e8c93b22dc3d7be1a66dd2d404b7f54306a Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 01:25:13 -0400 Subject: Return stream attribute in server-sent events --- lib/pleroma/constants.ex | 4 +++ lib/pleroma/web/mastodon_api/websocket_handler.ex | 4 +-- lib/pleroma/web/streamer.ex | 27 +++++++++------ lib/pleroma/web/views/streamer_view.ex | 42 ++++++++++++++++++----- 4 files changed, 57 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index ed60bcc37..77bc4bfac 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -94,4 +94,8 @@ defmodule Pleroma.Constants do "application/activity+json" ] ) + + const(public_streams, + do: ["public", "public:local", "public:media", "public:local:media"] + ) end diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 6233c3340..2707673ba 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -89,11 +89,11 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {:ok, state} end - def websocket_info({:render_with_user, view, template, item}, state) do + def websocket_info({:render_with_user, view, template, item, topic}, state) do user = %User{} = User.get_cached_by_ap_id(state.user.ap_id) unless Streamer.filtered_by_user?(user, item) do - websocket_info({:text, view.render(template, item, user)}, %{state | user: user}) + websocket_info({:text, view.render(template, item, user, topic)}, %{state | user: user}) else {:ok, state} end diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 6b5fdbf0c..70e46617a 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Streamer do require Logger + require Pleroma.Constants alias Pleroma.Activity alias Pleroma.Chat.MessageReference @@ -24,7 +25,7 @@ defmodule Pleroma.Web.Streamer do def registry, do: @registry - @public_streams ["public", "public:local", "public:media", "public:local:media"] + @public_streams Pleroma.Constants.public_streams() @local_streams ["public:local", "public:local:media"] @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"] @@ -223,8 +224,8 @@ defmodule Pleroma.Web.Streamer do end defp do_stream("follow_relationship", item) do - text = StreamerView.render("follow_relationships_update.json", item) user_topic = "user:#{item.follower.id}" + text = StreamerView.render("follow_relationships_update.json", item, user_topic) Logger.debug("Trying to push follow relationship update to #{user_topic}\n\n") @@ -270,9 +271,11 @@ defmodule Pleroma.Web.Streamer do defp do_stream(topic, %Notification{} = item) when topic in ["user", "user:notification"] do - Registry.dispatch(@registry, "#{topic}:#{item.user_id}", fn list -> + user_topic = "#{topic}:#{item.user_id}" + + Registry.dispatch(@registry, user_topic, fn list -> Enum.each(list, fn {pid, _auth} -> - send(pid, {:render_with_user, StreamerView, "notification.json", item}) + send(pid, {:render_with_user, StreamerView, "notification.json", item, user_topic}) end) end) end @@ -281,7 +284,7 @@ defmodule Pleroma.Web.Streamer do when topic in ["user", "user:pleroma_chat"] do topic = "#{topic}:#{user.id}" - text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}) + text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, _auth} -> @@ -309,7 +312,7 @@ defmodule Pleroma.Web.Streamer do end defp push_to_socket(topic, %Participation{} = participation) do - rendered = StreamerView.render("conversation.json", participation) + rendered = StreamerView.render("conversation.json", participation, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, _} -> @@ -337,12 +340,15 @@ defmodule Pleroma.Web.Streamer do Pleroma.Activity.get_create_by_object_ap_id(item.object.data["id"]) |> Map.put(:object, item.object) - anon_render = StreamerView.render("status_update.json", create_activity) + anon_render = StreamerView.render("status_update.json", create_activity, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, auth?} -> if auth? do - send(pid, {:render_with_user, StreamerView, "status_update.json", create_activity}) + send( + pid, + {:render_with_user, StreamerView, "status_update.json", create_activity, topic} + ) else send(pid, {:text, anon_render}) end @@ -351,12 +357,13 @@ defmodule Pleroma.Web.Streamer do end defp push_to_socket(topic, item) do - anon_render = StreamerView.render("update.json", item) + Logger.debug("topic=#{topic}") + anon_render = StreamerView.render("update.json", item, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, auth?} -> if auth? do - send(pid, {:render_with_user, StreamerView, "update.json", item}) + send(pid, {:render_with_user, StreamerView, "update.json", item, topic}) else send(pid, {:text, anon_render}) end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 0cdcb1918..f591da9a6 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -11,8 +11,11 @@ defmodule Pleroma.Web.StreamerView do alias Pleroma.User alias Pleroma.Web.MastodonAPI.NotificationView - def render("update.json", %Activity{} = activity, %User{} = user) do + require Pleroma.Constants + + def render("update.json", %Activity{} = activity, %User{} = user, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -25,8 +28,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("status_update.json", %Activity{} = activity, %User{} = user) do + def render("status_update.json", %Activity{} = activity, %User{} = user, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "status.update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -39,8 +43,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("notification.json", %Notification{} = notify, %User{} = user) do + def render("notification.json", %Notification{} = notify, %User{} = user, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "notification", payload: NotificationView.render( @@ -52,8 +57,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("update.json", %Activity{} = activity) do + def render("update.json", %Activity{} = activity, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -65,8 +71,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("status_update.json", %Activity{} = activity) do + def render("status_update.json", %Activity{} = activity, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "status.update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -78,7 +85,7 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("chat_update.json", %{chat_message_reference: cm_ref}) do + def render("chat_update.json", %{chat_message_reference: cm_ref}, topic) do # Explicitly giving the cmr for the object here, so we don't accidentally # send a later 'last_message' that was inserted between inserting this and # streaming it out @@ -93,6 +100,7 @@ defmodule Pleroma.Web.StreamerView do ) %{ + stream: render("stream.json", %{topic: topic}), event: "pleroma:chat_update", payload: representation @@ -101,8 +109,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("follow_relationships_update.json", item) do + def render("follow_relationships_update.json", item, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "pleroma:follow_relationships_update", payload: %{ @@ -123,8 +132,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("conversation.json", %Participation{} = participation) do + def render("conversation.json", %Participation{} = participation, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "conversation", payload: Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ @@ -150,6 +160,22 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end + def render("stream.json", %{topic: "user:pleroma_chat:" <> _}), do: ["user:pleroma_chat"] + def render("stream.json", %{topic: "user:notification:" <> _}), do: ["user:notification"] + def render("stream.json", %{topic: "user:" <> _}), do: ["user"] + def render("stream.json", %{topic: "direct:" <> _}), do: ["direct"] + def render("stream.json", %{topic: "list:" <> id}), do: ["list", id] + def render("stream.json", %{topic: "hashtag:" <> tag}), do: ["hashtag", tag] + + def render("stream.json", %{topic: "public:remote:media:" <> instance}), + do: ["public:remote:media", instance] + + def render("stream.json", %{topic: "public:remote:" <> instance}), + do: ["public:remote", instance] + + def render("stream.json", %{topic: stream}) when stream in Pleroma.Constants.public_streams(), + do: [stream] + defp maybe_error(%{error: :bad_topic}), do: %{error: "bad_topic"} defp maybe_error(%{error: :unauthorized}), do: %{error: "unauthorized"} defp maybe_error(%{error: :already_authenticated}), do: %{error: "already_authenticated"} -- cgit v1.2.3 From a348c2e4dd0551a603509501d718d9e0b995be90 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 01:29:11 -0400 Subject: Use pleroma: instead of pleroma. for ws events --- lib/pleroma/web/mastodon_api/websocket_handler.ex | 8 ++++---- lib/pleroma/web/views/streamer_view.ex | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 2707673ba..07c2b62e3 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -215,7 +215,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end defp handle_client_event( - %{"type" => "pleroma.authenticate", "token" => access_token} = _params, + %{"type" => "pleroma:authenticate", "token" => access_token} = _params, state ) do with {:auth, nil, nil} <- {:auth, state.user, state.oauth_token}, @@ -223,7 +223,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {[ {:text, StreamerView.render("pleroma_respond.json", %{ - type: "pleroma.authenticate", + type: "pleroma:authenticate", result: "success" })} ], %{state | user: user, oauth_token: oauth_token}} @@ -232,7 +232,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {[ {:text, StreamerView.render("pleroma_respond.json", %{ - type: "pleroma.authenticate", + type: "pleroma:authenticate", result: "error", error: :already_authenticated })} @@ -242,7 +242,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do {[ {:text, StreamerView.render("pleroma_respond.json", %{ - type: "pleroma.authenticate", + type: "pleroma:authenticate", result: "error", error: :unauthorized })} diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index f591da9a6..f97570b0a 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -148,7 +148,7 @@ defmodule Pleroma.Web.StreamerView do def render("pleroma_respond.json", %{type: type, result: result} = params) do %{ - event: "pleroma.respond", + event: "pleroma:respond", payload: %{ result: result, -- cgit v1.2.3 From eebc605bc25deead55c305d703c06ddb9d9b1107 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 08:27:43 -0400 Subject: Clear up debug statement --- lib/pleroma/web/streamer.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 70e46617a..48ca82421 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -357,7 +357,6 @@ defmodule Pleroma.Web.Streamer do end defp push_to_socket(topic, item) do - Logger.debug("topic=#{topic}") anon_render = StreamerView.render("update.json", item, topic) Registry.dispatch(@registry, topic, fn list -> -- cgit v1.2.3 From 844d1a14e0b4aabbb61a6693fa6dd3a0aa0dbc5b Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 15:31:56 -0400 Subject: Start writing api docs for streaming endpoint --- lib/pleroma/web/api_spec.ex | 10 +- .../web/api_spec/operations/streaming_operation.ex | 186 +++++++++++++++++++++ 2 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 lib/pleroma/web/api_spec/operations/streaming_operation.ex (limited to 'lib') diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 2d56dc643..163226ce5 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -10,6 +10,14 @@ defmodule Pleroma.Web.ApiSpec do @behaviour OpenApi + defp streaming_paths do + %{ + "/api/v1/streaming" => %OpenApiSpex.PathItem{ + get: Pleroma.Web.ApiSpec.StreamingOperation.streaming_operation() + } + } + end + @impl OpenApi def spec(opts \\ []) do %OpenApi{ @@ -45,7 +53,7 @@ defmodule Pleroma.Web.ApiSpec do } }, # populate the paths from a phoenix router - paths: OpenApiSpex.Paths.from_router(Router), + paths: Map.merge(streaming_paths(), OpenApiSpex.Paths.from_router(Router)), components: %OpenApiSpex.Components{ parameters: %{ "accountIdOrNickname" => diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex new file mode 100644 index 000000000..1ef7d72ef --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -0,0 +1,186 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.StreamingOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Response + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Helpers + alias Pleroma.Web.ApiSpec.Schemas.Status + + @spec open_api_operation(atom) :: Operation.t() + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + @spec streaming_operation() :: Operation.t() + def streaming_operation do + %Operation{ + tags: ["Timelines"], + summary: "Establish streaming connection", + description: "Receive statuses in real-time via WebSocket.", + security: [%{"oAuth" => ["read:statuses", "read:notifications"]}], + parameters: [ + Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header", + required: true + ), + Operation.parameter(:upgrade, :header, %Schema{type: :string}, "upgrade header", + required: true + ), + Operation.parameter( + :"sec-websocket-key", + :header, + %Schema{type: :string}, + "sec-websocket-key header", + required: true + ), + Operation.parameter( + :"sec-websocket-version", + :header, + %Schema{type: :string}, + "sec-websocket-version header", + required: true + ) + ], + responses: %{ + 101 => switching_protocols_response(), + 200 => + Operation.response( + "Server-sent events", + "application/json", + server_sent_events() + ) + } + } + end + + defp switching_protocols_response do + %Response{ + description: "Switching protocols", + headers: %{ + "connection" => %OpenApiSpex.Header{required: true}, + "upgrade" => %OpenApiSpex.Header{required: true}, + "sec-websocket-accept" => %OpenApiSpex.Header{required: true} + } + } + end + + defp server_sent_events do + %Schema{ + oneOf: [ + update_event(), + status_update_event(), + pleroma_respond_event() + ] + } + end + + defp stream do + %Schema{ + type: :array, + title: "Stream", + description: """ + The stream identifier. + The first item is the name of the stream. If the stream needs a differentiator, the second item will be the corresponding identifier. + Currently, for the following stream types, there is a second element in the array: + + - `list`: The second element is the id of the list, as a string. + - `hashtag`: The second element is the name of the hashtag. + - `public:remote:media` and `public:remote`: The second element is the domain of the corresponding instance. + """, + maxItems: 2, + minItems: 1, + items: %Schema{type: :string}, + example: ["hashtag", "mew"] + } + end + + defp get_schema(%Schema{} = schema), do: schema + defp get_schema(schema), do: schema.schema + + defp server_sent_event_helper(name, description, type, payload, opts \\ []) do + payload_type = opts[:payload_type] || :json + has_stream = opts[:has_stream] || true + + stream_properties = + if has_stream do + %{stream: stream()} + else + %{} + end + + stream_example = if has_stream, do: %{"stream" => get_schema(stream()).example}, else: %{} + + stream_required = if has_stream, do: [:stream], else: [] + + %Schema{ + type: :object, + title: name, + description: description, + required: [:event, :payload] ++ stream_required, + properties: + %{ + event: %Schema{ + title: "Event type", + description: "Type of the event.", + type: :string, + required: true, + enum: [type] + }, + payload: + if payload_type == :json do + %Schema{ + title: "Event payload", + description: "JSON-encoded string of #{get_schema(payload).title}", + allOf: [payload] + } + else + payload + end + } + |> Map.merge(stream_properties), + example: + %{ + "event" => type, + "payload" => get_schema(payload).example |> Jason.encode!() + } + |> Map.merge(stream_example) + } + end + + defp update_event do + server_sent_event_helper("New status", "A newly-posted status.", "update", Status) + end + + defp status_update_event do + server_sent_event_helper("Edit", "A status that was just edited", "status.update", Status) + end + + defp pleroma_respond_event do + server_sent_event_helper( + "Server response", + "A response to a client-sent event.", + "pleroma:respond", + %Schema{ + type: :object, + title: "Results", + required: [:result], + properties: %{ + result: %Schema{ + type: :string, + title: "Result of the request", + enum: ["success", "error", "ignored"] + }, + error: %Schema{ + type: :string, + title: "Error code", + description: "An error identifier. Only appears if `result` is `error`." + } + }, + example: %{"result" => "success"} + } + ) + end +end -- cgit v1.2.3 From dcef33f5f0c93883a634d12dc662b83d7ef6abfa Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 16:07:27 -0400 Subject: Document server-sent events of streaming --- .../web/api_spec/operations/streaming_operation.ex | 106 +++++++++++++++++++++ 1 file changed, 106 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex index 1ef7d72ef..cdef41603 100644 --- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -7,6 +7,10 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do alias OpenApiSpex.Response alias OpenApiSpex.Schema alias Pleroma.Web.ApiSpec.Helpers + alias Pleroma.Web.ApiSpec.NotificationOperation + alias Pleroma.Web.ApiSpec.Schemas.Chat + alias Pleroma.Web.ApiSpec.Schemas.Conversation + alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.Status @spec open_api_operation(atom) :: Operation.t() @@ -22,6 +26,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do summary: "Establish streaming connection", description: "Receive statuses in real-time via WebSocket.", security: [%{"oAuth" => ["read:statuses", "read:notifications"]}], + operationId: "WebsocketHandler.streaming", parameters: [ Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header", required: true @@ -72,6 +77,11 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do oneOf: [ update_event(), status_update_event(), + notification_event(), + chat_update_event(), + follow_relationships_update_event(), + conversation_event(), + delete_event(), pleroma_respond_event() ] } @@ -158,6 +168,102 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do server_sent_event_helper("Edit", "A status that was just edited", "status.update", Status) end + defp notification_event do + server_sent_event_helper( + "Notification", + "A new notification.", + "notification", + NotificationOperation.notification() + ) + end + + defp follow_relationships_update_event do + server_sent_event_helper( + "Follow relationships update", + "An update to follow relationships.", + "pleroma:follow_relationships_update", + %Schema{ + type: :object, + title: "Follow relationships update", + required: [:state, :follower, :following], + properties: %{ + state: %Schema{ + type: :string, + description: "Follow state of the relationship.", + enum: ["follow_pending", "follow_accept", "follow_reject", "unfollow"] + }, + follower: %Schema{ + type: :object, + description: "Information about the follower.", + required: [:id, :follower_count, :following_count], + properties: %{ + id: FlakeID, + follower_count: %Schema{type: :integer}, + following_count: %Schema{type: :integer} + } + }, + following: %Schema{ + type: :object, + description: "Information about the following person.", + required: [:id, :follower_count, :following_count], + properties: %{ + id: FlakeID, + follower_count: %Schema{type: :integer}, + following_count: %Schema{type: :integer} + } + } + }, + example: %{ + "state" => "follow_pending", + "follower" => %{ + "id" => "someUser1", + "follower_count" => 1, + "following_count" => 1 + }, + "following" => %{ + "id" => "someUser2", + "follower_count" => 1, + "following_count" => 1 + } + } + } + ) + end + + defp chat_update_event do + server_sent_event_helper( + "Chat update", + "A new chat message.", + "pleroma:chat_update", + Chat + ) + end + + defp conversation_event do + server_sent_event_helper( + "Conversation", + "An update about a conversation", + "conversation", + Conversation + ) + end + + defp delete_event do + server_sent_event_helper( + "Delete", + "A status that was just deleted.", + "delete", + %Schema{ + type: :string, + title: "Status id", + description: "Id of the deleted status", + allOf: [FlakeID], + example: "some-opaque-id" + }, + payload_type: :string + ) + end + defp pleroma_respond_event do server_sent_event_helper( "Server response", -- cgit v1.2.3 From 8829dcaee42b3ad1ee50f95b0586b22118771785 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 16:33:22 -0400 Subject: Document client-sent events in streaming --- .../web/api_spec/operations/streaming_operation.ex | 126 ++++++++++++++++++++- 1 file changed, 125 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex index cdef41603..18674c9a1 100644 --- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -6,13 +6,14 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Response alias OpenApiSpex.Schema - alias Pleroma.Web.ApiSpec.Helpers alias Pleroma.Web.ApiSpec.NotificationOperation alias Pleroma.Web.ApiSpec.Schemas.Chat alias Pleroma.Web.ApiSpec.Schemas.Conversation alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.Status + require Pleroma.Constants + @spec open_api_operation(atom) :: Operation.t() def open_api_operation(action) do operation = String.to_existing_atom("#{action}_operation") @@ -49,6 +50,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do required: true ) ], + requestBody: request_body("Client-sent events", client_sent_events()), responses: %{ 101 => switching_protocols_response(), 200 => @@ -289,4 +291,126 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do } ) end + + defp client_sent_events do + %Schema{ + oneOf: [ + subscribe_event(), + unsubscribe_event(), + authenticate_event() + ] + } + end + + defp request_body(description, schema, opts \\ []) do + %OpenApiSpex.RequestBody{ + description: description, + content: %{ + "application/json" => %OpenApiSpex.MediaType{ + schema: schema, + example: opts[:example], + examples: opts[:examples] + } + } + } + end + + defp client_sent_event_helper(name, description, type, properties, opts) do + required = opts[:required] || [] + + %Schema{ + type: :object, + title: name, + required: [:type] ++ required, + description: description, + properties: + %{ + type: %Schema{type: :string, enum: [type], description: "Type of the event."} + } + |> Map.merge(properties), + example: opts[:example] + } + end + + defp subscribe_event do + client_sent_event_helper( + "Subscribe", + "Subscribe to a stream.", + "subscribe", + stream_specifier(), + required: [:stream], + example: %{"type" => "subscribe", "stream" => "list", "list" => "1"} + ) + end + + defp unsubscribe_event do + client_sent_event_helper( + "Unsubscribe", + "Unsubscribe from a stream.", + "subscribe", + stream_specifier(), + required: [:stream], + example: %{ + "type" => "unsubscribe", + "stream" => "public:remote:media", + "instance" => "example.org" + } + ) + end + + defp authenticate_event do + client_sent_event_helper( + "Authenticate", + "Authenticate via an access token.", + "pleroma:authenticate", + %{ + token: %Schema{ + type: :string, + description: "An OAuth access token with corresponding permissions.", + example: "some token" + } + }, + required: [:token] + ) + end + + defp stream_specifier do + %{ + stream: %Schema{ + type: :string, + description: "The name of the stream.", + enum: + Pleroma.Constants.public_streams() ++ + [ + "public:remote", + "public:remote:media", + "user", + "user:pleroma_chat", + "user:notification", + "direct", + "list", + "hashtag" + ] + }, + list: %Schema{ + type: :string, + title: "List id", + description: "The id of the list. Required when `stream` is `list`.", + example: "some-id" + }, + tag: %Schema{ + type: :string, + title: "Hashtag name", + description: "The name of the hashtag. Required when `stream` is `hashtag`.", + example: "mew" + }, + instance: %Schema{ + type: :string, + title: "Domain name", + description: + "Domain name of the instance. Required when `stream` is `public:remote` or `public:remote:media`.", + example: "example.org" + } + } + end end -- cgit v1.2.3 From f393a15dd1217a7f6aec9e9acc7b983e7b165a91 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 16:46:32 -0400 Subject: Fix some specs about server-sent events in streaming --- .../web/api_spec/operations/streaming_operation.ex | 49 ++++++++++++++-------- 1 file changed, 32 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex index 18674c9a1..fa48b9613 100644 --- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -113,8 +113,8 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do defp get_schema(schema), do: schema.schema defp server_sent_event_helper(name, description, type, payload, opts \\ []) do - payload_type = opts[:payload_type] || :json - has_stream = opts[:has_stream] || true + payload_type = Keyword.get(opts, :payload_type, :json) + has_stream = Keyword.get(opts, :has_stream, true) stream_properties = if has_stream do @@ -127,6 +127,24 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do stream_required = if has_stream, do: [:stream], else: [] + payload_schema = + if payload_type == :json do + %Schema{ + title: "Event payload", + description: "JSON-encoded string of #{get_schema(payload).title}", + allOf: [payload] + } + else + payload + end + + payload_example = + if payload_type == :json do + get_schema(payload).example |> Jason.encode!() + else + get_schema(payload).example + end + %Schema{ type: :object, title: name, @@ -141,22 +159,13 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do required: true, enum: [type] }, - payload: - if payload_type == :json do - %Schema{ - title: "Event payload", - description: "JSON-encoded string of #{get_schema(payload).title}", - allOf: [payload] - } - else - payload - end + payload: payload_schema } |> Map.merge(stream_properties), example: %{ "event" => type, - "payload" => get_schema(payload).example |> Jason.encode!() + "payload" => payload_example } |> Map.merge(stream_example) } @@ -262,7 +271,8 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do allOf: [FlakeID], example: "some-opaque-id" }, - payload_type: :string + payload_type: :string, + has_stream: false ) end @@ -274,7 +284,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do %Schema{ type: :object, title: "Results", - required: [:result], + required: [:result, :type], properties: %{ result: %Schema{ type: :string, @@ -285,10 +295,15 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do type: :string, title: "Error code", description: "An error identifier. Only appears if `result` is `error`." + }, + type: %Schema{ + type: :string, + description: "Type of the request." } }, - example: %{"result" => "success"} - } + example: %{"result" => "success", "type" => "pleroma:authenticate"} + }, + has_stream: false ) end -- cgit v1.2.3 From c13f0a8460853d8fe9aa04cb5d57ea06b692a3bf Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 17:00:35 -0400 Subject: Add meta-info and query strings to streaming doc --- .../web/api_spec/operations/streaming_operation.ex | 89 +++++++++++++++------- 1 file changed, 61 insertions(+), 28 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex index fa48b9613..4c4888d8e 100644 --- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -25,31 +25,46 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do %Operation{ tags: ["Timelines"], summary: "Establish streaming connection", - description: "Receive statuses in real-time via WebSocket.", + description: """ + Receive statuses in real-time via WebSocket. + + You can specify the access token on the query string or through the `sec-websocket-protocol` header. Using + the query string to authenticate is considered unsafe and should not be used unless you have to (e.g. to maintain + your client's compatibility with Mastodon). + + You may specify a stream on the query string. If you do so and you are connecting to a stream that requires logged-in users, + you must specify the access token at the time of the connection (i.e. via query string or header). + + Otherwise, you have the option to authenticate after you have established the connection through client-sent events. + + The "Request body" section below describes what events clients can send through WebSocket, and the "Responses" section + describes what events server will send through WebSocket. + """, security: [%{"oAuth" => ["read:statuses", "read:notifications"]}], operationId: "WebsocketHandler.streaming", - parameters: [ - Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header", - required: true - ), - Operation.parameter(:upgrade, :header, %Schema{type: :string}, "upgrade header", - required: true - ), - Operation.parameter( - :"sec-websocket-key", - :header, - %Schema{type: :string}, - "sec-websocket-key header", - required: true - ), - Operation.parameter( - :"sec-websocket-version", - :header, - %Schema{type: :string}, - "sec-websocket-version header", - required: true - ) - ], + parameters: + [ + Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header", + required: true + ), + Operation.parameter(:upgrade, :header, %Schema{type: :string}, "upgrade header", + required: true + ), + Operation.parameter( + :"sec-websocket-key", + :header, + %Schema{type: :string}, + "sec-websocket-key header", + required: true + ), + Operation.parameter( + :"sec-websocket-version", + :header, + %Schema{type: :string}, + "sec-websocket-version header", + required: true + ) + ] ++ stream_params() ++ access_token_params(), requestBody: request_body("Client-sent events", client_sent_events()), responses: %{ 101 => switching_protocols_response(), @@ -63,6 +78,20 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do } end + defp stream_params do + stream_specifier() + |> Enum.map(fn {name, schema} -> + Operation.parameter(name, :query, schema, get_schema(schema).description) + end) + end + + defp access_token_params do + [ + Operation.parameter(:access_token, :query, token(), token().description), + Operation.parameter(:"sec-websocket-protocol", :header, token(), token().description) + ] + end + defp switching_protocols_response do %Response{ description: "Switching protocols", @@ -379,16 +408,20 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do "Authenticate via an access token.", "pleroma:authenticate", %{ - token: %Schema{ - type: :string, - description: "An OAuth access token with corresponding permissions.", - example: "some token" - } + token: token() }, required: [:token] ) end + defp token do + %Schema{ + type: :string, + description: "An OAuth access token with corresponding permissions.", + example: "some token" + } + end + defp stream_specifier do %{ stream: %Schema{ -- cgit v1.2.3 From 32de0683c4069f5877722dce0b4f5a9a6825b7c7 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 17:15:58 -0400 Subject: Fix unsubscribe event type in streaming doc --- lib/pleroma/web/api_spec/operations/streaming_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex index 4c4888d8e..ae3aeb4ab 100644 --- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -391,7 +391,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do client_sent_event_helper( "Unsubscribe", "Unsubscribe from a stream.", - "subscribe", + "unsubscribe", stream_specifier(), required: [:stream], example: %{ -- cgit v1.2.3 From 840dd01035581a37613f695facdd99fbf6ac8319 Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 1 Apr 2023 18:33:43 -0400 Subject: Fix duplicated schema names --- lib/pleroma/web/api_spec/operations/streaming_operation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex index ae3aeb4ab..b580bc2f0 100644 --- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -281,7 +281,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do defp conversation_event do server_sent_event_helper( - "Conversation", + "Conversation update", "An update about a conversation", "conversation", Conversation -- cgit v1.2.3 From b748efe66a099b66300f2beda42f5639911bab4a Mon Sep 17 00:00:00 2001 From: tusooa Date: Sat, 29 Jul 2023 12:55:43 -0400 Subject: Fix mentioning punycode domains when using Markdown --- lib/pleroma/formatter.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index a46c3e381..11d5af2fb 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -124,7 +124,7 @@ defmodule Pleroma.Formatter do end def markdown_to_html(text) do - Earmark.as_html!(text, %Earmark.Options{compact_output: true}) + Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false}) end def html_escape({text, mentions, hashtags}, type) do -- cgit v1.2.3 From 5ff3783d018476bad954881e39f714fac0630436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 22 Nov 2022 23:34:23 +0100 Subject: Use correct domain for fqn and InstanceView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 +- lib/pleroma/web/web_finger.ex | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ce125d608..a0d4aca66 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2252,7 +2252,7 @@ defmodule Pleroma.User do if String.contains?(user.nickname, "@") do user.nickname else - %{host: host} = URI.parse(user.ap_id) + host = Pleroma.Web.WebFinger.domain() user.nickname <> "@" <> host end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 1b01d7371..aef02a418 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do instance = Config.get(:instance) %{ - uri: Pleroma.Web.Endpoint.url(), + uri: Pleroma.Web.WebFinger.domain(), title: Keyword.get(instance, :name), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index f95dc2458..b28fad8d1 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -96,7 +96,7 @@ defmodule Pleroma.Web.WebFinger do |> XmlBuilder.to_doc() end - defp domain do + def domain do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end -- cgit v1.2.3 From 6b8c5e12dfe759ac1286e81e72ad7f8727e01386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 26 Oct 2023 23:30:38 +0200 Subject: Add contact account to InstanceView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 1b01d7371..06df63b0f 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -30,6 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do languages: Keyword.get(instance, :languages, ["en"]), registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), + contact_account: contact_account(Keyword.get(instance, :contact_username)), # Extra (not present in Mastodon): max_toot_chars: Keyword.get(instance, :limit), max_media_attachments: Keyword.get(instance, :max_media_attachments), @@ -141,4 +142,20 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do value_length: Config.get([:instance, :account_field_value_length]) } end + + defp contact_account(nil), do: nil + + defp contact_account("@" <> username) do + contact_account(username) + end + + defp contact_account(username) do + user = Pleroma.User.get_cached_by_nickname(username) + + if user do + Pleroma.Web.MastodonAPI.AccountView.render("show.json", %{user: user, for: nil}) + else + nil + end + end end -- cgit v1.2.3 From c6cedbb8106a16527e48ac8ae03907e1d66c5a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 28 Oct 2023 00:07:18 +0200 Subject: InstanceV2: skip auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/controllers/instance_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 3757a850a..3e664903a 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do plug(Pleroma.Web.ApiSpec.CastAndValidate) - plug(:skip_auth when action in [:show, :peers]) + plug(:skip_auth when action in [:show, :show2, :peers]) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.InstanceOperation -- cgit v1.2.3 From 93370b870a8d16b5dada5ef7c7e3ef2de378395e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 7 Aug 2022 23:56:52 +0200 Subject: Expose nonAnonymous field from Smithereen polls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/activity_pub/object_validators/question_validator.ex | 1 + lib/pleroma/web/api_spec/schemas/poll.ex | 11 ++++++++++- lib/pleroma/web/mastodon_api/views/poll_view.ex | 5 ++++- 3 files changed, 15 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 621085e6c..7f9d4d648 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do field(:closed, ObjectValidators.DateTime) field(:voters, {:array, ObjectValidators.ObjectID}, default: []) + field(:nonAnonymous, :boolean) embeds_many(:anyOf, QuestionOptionsValidator) embeds_many(:oneOf, QuestionOptionsValidator) end diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex index 91570582b..cb2ffdc68 100644 --- a/lib/pleroma/web/api_spec/schemas/poll.ex +++ b/lib/pleroma/web/api_spec/schemas/poll.ex @@ -56,6 +56,12 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do } }, description: "Possible answers for the poll." + }, + pleroma: %Schema{ + type: :object, + properties: %{ + non_anonymous: %Schema{type: :boolean, description: "Is the voters collection public?"} + } } }, example: %{ @@ -79,7 +85,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do votes_count: 4 } ], - emojis: [] + emojis: [], + pleroma: %{ + non_anonymous: false + } } }) end diff --git a/lib/pleroma/web/mastodon_api/views/poll_view.ex b/lib/pleroma/web/mastodon_api/views/poll_view.ex index 34e23873e..1e3c9f36d 100644 --- a/lib/pleroma/web/mastodon_api/views/poll_view.ex +++ b/lib/pleroma/web/mastodon_api/views/poll_view.ex @@ -21,7 +21,10 @@ defmodule Pleroma.Web.MastodonAPI.PollView do votes_count: votes_count, voters_count: voters_count(object), options: options, - emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"]) + emojis: Pleroma.Web.MastodonAPI.StatusView.build_emojis(object.data["emoji"]), + pleroma: %{ + non_anonymous: object.data["nonAnonymous"] || false + } } if params[:for] do -- cgit v1.2.3 From 50e7706b269d6008ae4778db86bc4462ffce5ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 20 Nov 2022 23:19:52 +0100 Subject: Verify link ownership with rel="me" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 63 +++++++++++++++++++++++++++++++- lib/pleroma/workers/background_worker.ex | 5 +++ 2 files changed, 66 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ce125d608..d81aa5252 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -8,6 +8,7 @@ defmodule Pleroma.User do import Ecto.Changeset import Ecto.Query import Ecto, only: [assoc: 2] + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] alias Ecto.Multi alias Pleroma.Activity @@ -595,9 +596,23 @@ defmodule Pleroma.User do defp put_fields(changeset) do if raw_fields = get_change(changeset, :raw_fields) do + old_fields = changeset.data.raw_fields + raw_fields = raw_fields |> Enum.filter(fn %{"name" => n} -> n != "" end) + |> Enum.map(fn field -> + previous = + old_fields + |> Enum.find(fn %{"value" => value} -> field["value"] == value end) + + if previous && Map.has_key?(previous, "verified_at") do + field + |> Map.put("verified_at", previous["verified_at"]) + else + field + end + end) fields = raw_fields @@ -1198,6 +1213,10 @@ defmodule Pleroma.User do def update_and_set_cache(changeset) do with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do + if get_change(changeset, :raw_fields) do + BackgroundWorker.enqueue("verify_fields_links", %{"user_id" => user.id}) + end + set_cache(user) end end @@ -1970,8 +1989,47 @@ defmodule Pleroma.User do maybe_delete_from_db(user) end + def perform(:verify_fields_links, user) do + profile_urls = [user.ap_id] + + fields = + user.raw_fields + |> Enum.map(&verify_field_link(&1, profile_urls)) + + changeset = + user + |> update_changeset(%{raw_fields: fields}) + + with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do + set_cache(user) + end + end + def perform(:set_activation_async, user, status), do: set_activation(user, status) + defp verify_field_link(field, profile_urls) do + verified_at = + with %{"value" => value} <- field, + {:verified_at, nil} <- {:verified_at, Map.get(field, "verified_at")}, + %{scheme: scheme, userinfo: nil, host: host} + when not_empty_string(host) and scheme in ["http", "https"] <- + URI.parse(value), + {:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host}, + attr <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do + if attr == "me" do + CommonUtils.to_masto_date(NaiveDateTime.utc_now()) + end + else + {:verified_at, value} when not_empty_string(value) -> + value + + _ -> + nil + end + + Map.put(field, "verified_at", verified_at) + end + @spec external_users_query() :: Ecto.Query.t() def external_users_query do User.Query.build(%{ @@ -2659,10 +2717,11 @@ defmodule Pleroma.User do # - display name def sanitize_html(%User{} = user, filter) do fields = - Enum.map(user.fields, fn %{"name" => name, "value" => value} -> + Enum.map(user.fields, fn %{"name" => name, "value" => value} = fields -> %{ "name" => name, - "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly) + "value" => HTML.filter_tags(value, Pleroma.HTML.Scrubber.LinksOnly), + "verified_at" => Map.get(fields, "verified_at") } end) diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 794417612..eef1c4f15 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -40,6 +40,11 @@ defmodule Pleroma.Workers.BackgroundWorker do Pleroma.FollowingRelationship.move_following(origin, target) end + def perform(%Job{args: %{"op" => "verify_fields_links", "user_id" => user_id}}) do + user = User.get_by_id(user_id) + User.perform(:verify_fields_links, user) + end + def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do Instance.perform(:delete_instance, host) end -- cgit v1.2.3 From c62696c8e7a28390880a68392bbd14929b66a56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 23 Oct 2023 16:31:29 +0200 Subject: Support /authorize-interaction route used by Mastodon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/router.ex | 2 ++ .../web/twitter_api/controllers/remote_follow_controller.ex | 7 +++++++ 2 files changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..93a28c9ad 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -465,6 +465,8 @@ defmodule Pleroma.Web.Router do get("/main/ostatus", UtilController, :show_subscribe_form) get("/ostatus_subscribe", RemoteFollowController, :follow) post("/ostatus_subscribe", RemoteFollowController, :do_follow) + + get("/authorize_interaction", RemoteFollowController, :authorize_interaction) end scope "/api/pleroma", Pleroma.Web.TwitterAPI do diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index 6229d5d05..178ad2b43 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -121,6 +121,13 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do render(conn, "followed.html", %{error: "Insufficient permissions: follow | write:follows."}) end + # GET /authorize_interaction + # + def authorize_interaction(conn, %{"uri" => uri}) do + conn + |> redirect(to: Routes.remote_follow_path(conn, :follow, %{acct: uri})) + end + defp handle_follow_error(conn, {:mfa_token, followee, _} = _) do render(conn, "follow_login.html", %{error: "Wrong username or password", followee: followee}) end -- cgit v1.2.3 From bf426c53b4e1c025d7857adf485976421175cdf6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 Nov 2023 15:11:14 -0500 Subject: Fix digest email processing, consolidate Oban queues The email related jobs can all share a single Oban queue --- lib/pleroma/workers/cron/digest_emails_worker.ex | 2 +- lib/pleroma/workers/cron/new_users_digest_worker.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 1540c1605..0292bbb3b 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do The worker to send digest emails. """ - use Oban.Worker, queue: "digest_emails" + use Oban.Worker, queue: "mailer" alias Pleroma.Config alias Pleroma.Emails diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 267fe2837..1c3e445aa 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "new_users_digest" + use Pleroma.Workers.WorkerHelper, queue: "mailer" @impl Oban.Worker def perform(_job) do -- cgit v1.2.3 From e23672d82f0a6e61bf4ada65cfca6cfbc647710d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 8 Nov 2023 12:21:34 -0500 Subject: Ensure benchee doesn't run unless we are executing benchmarks --- lib/mix/tasks/pleroma/benchmark.ex | 113 ------------------------------------- 1 file changed, 113 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/benchmark.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex deleted file mode 100644 index f32492169..000000000 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ /dev/null @@ -1,113 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Benchmark do - import Mix.Pleroma - use Mix.Task - - def run(["search"]) do - start_pleroma() - - Benchee.run(%{ - "search" => fn -> - Pleroma.Activity.search(nil, "cofe") - end - }) - end - - def run(["tag"]) do - start_pleroma() - - Benchee.run(%{ - "tag" => fn -> - %{"type" => "Create", "tag" => "cofe"} - |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() - end - }) - end - - def run(["render_timeline", nickname | _] = args) do - start_pleroma() - user = Pleroma.User.get_by_nickname(nickname) - - activities = - %{} - |> Map.put("type", ["Create", "Announce"]) - |> Map.put("blocking_user", user) - |> Map.put("muting_user", user) - |> Map.put("user", user) - |> Map.put("limit", 4096) - |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() - |> Enum.reverse() - - inputs = %{ - "1 activity" => Enum.take_random(activities, 1), - "10 activities" => Enum.take_random(activities, 10), - "20 activities" => Enum.take_random(activities, 20), - "40 activities" => Enum.take_random(activities, 40), - "80 activities" => Enum.take_random(activities, 80) - } - - inputs = - if Enum.at(args, 2) == "extended" do - Map.merge(inputs, %{ - "200 activities" => Enum.take_random(activities, 200), - "500 activities" => Enum.take_random(activities, 500), - "2000 activities" => Enum.take_random(activities, 2000), - "4096 activities" => Enum.take_random(activities, 4096) - }) - else - inputs - end - - Benchee.run( - %{ - "Standart rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity - }) - end - }, - inputs: inputs - ) - end - - def run(["adapters"]) do - start_pleroma() - - :ok = - Pleroma.Gun.Conn.open( - "https://httpbin.org/stream-bytes/1500", - :gun_connections - ) - - Process.sleep(1_500) - - Benchee.run( - %{ - "Without conn and without pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - pool: :no_pool, - receive_conn: false - ) - end, - "Without conn and with pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false) - end, - "With reused conn and without pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool) - end, - "With reused conn and with pool" => fn -> - {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") - end - }, - parallel: 10 - ) - end -end -- cgit v1.2.3 From c1402af2934219b6ab5dc40a7d87a8c916554647 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 14:49:15 +0400 Subject: B Getting: Add default implementation, delegate, prepare test support. --- lib/pleroma/config/getting.ex | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/config/getting.ex b/lib/pleroma/config/getting.ex index f9b66bba6..0de4782ea 100644 --- a/lib/pleroma/config/getting.ex +++ b/lib/pleroma/config/getting.ex @@ -5,4 +5,11 @@ defmodule Pleroma.Config.Getting do @callback get(any()) :: any() @callback get(any(), any()) :: any() + + def get(key), do: get(key, nil) + def get(key, default), do: impl().get(key, default) + + def impl() do + Application.get_env(:pleroma, :config_impl, Pleroma.Config) + end end -- cgit v1.2.3 From d3f8950588b444dfdf46d87d5631720cc14a907c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 14:49:50 +0400 Subject: B MeiliSearch, SearchIndexingWorker: Use Config.Getting, make tests async. --- lib/pleroma/search/meilisearch.ex | 32 ++++++++++++++++----------- lib/pleroma/workers/search_indexing_worker.ex | 6 +++-- 2 files changed, 23 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 7af7f460a..eed9fca1c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -3,6 +3,7 @@ defmodule Pleroma.Search.Meilisearch do require Pleroma.Constants alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config import Pleroma.Search.DatabaseSearch import Ecto.Query @@ -10,14 +11,14 @@ defmodule Pleroma.Search.Meilisearch do @behaviour Pleroma.Search.SearchBackend defp meili_headers do - private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) + private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) [{"Content-Type", "application/json"}] ++ if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}] end def meili_get(path) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) result = Pleroma.HTTP.get( @@ -31,7 +32,7 @@ defmodule Pleroma.Search.Meilisearch do end def meili_post(path, params) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) result = Pleroma.HTTP.post( @@ -46,7 +47,7 @@ defmodule Pleroma.Search.Meilisearch do end def meili_put(path, params) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) result = Pleroma.HTTP.request( @@ -63,15 +64,20 @@ defmodule Pleroma.Search.Meilisearch do end def meili_delete(path) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - Pleroma.HTTP.request( - :delete, - Path.join(endpoint, path), - "", - meili_headers(), - [] - ) + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + with {:ok, _} <- + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + [] + ) do + :ok + else + _ -> :error + end end @impl true diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 43b7bad1e..8476a2be5 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -3,10 +3,12 @@ defmodule Pleroma.Workers.SearchIndexingWorker do @impl Oban.Worker + alias Pleroma.Config.Getting, as: Config + def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do activity = Pleroma.Activity.get_by_id_with_object(activity_id) - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + search_module = Config.get([Pleroma.Search, :module]) search_module.add_to_index(activity) end @@ -14,7 +16,7 @@ defmodule Pleroma.Workers.SearchIndexingWorker do def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do object = Pleroma.Object.get_by_id(object_id) - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + search_module = Config.get([Pleroma.Search, :module]) search_module.remove_from_index(object) end -- cgit v1.2.3 From a1a25029da74949a79c73d400b6f2bc0bf1dc01a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 16:19:54 +0400 Subject: B DatabaseSearch: Fix local-only search. --- lib/pleroma/search/database_search.ex | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 9a340abf1..f4c405773 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Search.DatabaseSearch do alias Pleroma.Pagination alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Config require Pleroma.Constants @@ -17,7 +18,7 @@ defmodule Pleroma.Search.DatabaseSearch do @impl true def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + index_type = if Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) @@ -33,7 +34,7 @@ defmodule Pleroma.Search.DatabaseSearch do Activity |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() - |> restrict_public() + |> restrict_public(user) |> query_with(index_type, search_query, search_function) |> maybe_restrict_local(user) |> maybe_restrict_author(author) @@ -49,10 +50,10 @@ defmodule Pleroma.Search.DatabaseSearch do end @impl true - def add_to_index(_activity), do: nil + def add_to_index(_activity), do: :ok @impl true - def remove_from_index(_object), do: nil + def remove_from_index(_object), do: :ok def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) @@ -66,7 +67,19 @@ defmodule Pleroma.Search.DatabaseSearch do def maybe_restrict_blocked(query, _), do: query - def restrict_public(q) do + defp restrict_public(q, user) when not is_nil(user) do + intended_recipients = [ + Pleroma.Constants.as_public(), + Pleroma.Web.ActivityPub.Utils.as_local_public() + ] + + from([a, o] in q, + where: fragment("?->>'type' = 'Create'", a.data), + where: fragment("? && ?", ^intended_recipients, a.recipients) + ) + end + + defp restrict_public(q, _user) do from([a, o] in q, where: fragment("?->>'type' = 'Create'", a.data), where: ^Pleroma.Constants.as_public() in a.recipients @@ -134,7 +147,7 @@ defmodule Pleroma.Search.DatabaseSearch do end def maybe_restrict_local(q, user) do - limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) + limit = Config.get([:instance, :limit_to_local_content], :unauthenticated) case {limit, user} do {:all, _} -> restrict_local(q) -- cgit v1.2.3 From 59018d73c366d9297efe83d290c717d1a3e4756a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 16:43:50 +0400 Subject: B Meilisearch: Update to current API responses. --- lib/pleroma/search/meilisearch.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index eed9fca1c..2bff663e8 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Search.Meilisearch do ) do :ok else - _ -> :error + _ -> {:error, "Could not remove from index"} end end @@ -159,8 +159,7 @@ defmodule Pleroma.Search.Meilisearch do [maybe_search_data] ) - with {:ok, res} <- result, - true <- Map.has_key?(res, "updateId") do + with {:ok, %{"status" => "enqueued"}} <- result do # Added successfully :ok else -- cgit v1.2.3 From 3d62c71edf8782c5ceae5a0ea3ba5ec08dc5b948 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 17:13:27 +0400 Subject: Credo fixes. --- lib/pleroma/config/getting.ex | 2 +- lib/pleroma/search/database_search.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/getting.ex b/lib/pleroma/config/getting.ex index 0de4782ea..ec93fd02a 100644 --- a/lib/pleroma/config/getting.ex +++ b/lib/pleroma/config/getting.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Config.Getting do def get(key), do: get(key, nil) def get(key, default), do: impl().get(key, default) - def impl() do + def impl do Application.get_env(:pleroma, :config_impl, Pleroma.Config) end end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index f4c405773..c6311e0c7 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -4,11 +4,11 @@ defmodule Pleroma.Search.DatabaseSearch do alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object.Fetcher alias Pleroma.Pagination alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Config require Pleroma.Constants -- cgit v1.2.3 From 9a063deacc75d3545dcd7a7eb33263ecbf0361ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 12 Nov 2023 13:38:08 +0000 Subject: Count and display post quotes --- lib/pleroma/object.ex | 46 +++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 21 +++++++ .../object_validators/common_fields.ex | 1 + lib/pleroma/web/activity_pub/side_effects.ex | 8 +++ .../operations/pleroma_status_operation.ex | 45 +++++++++++++++ lib/pleroma/web/api_spec/schemas/status.ex | 7 ++- lib/pleroma/web/mastodon_api/views/status_view.ex | 3 +- .../pleroma_api/controllers/status_controller.ex | 66 ++++++++++++++++++++++ lib/pleroma/web/router.ex | 2 + 9 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/status_controller.ex (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index aa137d250..fa5baf1a4 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -328,6 +328,52 @@ defmodule Pleroma.Object do end end + def increase_quotes_count(ap_id) do + Object + |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id))) + |> update([o], + set: [ + data: + fragment( + """ + safe_jsonb_set(?, '{quotesCount}', + (coalesce((?->>'quotesCount')::int, 0) + 1)::varchar::jsonb, true) + """, + o.data, + o.data + ) + ] + ) + |> Repo.update_all([]) + |> case do + {1, [object]} -> set_cache(object) + _ -> {:error, "Not found"} + end + end + + def decrease_quotes_count(ap_id) do + Object + |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id))) + |> update([o], + set: [ + data: + fragment( + """ + safe_jsonb_set(?, '{quotesCount}', + (greatest(0, (?->>'quotesCount')::int - 1))::varchar::jsonb, true) + """, + o.data, + o.data + ) + ] + ) + |> Repo.update_all([]) + |> case do + {1, [object]} -> set_cache(object) + _ -> {:error, "Not found"} + end + end + def increase_vote_count(ap_id, name, actor) do with %Object{} = object <- Object.normalize(ap_id, fetch: false), "Question" <- object.data["type"] do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3979d418e..a81d33fb6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -96,6 +96,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop + defp increase_quotes_count_if_quote(%{ + "object" => %{"quoteUrl" => quote_ap_id} = object, + "type" => "Create" + }) do + if is_public?(object) do + Object.increase_quotes_count(quote_ap_id) + end + end + + defp increase_quotes_count_if_quote(_create_data), do: :noop + @object_types ~w[ChatMessage Question Answer Audio Video Image Event Article Note Page] @impl true def persist(%{"type" => type} = object, meta) when type in @object_types do @@ -299,6 +310,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do with {:ok, activity} <- insert(create_data, local, fake), {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), + _ <- increase_quotes_count_if_quote(create_data), {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, {:ok, _actor} <- increase_note_count_if_public(actor, activity), {:ok, _actor} <- update_last_status_at_if_public(actor, activity), @@ -1237,6 +1249,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_unauthenticated(query, _), do: query + defp restrict_quote_url(query, %{quote_url: quote_url}) do + from([_activity, object] in query, + where: fragment("(?)->'quoteUrl' = ?", object.data, ^quote_url) + ) + end + + defp restrict_quote_url(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1399,6 +1419,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_instance(opts) |> restrict_announce_object_actor(opts) |> restrict_filtered(opts) + |> restrict_quote_url(opts) |> maybe_restrict_deactivated_users(opts) |> exclude_poll_votes(opts) |> exclude_chat_messages(opts) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 835ed97b7..1a5d02601 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -57,6 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) + field(:quotes_count, :integer, default: 0) field(:inReplyTo, ObjectValidators.ObjectID) field(:quoteUrl, ObjectValidators.ObjectID) field(:url, ObjectValidators.BareUri) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 098c177c7..6a7ac2806 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -209,6 +209,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.increase_replies_count(in_reply_to) end + if quote_url = object.data["quoteUrl"] do + Object.increase_quotes_count(quote_url) + end + reply_depth = (meta[:depth] || 0) + 1 # FIXME: Force inReplyTo to replies @@ -305,6 +309,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.decrease_replies_count(in_reply_to) end + if quote_url = deleted_object.data["quoteUrl"] do + Object.decrease_quotes_count(quote_url) + end + MessageReference.delete_for_object(deleted_object) ap_streamer().stream_out(object) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex new file mode 100644 index 000000000..6e69c5269 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaStatusOperation do + alias OpenApiSpex.Operation + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.StatusOperation + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def quotes_operation do + %Operation{ + tags: ["Retrieve status information"], + summary: "Quoted by", + description: "View quotes for a given status", + operationId: "PleromaAPI.StatusController.quotes", + parameters: [id_param() | pagination_params()], + security: [%{"oAuth" => ["read:statuses"]}], + responses: %{ + 200 => + Operation.response( + "Array of Status", + "application/json", + StatusOperation.array_of_statuses() + ), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def id_param do + Operation.parameter(:id, :path, FlakeID, "Status ID", + example: "9umDrYheeY451cQnEe", + required: true + ) + end +end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 07f03134a..a4052803b 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -213,6 +213,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do type: :boolean, description: "`true` if the quoted post is visible to the user" }, + quotes_count: %Schema{ + type: :integer, + description: "How many statuses quoted this status" + }, local: %Schema{ type: :boolean, description: "`true` if the post was made on the local instance" @@ -367,7 +371,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do "in_reply_to_account_acct" => nil, "local" => true, "spoiler_text" => %{"text/plain" => ""}, - "thread_muted" => false + "thread_muted" => false, + "quotes_count" => 0 }, "poll" => nil, "reblog" => nil, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d070262cc..e3b5760fa 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -447,7 +447,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do thread_muted: thread_muted?, emoji_reactions: emoji_reactions, parent_visible: visible_for_user?(reply_to, opts[:for]), - pinned_at: pinned_at + pinned_at: pinned_at, + quotes_count: object.data["quotesCount"] || 0 } } end diff --git a/lib/pleroma/web/pleroma_api/controllers/status_controller.ex b/lib/pleroma/web/pleroma_api/controllers/status_controller.ex new file mode 100644 index 000000000..482662fdd --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/status_controller.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.StatusController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + + require Ecto.Query + require Pleroma.Constants + + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + + plug( + OAuthScopesPlug, + %{scopes: ["read:statuses"], fallback: :proceed_unauthenticated} when action == :quotes + ) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaStatusOperation + + @doc "GET /api/v1/pleroma/statuses/:id/quotes" + def quotes(%{assigns: %{user: user}} = conn, %{id: id} = params) do + with %Activity{object: object} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + params = + params + |> Map.put(:type, "Create") + |> Map.put(:blocking_user, user) + |> Map.put(:quote_url, object.data["id"]) + + recipients = + if user do + [Pleroma.Constants.as_public()] ++ [user.ap_id | User.following(user)] + else + [Pleroma.Constants.as_public()] + end + + activities = + recipients + |> ActivityPub.fetch_activities(params) + |> Enum.reverse() + + conn + |> add_link_headers(activities) + |> put_view(StatusView) + |> render("index.json", + activities: activities, + for: user, + as: :activity + ) + else + nil -> {:error, :not_found} + false -> {:error, :not_found} + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..d40af499e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -578,6 +578,8 @@ defmodule Pleroma.Web.Router do pipe_through(:api) get("/accounts/:id/favourites", AccountController, :favourites) get("/accounts/:id/endorsements", AccountController, :endorsements) + + get("/statuses/:id/quotes", StatusController, :quotes) end scope [] do -- cgit v1.2.3 From 66cb3294ed942d461cabc32881e2a10ebfd182af Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 2 Nov 2022 22:49:55 -0400 Subject: Switch to PromEx for prometheus metrics Recommending use of the separate HTTP server for exposing the metrics and securing it externally on your firewall or reverse proxy. It will listen on port 4021 by default. --- lib/pleroma/application.ex | 25 +------------------ lib/pleroma/config/transfer_task.ex | 3 +-- lib/pleroma/prom_ex.ex | 49 +++++++++++++++++++++++++++++++++++++ lib/pleroma/repo.ex | 2 -- lib/pleroma/web/endpoint.ex | 41 ------------------------------- 5 files changed, 51 insertions(+), 69 deletions(-) create mode 100644 lib/pleroma/prom_ex.ex (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7bbc132f1..52cd6e9a9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -54,7 +54,6 @@ defmodule Pleroma.Application do Config.DeprecationWarnings.warn() Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.ApplicationRequirements.verify!() - setup_instrumenters() load_custom_modules() Pleroma.Docs.JSON.compile() limiters_setup() @@ -91,6 +90,7 @@ defmodule Pleroma.Application do # Define workers and child supervisors to be supervised children = [ + Pleroma.PromEx, Pleroma.Repo, Config.TransferTask, Pleroma.Emoji, @@ -170,29 +170,6 @@ defmodule Pleroma.Application do end end - defp setup_instrumenters do - require Prometheus.Registry - - if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do - :ok = - :telemetry.attach( - "prometheus-ecto", - [:pleroma, :repo, :query], - &Pleroma.Repo.Instrumenter.handle_event/4, - %{} - ) - - Pleroma.Repo.Instrumenter.setup() - end - - Pleroma.Web.Endpoint.MetricsExporter.setup() - Pleroma.Web.Endpoint.PipelineInstrumenter.setup() - - # Note: disabled until prometheus-phx is integrated into prometheus-phoenix: - # Pleroma.Web.Endpoint.Instrumenter.setup() - PrometheusPhx.setup() - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 44a984019..6fd05b0e0 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -55,8 +55,7 @@ defmodule Pleroma.Config.TransferTask do started_applications = Application.started_applications() - # TODO: some problem with prometheus after restart! - reject = [nil, :prometheus, :postgrex] + reject = [nil, :postgrex] reject = if restart_pleroma? do diff --git a/lib/pleroma/prom_ex.ex b/lib/pleroma/prom_ex.ex new file mode 100644 index 000000000..6608708b7 --- /dev/null +++ b/lib/pleroma/prom_ex.ex @@ -0,0 +1,49 @@ +defmodule Pleroma.PromEx do + use PromEx, otp_app: :pleroma + + alias PromEx.Plugins + + @impl true + def plugins do + [ + # PromEx built in plugins + Plugins.Application, + Plugins.Beam, + {Plugins.Phoenix, router: Pleroma.Web.Router, endpoint: Pleroma.Web.Endpoint}, + Plugins.Ecto, + Plugins.Oban + # Plugins.PhoenixLiveView, + # Plugins.Absinthe, + # Plugins.Broadway, + + # Add your own PromEx metrics plugins + # Pleroma.Users.PromExPlugin + ] + end + + @impl true + def dashboard_assigns do + [ + datasource_id: Pleroma.Config.get([Pleroma.PromEx, :datasource]), + default_selected_interval: "30s" + ] + end + + @impl true + def dashboards do + [ + # PromEx built in Grafana dashboards + {:prom_ex, "application.json"}, + {:prom_ex, "beam.json"}, + {:prom_ex, "phoenix.json"}, + {:prom_ex, "ecto.json"}, + {:prom_ex, "oban.json"} + # {:prom_ex, "phoenix_live_view.json"}, + # {:prom_ex, "absinthe.json"}, + # {:prom_ex, "broadway.json"}, + + # Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"} + # {:pleroma, "/grafana_dashboards/user_metrics.json"} + ] + end +end diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 515b0c1ff..a50a59b3b 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -11,8 +11,6 @@ defmodule Pleroma.Repo do import Ecto.Query require Logger - defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter) - @doc """ Dynamically loads the repository url from the DATABASE_URL environment variable. diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 65dd72c49..307fa069e 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -151,47 +151,6 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Web.Plugs.RemoteIp) - defmodule Instrumenter do - use Prometheus.PhoenixInstrumenter - end - - defmodule PipelineInstrumenter do - use Prometheus.PlugPipelineInstrumenter - end - - defmodule MetricsExporter do - use Prometheus.PlugExporter - end - - defmodule MetricsExporterCaller do - @behaviour Plug - - def init(opts), do: opts - - def call(conn, opts) do - prometheus_config = Application.get_env(:prometheus, MetricsExporter, []) - ip_whitelist = List.wrap(prometheus_config[:ip_whitelist]) - - cond do - !prometheus_config[:enabled] -> - conn - - ip_whitelist != [] and - !Enum.find(ip_whitelist, fn ip -> - Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip} - end) -> - conn - - true -> - MetricsExporter.call(conn, opts) - end - end - end - - plug(PipelineInstrumenter) - - plug(MetricsExporterCaller) - plug(Pleroma.Web.Router) @doc """ -- cgit v1.2.3 From 13baba90f6deb27b0d6301a705db753cc32bb141 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 9 Nov 2022 13:45:57 -0500 Subject: Replace ImageMagick with Vips for Media Preview Proxy --- lib/pleroma/helpers/media_helper.ex | 57 +++++++++---------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index 24c845fcd..07dfea55b 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -8,11 +8,12 @@ defmodule Pleroma.Helpers.MediaHelper do """ alias Pleroma.HTTP + alias Vix.Vips.Operation require Logger def missing_dependencies do - Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> + Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> if Pleroma.Utils.command_available?(executable) do acc else @@ -22,54 +23,22 @@ defmodule Pleroma.Helpers.MediaHelper do end def image_resize(url, options) do - with executable when is_binary(executable) <- System.find_executable("convert"), - {:ok, args} <- prepare_image_resize_args(options), - {:ok, env} <- HTTP.get(url, [], pool: :media), - {:ok, fifo_path} <- mkfifo() do - args = List.flatten([fifo_path, args]) - run_fifo(fifo_path, env, executable, args) + with {:ok, env} <- HTTP.get(url, [], pool: :media), + {:ok, resized} <- + Operation.thumbnail_buffer(env.body, options.max_width, + height: options.max_height, + size: :VIPS_SIZE_DOWN + ) do + if options[:format] == "png" do + Operation.pngsave_buffer(resized, Q: options[:quality]) + else + Operation.jpegsave_buffer(resized, Q: options[:quality], interlace: true) + end else - nil -> {:error, {:convert, :command_not_found}} {:error, _} = error -> error end end - defp prepare_image_resize_args( - %{max_width: max_width, max_height: max_height, format: "png"} = options - ) do - quality = options[:quality] || 85 - resize = Enum.join([max_width, "x", max_height, ">"]) - - args = [ - "-resize", - resize, - "-quality", - to_string(quality), - "png:-" - ] - - {:ok, args} - end - - defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do - quality = options[:quality] || 85 - resize = Enum.join([max_width, "x", max_height, ">"]) - - args = [ - "-interlace", - "Plane", - "-resize", - resize, - "-quality", - to_string(quality), - "jpg:-" - ] - - {:ok, args} - end - - defp prepare_image_resize_args(_), do: {:error, :missing_options} - # Note: video thumbnail is intentionally not resized (always has original dimensions) def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), -- cgit v1.2.3 From 481b6ac0d5c610e840fc4b88a357efa28f510ab8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 10 Nov 2022 11:07:49 -0500 Subject: Add Pleroma.Upload.Filter.HeifToJpeg based on vips --- lib/pleroma/upload/filter/heif_to_jpeg.ex | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 lib/pleroma/upload/filter/heif_to_jpeg.ex (limited to 'lib') diff --git a/lib/pleroma/upload/filter/heif_to_jpeg.ex b/lib/pleroma/upload/filter/heif_to_jpeg.ex new file mode 100644 index 000000000..a2095ba01 --- /dev/null +++ b/lib/pleroma/upload/filter/heif_to_jpeg.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.HeifToJpeg do + @behaviour Pleroma.Upload.Filter + alias Pleroma.Upload + alias Vix.Vips.Operation + + @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} + @type conversions :: conversion() | [conversion()] + + @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} + def filter(%Pleroma.Upload{content_type: "image/avif"} = upload), do: apply_filter(upload) + def filter(%Pleroma.Upload{content_type: "image/heic"} = upload), do: apply_filter(upload) + def filter(%Pleroma.Upload{content_type: "image/heif"} = upload), do: apply_filter(upload) + + def filter(_), do: {:ok, :noop} + + defp apply_filter(%Pleroma.Upload{name: name, path: path, tempfile: tempfile} = upload) do + ext = String.split(path, ".") |> List.last() + + try do + name = name |> String.replace_suffix(ext, "jpg") + path = path |> String.replace_suffix(ext, "jpg") + {:ok, {vixdata, _vixflags}} = Operation.heifload(tempfile) + {:ok, jpegdata} = Operation.jpegsave_buffer(vixdata) + :ok = File.write(tempfile, jpegdata) + + {:ok, :filtered, %Upload{upload | name: name, path: path, content_type: "image/jpeg"}} + rescue + e in ErlangError -> + {:error, "#{__MODULE__}: #{inspect(e)}"} + end + end +end -- cgit v1.2.3 From a4b6e5613fe02fac1a0ec80b8fd6940d2b22f85d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 Nov 2023 22:03:20 +0000 Subject: Revert "Add Pleroma.Upload.Filter.HeifToJpeg based on vips" This reverts commit 31d4448ee61b4afac6aa23f8c0287d13aed411a1. This functionality is not reliably working with vips/vix due to codec patent junk --- lib/pleroma/upload/filter/heif_to_jpeg.ex | 36 ------------------------------- 1 file changed, 36 deletions(-) delete mode 100644 lib/pleroma/upload/filter/heif_to_jpeg.ex (limited to 'lib') diff --git a/lib/pleroma/upload/filter/heif_to_jpeg.ex b/lib/pleroma/upload/filter/heif_to_jpeg.ex deleted file mode 100644 index a2095ba01..000000000 --- a/lib/pleroma/upload/filter/heif_to_jpeg.ex +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Upload.Filter.HeifToJpeg do - @behaviour Pleroma.Upload.Filter - alias Pleroma.Upload - alias Vix.Vips.Operation - - @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} - @type conversions :: conversion() | [conversion()] - - @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} - def filter(%Pleroma.Upload{content_type: "image/avif"} = upload), do: apply_filter(upload) - def filter(%Pleroma.Upload{content_type: "image/heic"} = upload), do: apply_filter(upload) - def filter(%Pleroma.Upload{content_type: "image/heif"} = upload), do: apply_filter(upload) - - def filter(_), do: {:ok, :noop} - - defp apply_filter(%Pleroma.Upload{name: name, path: path, tempfile: tempfile} = upload) do - ext = String.split(path, ".") |> List.last() - - try do - name = name |> String.replace_suffix(ext, "jpg") - path = path |> String.replace_suffix(ext, "jpg") - {:ok, {vixdata, _vixflags}} = Operation.heifload(tempfile) - {:ok, jpegdata} = Operation.jpegsave_buffer(vixdata) - :ok = File.write(tempfile, jpegdata) - - {:ok, :filtered, %Upload{upload | name: name, path: path, content_type: "image/jpeg"}} - rescue - e in ErlangError -> - {:error, "#{__MODULE__}: #{inspect(e)}"} - end - end -end -- cgit v1.2.3 From 66f5ae0c5a0ab57dc6bf3f52bbf976128259800f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 8 Aug 2023 19:08:59 +0200 Subject: router: Make /federation_status publicly available --- lib/pleroma/web/router.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9abad65b0..eb8576b02 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -224,6 +224,12 @@ defmodule Pleroma.Web.Router do post("/remote_interaction", UtilController, :remote_interaction) end + scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do + pipe_through(:pleroma_api) + + get("/federation_status", InstancesController, :show) + end + scope "/api/v1/pleroma", Pleroma.Web do pipe_through(:pleroma_api) post("/uploader_callback/:upload_path", UploaderController, :callback) @@ -604,7 +610,6 @@ defmodule Pleroma.Web.Router do scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do pipe_through(:api) get("/accounts/:id/scrobbles", ScrobbleController, :index) - get("/federation_status", InstancesController, :show) end scope "/api/v2/pleroma", Pleroma.Web.PleromaAPI do -- cgit v1.2.3 From a5aa8ea79603e22541de04c26293dc87bd2f2ed8 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Sun, 29 Oct 2023 18:58:57 +0200 Subject: Add support for configuring a favicon and embed PWA manifest in server-generated-meta --- lib/pleroma/web/fallback/redirect_controller.ex | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 1a86f7a53..005a5da8b 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -18,9 +18,22 @@ defmodule Pleroma.Web.Fallback.RedirectController do end def redirector(conn, _params, code \\ 200) do + {:ok, index_content} = File.read(index_file_path()) + + title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" + + response = + index_content + |> String.replace( + "", + title <> favicon <> manifest + ) + conn |> put_resp_content_type("text/html") - |> send_file(code, index_file_path()) + |> send_resp(code, response) end def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} = params) do @@ -38,10 +51,15 @@ defmodule Pleroma.Web.Fallback.RedirectController do tags = build_tags(conn, params) preloads = preload_data(conn, params) title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" response = index_content - |> String.replace("", tags <> preloads <> title) + |> String.replace( + "", + tags <> preloads <> title <> favicon <> manifest + ) conn |> put_resp_content_type("text/html") @@ -56,10 +74,12 @@ defmodule Pleroma.Web.Fallback.RedirectController do {:ok, index_content} = File.read(index_file_path()) preloads = preload_data(conn, params) title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" response = index_content - |> String.replace("", preloads <> title) + |> String.replace("", preloads <> title <> favicon <> manifest) conn |> put_resp_content_type("text/html") -- cgit v1.2.3 From 5d3e145dc4090dc5709a649ee5d646b637da252e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 14 Nov 2023 11:01:02 +0100 Subject: RedirectController: Unify server-generated-meta insertion code --- lib/pleroma/web/fallback/redirect_controller.ex | 33 +++++++++++-------------- 1 file changed, 14 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 005a5da8b..4a0885fab 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -17,19 +17,24 @@ defmodule Pleroma.Web.Fallback.RedirectController do |> json(%{error: "Not implemented"}) end - def redirector(conn, _params, code \\ 200) do - {:ok, index_content} = File.read(index_file_path()) - + def add_generated_metadata(page_content, extra \\ "") do title = "#{Pleroma.Config.get([:instance, :name])}" favicon = "" manifest = "" + page_content + |> String.replace( + "", + title <> favicon <> manifest <> extra + ) + end + + def redirector(conn, _params, code \\ 200) do + {:ok, index_content} = File.read(index_file_path()) + response = index_content - |> String.replace( - "", - title <> favicon <> manifest - ) + |> add_generated_metadata() conn |> put_resp_content_type("text/html") @@ -47,19 +52,12 @@ defmodule Pleroma.Web.Fallback.RedirectController do def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = build_tags(conn, params) preloads = preload_data(conn, params) - title = "#{Pleroma.Config.get([:instance, :name])}" - favicon = "" - manifest = "" response = index_content - |> String.replace( - "", - tags <> preloads <> title <> favicon <> manifest - ) + |> add_generated_metadata(tags <> preloads) conn |> put_resp_content_type("text/html") @@ -73,13 +71,10 @@ defmodule Pleroma.Web.Fallback.RedirectController do def redirector_with_preload(conn, params) do {:ok, index_content} = File.read(index_file_path()) preloads = preload_data(conn, params) - title = "#{Pleroma.Config.get([:instance, :name])}" - favicon = "" - manifest = "" response = index_content - |> String.replace("", preloads <> title <> favicon <> manifest) + |> add_generated_metadata(preloads) conn |> put_resp_content_type("text/html") -- cgit v1.2.3 From 8ac7cc98c1ba241ce104cc2ee15723f087c367d2 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 16 Nov 2023 15:48:37 +0100 Subject: MastoAPI AccountView: Change last_status_at to be a date Changed in Mastodon 3.1.0 with: https://github.com/tootsuite/mastodon/pull/12966 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index cc3e3582f..34478cec8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors +# Copyright © 2017-2023 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.AccountView do @@ -249,6 +249,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do nil end + last_status_at = + if not is_nil(user.last_status_at) do + user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601() + else + nil + end + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -277,7 +284,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do actor_type: user.actor_type } }, - last_status_at: user.last_status_at, + last_status_at: last_status_at, # Pleroma extensions # Note: it's insecure to output :email but fully-qualified nickname may serve as safe stub -- cgit v1.2.3 From 0c6a54b37cb208e577fc9e40782bb8b820730428 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 Nov 2023 17:04:47 -0500 Subject: Upload.Filter.AnalyzeMetadata: Blurhash with a Rust NIF, and use Vix to retrieve image metadata --- lib/pleroma/upload/filter/analyze_metadata.ex | 36 +++++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 9a76a998b..15dec5564 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -8,22 +8,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do """ require Logger + alias Vix.Vips.Image + alias Vix.Vips.Operation + @behaviour Pleroma.Upload.Filter @spec filter(Pleroma.Upload.t()) :: {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do try do - image = - file - |> Mogrify.open() - |> Mogrify.verbose() + {:ok, image} = Image.new_from_file(file) + {width, height} = {Image.width(image), Image.height(image)} upload = upload - |> Map.put(:width, image.width) - |> Map.put(:height, image.height) - |> Map.put(:blurhash, get_blurhash(file)) + |> Map.put(:width, width) + |> Map.put(:height, height) + |> Map.put(:blurhash, get_blurhash(image)) {:ok, :filtered, upload} rescue @@ -53,7 +54,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do def filter(_), do: {:ok, :noop} defp get_blurhash(file) do - with {:ok, blurhash} <- :eblurhash.magick(file) do + with {:ok, blurhash} <- vips_blurhash(file) do blurhash else _ -> nil @@ -80,4 +81,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {:error, _} = error -> error end end + + defp vips_blurhash(image = %Vix.Vips.Image{}) do + {:ok, resized_image} = Operation.thumbnail_image(image, 20) + {height, width} = {Image.height(resized_image), Image.width(resized_image)} + max = max(height, width) + {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} + + {:ok, rgba} = + if Image.has_alpha?(resized_image) do + Image.to_list(resized_image) + else + Operation.bandjoin_const!(resized_image, [255]) + |> Image.to_list() + end + + rgba = List.flatten(rgba) + + Blurhash.encode(x, y, width, height, rgba) + end end -- cgit v1.2.3 From 88cc7e6a0431385b5fce292179c7eb05e64e3a24 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 17 Nov 2023 11:06:31 -0500 Subject: Resize images to 100 pixels before hashing --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 15dec5564..99d1ec5bc 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end defp vips_blurhash(image = %Vix.Vips.Image{}) do - {:ok, resized_image} = Operation.thumbnail_image(image, 20) + {:ok, resized_image} = Operation.thumbnail_image(image, 100) {height, width} = {Image.height(resized_image), Image.width(resized_image)} max = max(height, width) {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} -- cgit v1.2.3 From 510a7b64f1354d4313ab565d557f422b7e059432 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Wed, 15 Nov 2023 00:43:58 -0700 Subject: Add optional URL value for scrobbles --- lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex | 6 +++++- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/pleroma_api/views/scrobble_view.ex | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index ca40da930..68c586b73 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -59,6 +59,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, length: %Schema{type: :integer, description: "The length of the media playing"}, + url: %Schema{type: :string, description: "A URL referencing the media playing"}, visibility: %Schema{ allOf: [VisibilityScope], default: "public", @@ -69,7 +70,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "title" => "Some Title", "artist" => "Some Artist", "album" => "Some Album", - "length" => 180_000 + "length" => 180_000, + "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title" } } end @@ -83,6 +85,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do title: %Schema{type: :string, description: "The title of the media playing"}, album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, + url: %Schema{type: :string, description: "A URL referencing the media playing"}, length: %Schema{ type: :integer, description: "The length of the media playing", @@ -97,6 +100,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, + "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at" => "2019-09-28T12:40:45.000Z" } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index ca1329284..00cbacbb4 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp listen_object(draft) do object = draft.params - |> Map.take([:album, :artist, :title, :length]) + |> Map.take([:album, :artist, :title, :length, :url]) |> Map.new(fn {key, value} -> {to_string(key), value} end) |> Map.put("type", "Audio") |> Map.put("to", draft.to) diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index a5985fb2a..7a983f8b5 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -27,6 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do title: object.data["title"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(), album: object.data["album"] |> HTML.strip_tags(), + url: object.data["url"], length: object.data["length"] } end -- cgit v1.2.3 From 299c548b124377e51f6c089bc0df31b2989be3ef Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 23 Nov 2023 16:15:53 -0500 Subject: Prevent a blurhash failure from breaking all metadata collection --- lib/pleroma/upload/filter/analyze_metadata.ex | 34 ++++++++++++++------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 99d1ec5bc..92b80b1b1 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -83,21 +83,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end defp vips_blurhash(image = %Vix.Vips.Image{}) do - {:ok, resized_image} = Operation.thumbnail_image(image, 100) - {height, width} = {Image.height(resized_image), Image.width(resized_image)} - max = max(height, width) - {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} - - {:ok, rgba} = - if Image.has_alpha?(resized_image) do - Image.to_list(resized_image) - else - Operation.bandjoin_const!(resized_image, [255]) - |> Image.to_list() - end - - rgba = List.flatten(rgba) - - Blurhash.encode(x, y, width, height, rgba) + with {:ok, resized_image} <- Operation.thumbnail_image(image, 100), + {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, + max <- max(height, width), + {x, y} <- {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} do + {:ok, rgba} = + if Image.has_alpha?(resized_image) do + Image.to_list(resized_image) + else + Operation.bandjoin_const!(resized_image, [255]) + |> Image.to_list() + end + + rgba = List.flatten(rgba) + + Blurhash.encode(x, y, width, height, rgba) + else + _ -> nil + end end end -- cgit v1.2.3 From 27df2c0ce6c214f36db742af702fc239f80764a7 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Mon, 27 Nov 2023 03:34:31 -0700 Subject: Fix #strip_report_status_data --- lib/pleroma/web/activity_pub/utils.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 437220077..073ccd615 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -855,6 +855,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do Enum.map(reported_activities, fn act when is_map(act) -> act["id"] act when is_binary(act) -> act + _other -> nil end) new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) -- cgit v1.2.3 From 4ef56c5b65a4d1e7e90a87f1a1a507df523a4b4b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 Nov 2023 18:44:11 +0400 Subject: ActivityPub.Utils: Only treat object ids as valid while stripping --- lib/pleroma/web/activity_pub/utils.ex | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 073ccd615..b32f19740 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do alias Ecto.UUID alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -852,10 +853,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do [actor | reported_activities] = activity.data["object"] stripped_activities = - Enum.map(reported_activities, fn - act when is_map(act) -> act["id"] - act when is_binary(act) -> act - _other -> nil + Enum.reduce(reported_activities, [], fn act, acc -> + case ObjectID.cast(act) do + {:ok, act} -> [act | acc] + _ -> acc + end end) new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) -- cgit v1.2.3 From 03db495e1d88f34bef8d556b0f88806c3260d403 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 12:23:41 +0400 Subject: AnalyzeMetadata: Switch to rinpatch_blurhash --- lib/pleroma/upload/filter/analyze_metadata.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 92b80b1b1..710fd02d2 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -87,17 +87,17 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, max <- max(height, width), {x, y} <- {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} do - {:ok, rgba} = + {:ok, rgb} = if Image.has_alpha?(resized_image) do - Image.to_list(resized_image) + # remove alpha channel + resized_image + |> Operation.extract_band!(0, n: 3) + |> Image.write_to_binary() else - Operation.bandjoin_const!(resized_image, [255]) - |> Image.to_list() + Image.write_to_binary(resized_image) end - rgba = List.flatten(rgba) - - Blurhash.encode(x, y, width, height, rgba) + Blurhash.encode(rgb, width, height, x, y) else _ -> nil end -- cgit v1.2.3 From b3214be32f83541626b198b86ee065b92862f4cf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 12:33:54 +0400 Subject: AnayzeMetadata: Fix error case that would never match --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 710fd02d2..ef75d73d3 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -78,7 +78,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do %{width: width, height: height} else nil -> {:error, {:ffprobe, :command_not_found}} - {:error, _} = error -> error + error -> {:error, error} end end -- cgit v1.2.3 From ccc2adee4111367d67646f6f2828e03b861dd393 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 13:13:43 +0400 Subject: Linting --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index ef75d73d3..e510ae3e6 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -82,7 +82,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end end - defp vips_blurhash(image = %Vix.Vips.Image{}) do + defp vips_blurhash(%Vix.Vips.Image{} = image) do with {:ok, resized_image} <- Operation.thumbnail_image(image, 100), {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, max <- max(height, width), -- cgit v1.2.3 From e216603477e2c393a586f7eb0bc8183e73bf4cd7 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Wed, 29 Nov 2023 07:55:44 -0700 Subject: Change url to externalLink as requested by hj here: https://shigusegubu.club/notice/AcIjZjackKAt6e522a --- lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex | 8 ++++---- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/pleroma_api/views/scrobble_view.ex | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index 68c586b73..141b60533 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, length: %Schema{type: :integer, description: "The length of the media playing"}, - url: %Schema{type: :string, description: "A URL referencing the media playing"}, + externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, visibility: %Schema{ allOf: [VisibilityScope], default: "public", @@ -71,7 +71,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, - "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title" + "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title" } } end @@ -85,7 +85,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do title: %Schema{type: :string, description: "The title of the media playing"}, album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, - url: %Schema{type: :string, description: "A URL referencing the media playing"}, + externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, length: %Schema{ type: :integer, description: "The length of the media playing", @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, - "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title", + "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at" => "2019-09-28T12:40:45.000Z" } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 00cbacbb4..8910ad5b8 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp listen_object(draft) do object = draft.params - |> Map.take([:album, :artist, :title, :length, :url]) + |> Map.take([:album, :artist, :title, :length, :externalLink]) |> Map.new(fn {key, value} -> {to_string(key), value} end) |> Map.put("type", "Audio") |> Map.put("to", draft.to) diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index 7a983f8b5..edf0a2390 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do title: object.data["title"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(), album: object.data["album"] |> HTML.strip_tags(), - url: object.data["url"], + externalLink: object.data["externalLink"], length: object.data["length"] } end -- cgit v1.2.3 From 15a8acbd6cc6ecc89b49956acfeba49d02270c1d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 30 Nov 2023 09:40:17 +0400 Subject: MRF, Docs.Generator: Ensure code is loaded before checking it --- lib/pleroma/docs/generator.ex | 2 ++ lib/pleroma/web/activity_pub/mrf.ex | 3 +++ 2 files changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 6508f1947..456a8fd54 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Docs.Generator do # This shouldn't be needed as all modules are expected to have module_info/1, # but in test enviroments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). + Code.ensure_loaded(module) + if function_exported?(module, :module_info, 1) do module.module_info(:attributes) |> Keyword.get_values(:behaviour) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index ff9f84497..a0deac300 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -54,6 +54,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do @required_description_keys [:key, :related_policy] def filter_one(policy, message) do + Code.ensure_loaded(policy) + should_plug_history? = if function_exported?(policy, :history_awareness, 0) do policy.history_awareness() @@ -188,6 +190,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do def config_descriptions(policies) do Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc -> + Code.ensure_loaded(policy) if function_exported?(policy, :config_description, 0) do description = @default_description -- cgit v1.2.3 From eb6be30602f95cf86aef465229dacbdcb92c3c58 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 30 Nov 2023 10:05:00 +0400 Subject: Linting --- lib/pleroma/web/activity_pub/mrf.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index a0deac300..69054e25b 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -191,6 +191,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do def config_descriptions(policies) do Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc -> Code.ensure_loaded(policy) + if function_exported?(policy, :config_description, 0) do description = @default_description -- cgit v1.2.3 From 1ad0d94d6f1f11c848f277877cc14b3a47a7ae95 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 5 Dec 2023 16:35:41 -0500 Subject: Change set_reachable/1 to an upsert --- lib/pleroma/instances/instance.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 9756c66dc..101e5dc88 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -97,13 +97,9 @@ defmodule Pleroma.Instances.Instance do def reachable?(url_or_host) when is_binary(url_or_host), do: true def set_reachable(url_or_host) when is_binary(url_or_host) do - with host <- host(url_or_host), - %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do - {:ok, _instance} = - existing_record - |> changeset(%{unreachable_since: nil}) - |> Repo.update() - end + %Instance{host: host(url_or_host)} + |> changeset(%{unreachable_since: nil}) + |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host) end def set_reachable(_), do: {:error, nil} -- cgit v1.2.3 From ef8a2134bc9e2e9f3aa08172b982c347332e14c1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 7 Dec 2023 11:25:18 +0400 Subject: AccountView: Add test, refactor --- lib/pleroma/web/mastodon_api/views/account_view.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 34478cec8..237de3055 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -250,11 +250,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end last_status_at = - if not is_nil(user.last_status_at) do + user.last_status_at && user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601() - else - nil - end %{ id: to_string(user.id), -- cgit v1.2.3 From b70ca7d54e052abe0cb572e37e1af9c7caa815db Mon Sep 17 00:00:00 2001 From: kPherox Date: Thu, 7 Dec 2023 09:53:24 +0000 Subject: fix: AnalyzeMetadata filter no longer depends on ImageMagick's commands --- lib/pleroma/application_requirements.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 44b1c1705..477eac35b 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -168,8 +168,6 @@ defmodule Pleroma.ApplicationRequirements do check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"), check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"), check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"), - check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"), - check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert"), check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe") ] -- cgit v1.2.3 From 074b31d9ab30bbecba3fff4335dfec39af5cf9b8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 7 Dec 2023 22:27:19 -0500 Subject: Optimistic Inbox Rework inbound federation to accept requests optimistically. The HTTP Signatures Plug will not attempt to fetch the actor or key and will fail early. If the signature cannot be validated we pass the required data into the Oban job with a reduced priority and increase the timeout to 20 seconds. The Oban job will handle the actor and key fetching before attempting to validate the activity again. This job will be retried 5 times by default. Another welcome side effect is that actors who change their keys can federate to Pleroma instances immediately instead of needing to wait the default value of 86400s / 24 hours before the key will be fetched again. --- lib/pleroma/signature.ex | 2 +- lib/pleroma/user.ex | 11 +++++++- .../web/activity_pub/activity_pub_controller.ex | 7 ++--- lib/pleroma/web/federator.ex | 7 +++++ lib/pleroma/workers/receiver_worker.ex | 33 ++++++++++++++++++++-- 5 files changed, 51 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 5cfdae051..42cceba28 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Signature do with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), {:ok, actor_id} <- key_id_to_actor_id(kid), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), - {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do + {:ok, public_key} <- User.get_or_fetch_public_key_for_ap_id(actor_id) do {:ok, public_key} else e -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ce125d608..4c9aeffcb 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2135,7 +2135,7 @@ defmodule Pleroma.User do def public_key(_), do: {:error, "key not found"} - def get_public_key_for_ap_id(ap_id) do + def get_or_fetch_public_key_for_ap_id(ap_id) do with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id), {:ok, public_key} <- public_key(user) do {:ok, public_key} @@ -2144,6 +2144,15 @@ defmodule Pleroma.User do end end + def get_public_key_for_ap_id(ap_id) do + with {:ok, %User{} = user} <- get_cached_by_ap_id(ap_id), + {:ok, public_key} <- public_key(user) do + {:ok, public_key} + else + _ -> :error + end + end + @doc "Gets or fetch a user by uri or nickname." @spec get_or_fetch(String.t()) :: {:ok, User.t()} | {:error, String.t()} def get_or_fetch("http://" <> _host = uri), do: get_or_fetch_by_ap_id(uri) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 1357c379c..3b2193ca3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -287,10 +287,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do json(conn, "ok") end - def inbox(%{assigns: %{valid_signature: false}} = conn, _params) do - conn - |> put_status(:bad_request) - |> json("Invalid HTTP Signature") + def inbox(%{assigns: %{valid_signature: false}, req_headers: req_headers} = conn, params) do + Federator.incoming_ap_doc(%{req_headers: req_headers, params: params}) + json(conn, "ok") end # POST /relay/inbox -or- POST /internal/fetch/inbox diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 84b77cda1..1b8b22b7e 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -35,6 +35,13 @@ defmodule Pleroma.Web.Federator do end # Client API + def incoming_ap_doc(%{params: params, req_headers: req_headers}) do + ReceiverWorker.enqueue( + "incoming_ap_doc", + %{"req_headers" => req_headers, "params" => params, "timeout" => :timer.seconds(20)}, + priority: 5 + ) + end def incoming_ap_doc(params) do ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index cf1bb62b4..b04e2974a 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -3,24 +3,51 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ReceiverWorker do + alias Pleroma.Signature + alias Pleroma.User alias Pleroma.Web.Federator use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" @impl Oban.Worker + + def perform(%Job{ + args: %{"op" => "incoming_ap_doc", "req_headers" => req_headers, "params" => params} + }) do + conn_data = %{params: params, req_headers: req_headers} + + with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), + {:ok, _public_key} <- Signature.refetch_public_key(conn_data), + {:signature, true} <- {:signature, HTTPSignatures.validate_conn(conn_data)}, + {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do + {:ok, res} + else + e -> process_errors(e) + end + end + def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do {:ok, res} else + e -> process_errors(e) + end + end + + @impl Oban.Worker + def timeout(%_{args: %{"timeout" => timeout}}), do: timeout + + def timeout(_job), do: :timer.seconds(5) + + defp process_errors(errors) do + case errors do {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} {:error, :already_present} -> {:cancel, :already_present} {:error, {:validate_object, reason}} -> {:cancel, reason} {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} + {:signature, false} -> {:error, :invalid_signature} e -> e end end - - @impl Oban.Worker - def timeout(_job), do: :timer.seconds(5) end -- cgit v1.2.3 From ce5acd4158a351c59a79a4a0692eb2ef9775b554 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 8 Dec 2023 18:10:22 -0500 Subject: get_cached_by_ap_id/1 returns a single result, not a tuple --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 4c9aeffcb..3f729fdcc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2145,7 +2145,7 @@ defmodule Pleroma.User do end def get_public_key_for_ap_id(ap_id) do - with {:ok, %User{} = user} <- get_cached_by_ap_id(ap_id), + with %User{} = user <- get_cached_by_ap_id(ap_id), {:ok, public_key} <- public_key(user) do {:ok, public_key} else -- cgit v1.2.3 From 97cf78f63d312d0475ac8908d0b093cb5eff18d5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 8 Dec 2023 18:24:30 -0500 Subject: Remove unnecessary forced refresh of user --- lib/pleroma/user.ex | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3f729fdcc..c7753ca5d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1037,16 +1037,6 @@ defmodule Pleroma.User do defp maybe_send_registration_email(_), do: {:ok, :noop} - def needs_update?(%User{local: true}), do: false - - def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true - - def needs_update?(%User{local: false} = user) do - NaiveDateTime.diff(NaiveDateTime.utc_now(), user.last_refreshed_at) >= 86_400 - end - - def needs_update?(_), do: true - @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()} # "Locked" (self-locked) users demand explicit authorization of follow requests @@ -2059,15 +2049,13 @@ defmodule Pleroma.User do def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) def get_or_fetch_by_ap_id(ap_id) do - cached_user = get_cached_by_ap_id(ap_id) - - maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id) + user = get_cached_by_ap_id(ap_id) || fetch_by_ap_id(ap_id) - case {cached_user, maybe_fetched_user} do - {_, {:ok, %User{} = user}} -> + case user do + %User{} = user -> {:ok, user} - {%User{} = user, _} -> + {:ok, %User{} = user} -> {:ok, user} _ -> -- cgit v1.2.3 From 1d417d2a364eb8ae2035275927f6284951fe128b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 8 Dec 2023 21:49:25 -0500 Subject: Our version of Oban only supports priorities 0-3 --- lib/pleroma/web/federator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 1b8b22b7e..669a5bb0b 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Web.Federator do ReceiverWorker.enqueue( "incoming_ap_doc", %{"req_headers" => req_headers, "params" => params, "timeout" => :timer.seconds(20)}, - priority: 5 + priority: 2 ) end -- cgit v1.2.3 From 4039106500ed0adba3ac6dbd480954aa76ce6bc3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 8 Dec 2023 21:51:36 -0500 Subject: Fix the req_headers formatting --- lib/pleroma/workers/receiver_worker.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index b04e2974a..562112dd2 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -14,6 +14,10 @@ defmodule Pleroma.Workers.ReceiverWorker do def perform(%Job{ args: %{"op" => "incoming_ap_doc", "req_headers" => req_headers, "params" => params} }) do + # Oban's serialization converts our tuple headers to lists. + # Revert it for the signature validation. + req_headers = Enum.into(req_headers, [], &List.to_tuple(&1)) + conn_data = %{params: params, req_headers: req_headers} with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), -- cgit v1.2.3 From 82724f6664c05e746eb241fb60aefd9a931b372d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 17:43:54 -0500 Subject: Do not retry fetching deleted objects --- lib/pleroma/workers/receiver_worker.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 562112dd2..d4e8e715e 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -51,6 +51,7 @@ defmodule Pleroma.Workers.ReceiverWorker do {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} {:signature, false} -> {:error, :invalid_signature} + {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason} e -> e end end -- cgit v1.2.3 From 94daa3e8c1ec42f40214dc13f061f4319764f715 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 18:21:00 -0500 Subject: Revert "Remove unnecessary forced refresh of user" This reverts commit 97cf78f63d312d0475ac8908d0b093cb5eff18d5. --- lib/pleroma/user.ex | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c7753ca5d..3f729fdcc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1037,6 +1037,16 @@ defmodule Pleroma.User do defp maybe_send_registration_email(_), do: {:ok, :noop} + def needs_update?(%User{local: true}), do: false + + def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true + + def needs_update?(%User{local: false} = user) do + NaiveDateTime.diff(NaiveDateTime.utc_now(), user.last_refreshed_at) >= 86_400 + end + + def needs_update?(_), do: true + @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()} # "Locked" (self-locked) users demand explicit authorization of follow requests @@ -2049,13 +2059,15 @@ defmodule Pleroma.User do def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) def get_or_fetch_by_ap_id(ap_id) do - user = get_cached_by_ap_id(ap_id) || fetch_by_ap_id(ap_id) + cached_user = get_cached_by_ap_id(ap_id) + + maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id) - case user do - %User{} = user -> + case {cached_user, maybe_fetched_user} do + {_, {:ok, %User{} = user}} -> {:ok, user} - {:ok, %User{} = user} -> + {%User{} = user, _} -> {:ok, user} _ -> -- cgit v1.2.3 From d417f7321801890381de4daa53c1a78ef4650d2c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 18:40:29 -0500 Subject: Process inbound Delete activities at lowest priority --- lib/pleroma/web/federator.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 669a5bb0b..8621d984c 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -43,6 +43,10 @@ defmodule Pleroma.Web.Federator do ) end + def incoming_ap_doc(%{"type" => "Delete"} = params) do + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) + end + def incoming_ap_doc(params) do ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) end -- cgit v1.2.3 From c068a218eac02044cb5823abb1917051e1815b4f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 10 Dec 2023 18:57:46 +0400 Subject: Backup Tests: Split out async tests, use mox. --- lib/pleroma/user/backup.ex | 60 ++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 447fca2a1..c7359313a 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -35,6 +35,8 @@ defmodule Pleroma.User.Backup do timestamps() end + @config_impl Application.compile_env(__MODULE__, :config_impl, Pleroma.Config) + def create(user, admin_id \\ nil) do with :ok <- validate_limit(user, admin_id), {:ok, backup} <- user |> new() |> Repo.insert() do @@ -124,7 +126,10 @@ defmodule Pleroma.User.Backup do |> Repo.update() end - def process(%__MODULE__{} = backup) do + def process( + %__MODULE__{} = backup, + processor_module \\ __MODULE__.Processor + ) do set_state(backup, :running, 0) current_pid = self() @@ -132,7 +137,7 @@ defmodule Pleroma.User.Backup do task = Task.Supervisor.async_nolink( Pleroma.TaskSupervisor, - __MODULE__, + processor_module, :do_process, [backup, current_pid] ) @@ -140,25 +145,8 @@ defmodule Pleroma.User.Backup do wait_backup(backup, backup.processed_number, task) end - def do_process(backup, current_pid) do - with {:ok, zip_file} <- export(backup, current_pid), - {:ok, %{size: size}} <- File.stat(zip_file), - {:ok, _upload} <- upload(backup, zip_file) do - backup - |> cast( - %{ - file_size: size, - processed: true, - state: :complete - }, - [:file_size, :processed, :state] - ) - |> Repo.update() - end - end - defp wait_backup(backup, current_processed, task) do - wait_time = Pleroma.Config.get([__MODULE__, :process_wait_time]) + wait_time = @config_impl.get([__MODULE__, :process_wait_time]) receive do {:progress, new_processed} -> @@ -365,3 +353,35 @@ defmodule Pleroma.User.Backup do ) end end + +defmodule Pleroma.User.Backup.ProcessorAPI do + @callback do_process(%Pleroma.User.Backup{}, pid()) :: + {:ok, %Pleroma.User.Backup{}} | {:error, any()} +end + +defmodule Pleroma.User.Backup.Processor do + @behaviour Pleroma.User.Backup.ProcessorAPI + + alias Pleroma.Repo + alias Pleroma.User.Backup + + import Ecto.Changeset + + @impl true + def do_process(backup, current_pid) do + with {:ok, zip_file} <- Backup.export(backup, current_pid), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- Backup.upload(backup, zip_file) do + backup + |> cast( + %{ + file_size: size, + processed: true, + state: :complete + }, + [:file_size, :processed, :state] + ) + |> Repo.update() + end + end +end -- cgit v1.2.3 From 06fc196772a967a44593a10ceaf00396ce6b8de3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 10 Dec 2023 19:46:25 +0400 Subject: Backup: Fix config --- lib/pleroma/user/backup.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index c7359313a..04dc8e0ce 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -35,7 +35,7 @@ defmodule Pleroma.User.Backup do timestamps() end - @config_impl Application.compile_env(__MODULE__, :config_impl, Pleroma.Config) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) def create(user, admin_id \\ nil) do with :ok <- validate_limit(user, admin_id), -- cgit v1.2.3 From 223c1bac8dcbfd6b2454b79077545d3465688d7f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 10 Dec 2023 12:55:41 -0500 Subject: Cancel the job if the signature is still invalid after a refetch of the public key --- lib/pleroma/workers/receiver_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index d4e8e715e..1dddd8d2e 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Workers.ReceiverWorker do {:error, {:validate_object, reason}} -> {:cancel, reason} {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} - {:signature, false} -> {:error, :invalid_signature} + {:signature, false} -> {:cancel, :invalid_signature} {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason} e -> e end -- cgit v1.2.3 From 18deea59b441a89c1e6870d29dd0a6c0f3070f55 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 10 Dec 2023 13:22:55 -0500 Subject: ActivityPub.make_user_from_ap_id/1 fetches the whole actor object including updating the public key for us --- lib/pleroma/signature.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 42cceba28..5cfdae051 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Signature do with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), {:ok, actor_id} <- key_id_to_actor_id(kid), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), - {:ok, public_key} <- User.get_or_fetch_public_key_for_ap_id(actor_id) do + {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} else e -> -- cgit v1.2.3 From c0a50b7c3e340cd621827922200daa0f29dc6e15 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 10 Dec 2023 13:24:04 -0500 Subject: User.get_or_fetch_public_key_for_ap_id/1 is no longer required. --- lib/pleroma/user.ex | 9 --------- 1 file changed, 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3f729fdcc..0706f5607 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2135,15 +2135,6 @@ defmodule Pleroma.User do def public_key(_), do: {:error, "key not found"} - def get_or_fetch_public_key_for_ap_id(ap_id) do - with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id), - {:ok, public_key} <- public_key(user) do - {:ok, public_key} - else - _ -> :error - end - end - def get_public_key_for_ap_id(ap_id) do with %User{} = user <- get_cached_by_ap_id(ap_id), {:ok, public_key} <- public_key(user) do -- cgit v1.2.3 From 90a47ca050c5839e8b4dc3bac315dc436d49152d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 11 Dec 2023 09:25:05 +0400 Subject: S3 Test: Remove global state dependencies --- lib/pleroma/upload.ex | 15 ++++++++------- lib/pleroma/uploaders/s3.ex | 15 ++++++++++----- 2 files changed, 18 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 4aee9326f..bedd7889a 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -34,7 +34,6 @@ defmodule Pleroma.Upload do """ alias Ecto.UUID - alias Pleroma.Config alias Pleroma.Maps alias Pleroma.Web.ActivityPub.Utils require Logger @@ -76,6 +75,8 @@ defmodule Pleroma.Upload do :path ] + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + defp get_description(upload) do case {upload.description, Pleroma.Config.get([Pleroma.Upload, :default_description])} do {description, _} when is_binary(description) -> description @@ -244,18 +245,18 @@ defmodule Pleroma.Upload do defp url_from_spec(_upload, _base_url, {:url, url}), do: url def base_url do - uploader = Config.get([Pleroma.Upload, :uploader]) - upload_base_url = Config.get([Pleroma.Upload, :base_url]) - public_endpoint = Config.get([uploader, :public_endpoint]) + uploader = @config_impl.get([Pleroma.Upload, :uploader]) + upload_base_url = @config_impl.get([Pleroma.Upload, :base_url]) + public_endpoint = @config_impl.get([uploader, :public_endpoint]) case uploader do Pleroma.Uploaders.Local -> upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" Pleroma.Uploaders.S3 -> - bucket = Config.get([Pleroma.Uploaders.S3, :bucket]) - truncated_namespace = Config.get([Pleroma.Uploaders.S3, :truncated_namespace]) - namespace = Config.get([Pleroma.Uploaders.S3, :bucket_namespace]) + bucket = @config_impl.get([Pleroma.Uploaders.S3, :bucket]) + truncated_namespace = @config_impl.get([Pleroma.Uploaders.S3, :truncated_namespace]) + namespace = @config_impl.get([Pleroma.Uploaders.S3, :bucket_namespace]) bucket_with_namespace = cond do diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index 19287c532..7b32bd8a5 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -6,7 +6,8 @@ defmodule Pleroma.Uploaders.S3 do @behaviour Pleroma.Uploaders.Uploader require Logger - alias Pleroma.Config + @ex_aws_impl Application.compile_env(:pleroma, [__MODULE__, :ex_aws_impl], ExAws) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) # The file name is re-encoded with S3's constraints here to comply with previous # links with less strict filenames @@ -22,7 +23,7 @@ defmodule Pleroma.Uploaders.S3 do @impl true def put_file(%Pleroma.Upload{} = upload) do - config = Config.get([__MODULE__]) + config = @config_impl.get([__MODULE__]) bucket = Keyword.get(config, :bucket) streaming = Keyword.get(config, :streaming_enabled) @@ -56,7 +57,7 @@ defmodule Pleroma.Uploaders.S3 do ]) end - case ExAws.request(op) do + case @ex_aws_impl.request(op) do {:ok, _} -> {:ok, {:file, s3_name}} @@ -69,9 +70,9 @@ defmodule Pleroma.Uploaders.S3 do @impl true def delete_file(file) do [__MODULE__, :bucket] - |> Config.get() + |> @config_impl.get() |> ExAws.S3.delete_object(file) - |> ExAws.request() + |> @ex_aws_impl.request() |> case do {:ok, %{status_code: 204}} -> :ok error -> {:error, inspect(error)} @@ -83,3 +84,7 @@ defmodule Pleroma.Uploaders.S3 do String.replace(name, @regex, "-") end end + +defmodule Pleroma.Uploaders.S3.ExAwsAPI do + @callback request(op :: ExAws.Operation.t()) :: {:ok, ExAws.Operation.t()} | {:error, term()} +end -- cgit v1.2.3 From 228966e6dd1bd4a01d0c01609d9f6ad2279d278f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 11:09:54 +0400 Subject: Exiftool.ReadDescription: Remove wrong spec. --- lib/pleroma/upload/filter/exiftool/read_description.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/exiftool/read_description.ex b/lib/pleroma/upload/filter/exiftool/read_description.ex index 543b22031..8c1ed82f8 100644 --- a/lib/pleroma/upload/filter/exiftool/read_description.ex +++ b/lib/pleroma/upload/filter/exiftool/read_description.ex @@ -10,8 +10,6 @@ defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do """ @behaviour Pleroma.Upload.Filter - @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()} - def filter(%Pleroma.Upload{description: description}) when is_binary(description), do: {:ok, :noop} -- cgit v1.2.3 From 22c4d89dbbc45169dfc325e64527e6dd30f7c422 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 12:48:55 +0400 Subject: ScheduledActivity: Use config mocking --- lib/pleroma/scheduled_activity.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 0ed51ad07..63c6cb45b 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -6,7 +6,6 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema alias Ecto.Multi - alias Pleroma.Config alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User @@ -20,6 +19,8 @@ defmodule Pleroma.ScheduledActivity do @min_offset :timer.minutes(5) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + schema "scheduled_activities" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:scheduled_at, :naive_datetime) @@ -87,7 +88,7 @@ defmodule Pleroma.ScheduledActivity do |> where([sa], type(sa.scheduled_at, :date) == type(^scheduled_at, :date)) |> select([sa], count(sa.id)) |> Repo.one() - |> Kernel.>=(Config.get([ScheduledActivity, :daily_user_limit])) + |> Kernel.>=(@config_impl.get([ScheduledActivity, :daily_user_limit])) end def exceeds_total_user_limit?(user_id) do @@ -95,7 +96,7 @@ defmodule Pleroma.ScheduledActivity do |> where(user_id: ^user_id) |> select([sa], count(sa.id)) |> Repo.one() - |> Kernel.>=(Config.get([ScheduledActivity, :total_user_limit])) + |> Kernel.>=(@config_impl.get([ScheduledActivity, :total_user_limit])) end def far_enough?(scheduled_at) when is_binary(scheduled_at) do @@ -123,7 +124,7 @@ defmodule Pleroma.ScheduledActivity do def create(%User{} = user, attrs) do Multi.new() |> Multi.insert(:scheduled_activity, new(user, attrs)) - |> maybe_add_jobs(Config.get([ScheduledActivity, :enabled])) + |> maybe_add_jobs(@config_impl.get([ScheduledActivity, :enabled])) |> Repo.transaction() |> transaction_response end -- cgit v1.2.3 From 00def0875b2014ab18a7b159de855a3c05dd39a4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 13:28:11 +0400 Subject: RichMediaTest: Use mocked config --- lib/pleroma/web/rich_media/helpers.ex | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 0488df30e..61000bb9b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -4,11 +4,12 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + @options [ pool: :media, max_body: 2_000_000, @@ -17,7 +18,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Config.get([Pleroma.Formatter, :validate_tld]) + validate_tld = @config_impl.get([Pleroma.Formatter, :validate_tld]) page_url |> Linkify.Parser.url?(validate_tld: validate_tld) @@ -27,10 +28,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp validate_page_url(%URI{host: host, scheme: "https", authority: authority}) when is_binary(authority) do cond do - host in Config.get([:rich_media, :ignore_hosts], []) -> + host in @config_impl.get([:rich_media, :ignore_hosts], []) -> :error - get_tld(host) in Config.get([:rich_media, :ignore_tld], []) -> + get_tld(host) in @config_impl.get([:rich_media, :ignore_tld], []) -> :error true -> @@ -56,7 +57,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do end def fetch_data_for_object(object) do - with true <- Config.get([:rich_media, :enabled]), + with true <- @config_impl.get([:rich_media, :enabled]), {:ok, page_url} <- HTML.extract_first_external_url_from_object(object), :ok <- validate_page_url(page_url), @@ -68,7 +69,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do end def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do - with true <- Config.get([:rich_media, :enabled]), + with true <- @config_impl.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity, fetch: false) do fetch_data_for_object(object) else -- cgit v1.2.3 From dca41cc4a37999a6971b70ef5e0996e528b79bf5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 15:25:52 +0400 Subject: EmailTest: use config mock --- lib/pleroma/emails/mailer.ex | 3 ++- lib/pleroma/user.ex | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 101442130..02c148e9f 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -14,9 +14,10 @@ defmodule Pleroma.Emails.Mailer do @otp_app :pleroma @mailer_config [otp: :pleroma] + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) @spec enabled?() :: boolean() - def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) + def enabled?, do: @config_impl.get([__MODULE__, :enabled]) @doc "add email to queue" def deliver_async(email, config \\ []) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0706f5607..f939687eb 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -86,6 +86,7 @@ defmodule Pleroma.User do ] @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) schema "users" do field(:bio, :string, default: "") @@ -1000,7 +1001,7 @@ defmodule Pleroma.User do @spec maybe_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop} def maybe_send_confirmation_email(%User{is_confirmed: false, email: email} = user) when is_binary(email) do - if Config.get([:instance, :account_activation_required]) do + if @config_impl.get([:instance, :account_activation_required]) do send_confirmation_email(user) {:ok, :enqueued} else -- cgit v1.2.3 From e7af2addd85c66edc62262e5e799649fef6a9911 Mon Sep 17 00:00:00 2001 From: Alexander Tumin Date: Tue, 12 Dec 2023 15:08:20 +0300 Subject: Add media proxy to opengraph rich media cards --- lib/pleroma/web/mastodon_api/views/status_view.ex | 27 ++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d070262cc..8825f3715 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -562,25 +562,24 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do page_url = page_url_data |> to_string - image_url_data = - if is_binary(rich_media["image"]) do - URI.parse(rich_media["image"]) - else - nil - end - - image_url = build_image_url(image_url_data, page_url_data) + image_url = proxied_url(rich_media["image"], page_url_data) + audio_url = proxied_url(rich_media["audio"], page_url_data) + video_url = proxied_url(rich_media["video"], page_url_data) %{ type: "link", provider_name: page_url_data.host, provider_url: page_url_data.scheme <> "://" <> page_url_data.host, url: page_url, - image: image_url |> MediaProxy.url(), + image: image_url, title: rich_media["title"] || "", description: rich_media["description"] || "", pleroma: %{ - opengraph: rich_media + opengraph: + rich_media + |> Maps.put_if_present("image", image_url) + |> Maps.put_if_present("audio", audio_url) + |> Maps.put_if_present("video", video_url) } } end @@ -817,4 +816,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp get_source_content_type(_source) do Utils.get_content_type(nil) end + + defp proxied_url(url, page_url_data) do + if is_binary(url) do + build_image_url(URI.parse(url), page_url_data) |> MediaProxy.url() + else + nil + end + end end -- cgit v1.2.3 From cca6c20eb6acd0f510054d1a1050d12ea2a32482 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 19:35:19 +0400 Subject: Revert "EmailTest: use config mock" This reverts commit dca41cc4a37999a6971b70ef5e0996e528b79bf5. --- lib/pleroma/emails/mailer.ex | 3 +-- lib/pleroma/user.ex | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 02c148e9f..101442130 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -14,10 +14,9 @@ defmodule Pleroma.Emails.Mailer do @otp_app :pleroma @mailer_config [otp: :pleroma] - @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) @spec enabled?() :: boolean() - def enabled?, do: @config_impl.get([__MODULE__, :enabled]) + def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) @doc "add email to queue" def deliver_async(email, config \\ []) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f939687eb..0706f5607 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -86,7 +86,6 @@ defmodule Pleroma.User do ] @cachex Pleroma.Config.get([:cachex, :provider], Cachex) - @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) schema "users" do field(:bio, :string, default: "") @@ -1001,7 +1000,7 @@ defmodule Pleroma.User do @spec maybe_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop} def maybe_send_confirmation_email(%User{is_confirmed: false, email: email} = user) when is_binary(email) do - if @config_impl.get([:instance, :account_activation_required]) do + if Config.get([:instance, :account_activation_required]) do send_confirmation_email(user) {:ok, :enqueued} else -- cgit v1.2.3 From cb1b52d980e23c7d4f06dda5891bd2cbbdafeb95 Mon Sep 17 00:00:00 2001 From: Yonle Date: Fri, 15 Dec 2023 07:13:31 +0700 Subject: ap userview: add outbox field. Signed-off-by: Yonle --- lib/pleroma/web/activity_pub/views/user_view.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index f69fca075..24ee683ae 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -46,6 +46,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "following" => "#{user.ap_id}/following", "followers" => "#{user.ap_id}/followers", "inbox" => "#{user.ap_id}/inbox", + "outbox" => "#{user.ap_id}/outbox", "name" => "Pleroma", "summary" => "An internal service actor for this Pleroma instance. No user-serviceable parts inside.", -- cgit v1.2.3 From 3fbc80eb585b44e2f315b9bf02b655b50ee473d3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 16 Dec 2023 20:26:08 +0400 Subject: B ActivityPub.Publisher: Prioritize direct mentions --- lib/pleroma/web/activity_pub/publisher.ex | 104 +++++++++++++++++------------- lib/pleroma/web/federator/publisher.ex | 7 +- 2 files changed, 64 insertions(+), 47 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index af6aa0781..373ceef32 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -118,7 +118,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end - @spec recipients(User.t(), Activity.t()) :: list(User.t()) | [] + @spec recipients(User.t(), Activity.t()) :: {[User.t()], [User.t()]} defp recipients(actor, activity) do followers = if actor.follower_address in activity.recipients do @@ -138,7 +138,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [] end - Pleroma.Web.Federator.Publisher.remote_users(actor, activity) ++ followers ++ fetchers + mentioned = Pleroma.Web.Federator.Publisher.remote_users(actor, activity) + non_mentioned = (followers ++ fetchers) -- mentioned + + {mentioned, non_mentioned} end defp get_cc_ap_ids(ap_id, recipients) do @@ -195,34 +198,39 @@ defmodule Pleroma.Web.ActivityPub.Publisher do public = is_public?(activity) {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - recipients = recipients(actor, activity) + {priority_recipients, recipients} = recipients(actor, activity) inboxes = - recipients - |> Enum.map(fn actor -> actor.inbox end) - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - |> Instances.filter_reachable() + [priority_recipients, recipients] + |> Enum.map(fn recipients -> + recipients + |> Enum.map(fn actor -> actor.inbox end) + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Instances.filter_reachable() + end) Repo.checkout(fn -> - Enum.each(inboxes, fn {inbox, unreachable_since} -> - %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) - - # Get all the recipients on the same host and add them to cc. Otherwise, a remote - # instance would only accept a first message for the first recipient and ignore the rest. - cc = get_cc_ap_ids(ap_id, recipients) - - json = - data - |> Map.put("cc", cc) - |> Jason.encode!() - - Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ - inbox: inbox, - json: json, - actor_id: actor.id, - id: activity.data["id"], - unreachable_since: unreachable_since - }) + Enum.each(inboxes, fn inboxes -> + Enum.each(inboxes, fn {inbox, unreachable_since} -> + %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) + + # Get all the recipients on the same host and add them to cc. Otherwise, a remote + # instance would only accept a first message for the first recipient and ignore the rest. + cc = get_cc_ap_ids(ap_id, recipients) + + json = + data + |> Map.put("cc", cc) + |> Jason.encode!() + + Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ + inbox: inbox, + json: json, + actor_id: actor.id, + id: activity.data["id"], + unreachable_since: unreachable_since + }) + end) end) end) end @@ -239,25 +247,33 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) - recipients(actor, activity) - |> Enum.map(fn %User{} = user -> - determine_inbox(activity, user) - end) - |> Enum.uniq() - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - |> Instances.filter_reachable() - |> Enum.each(fn {inbox, unreachable_since} -> - Pleroma.Web.Federator.Publisher.enqueue_one( - __MODULE__, - %{ - inbox: inbox, - json: json, - actor_id: actor.id, - id: activity.data["id"], - unreachable_since: unreachable_since - } - ) + {priority_recipients, recipients} = recipients(actor, activity) + + [{priority_recipients, 0}, {recipients, 1}] + |> Enum.map(fn {recipients, priority} -> + recipients + |> Enum.map(fn %User{} = user -> + determine_inbox(activity, user) + end) + |> Enum.uniq() + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Instances.filter_reachable() + |> Enum.each(fn {inbox, unreachable_since} -> + Pleroma.Web.Federator.Publisher.enqueue_one( + __MODULE__, + %{ + inbox: inbox, + json: json, + actor_id: actor.id, + id: activity.data["id"], + unreachable_since: unreachable_since + }, + priority: priority + ) + end) end) + + :ok end def gather_webfinger_links(%User{} = user) do diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index a45796e9d..8c6547208 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -29,11 +29,12 @@ defmodule Pleroma.Web.Federator.Publisher do @doc """ Enqueue publishing a single activity. """ - @spec enqueue_one(module(), Map.t()) :: :ok - def enqueue_one(module, %{} = params) do + @spec enqueue_one(module(), Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} + def enqueue_one(module, %{} = params, worker_args \\ []) do PublisherWorker.enqueue( "publish_one", - %{"module" => to_string(module), "params" => params} + %{"module" => to_string(module), "params" => params}, + worker_args ) end -- cgit v1.2.3 From c212fc1dcfae578e83a0a8fb036da30ca0b92f56 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 16 Dec 2023 20:32:13 +0400 Subject: User: Ignore non-local users when setting 'last active at' --- lib/pleroma/user.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0706f5607..ddeaf0624 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2681,6 +2681,8 @@ defmodule Pleroma.User do |> update_and_set_cache() end + def update_last_active_at(user), do: user + def active_user_count(days \\ 30) do active_after = Timex.shift(NaiveDateTime.utc_now(), days: -days) -- cgit v1.2.3 From c1423ddca3c82140e537037fdf133ee057c2dac0 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 17 Dec 2023 11:15:58 +0400 Subject: ActivityPub.Publisher: Filter inboxes --- lib/pleroma/web/activity_pub/publisher.ex | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 373ceef32..a580994b1 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -118,7 +118,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end - @spec recipients(User.t(), Activity.t()) :: {[User.t()], [User.t()]} + @spec recipients(User.t(), Activity.t()) :: [[User.t()]] defp recipients(actor, activity) do followers = if actor.follower_address in activity.recipients do @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do mentioned = Pleroma.Web.Federator.Publisher.remote_users(actor, activity) non_mentioned = (followers ++ fetchers) -- mentioned - {mentioned, non_mentioned} + [mentioned, non_mentioned] end defp get_cc_ap_ids(ap_id, recipients) do @@ -198,7 +198,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do public = is_public?(activity) {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - {priority_recipients, recipients} = recipients(actor, activity) + [priority_recipients, recipients] = recipients(actor, activity) inboxes = [priority_recipients, recipients] @@ -247,16 +247,19 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) - {priority_recipients, recipients} = recipients(actor, activity) - - [{priority_recipients, 0}, {recipients, 1}] - |> Enum.map(fn {recipients, priority} -> - recipients - |> Enum.map(fn %User{} = user -> - determine_inbox(activity, user) + [priority_inboxes, inboxes] = + recipients(actor, activity) + |> Enum.map(fn recipients -> + recipients + |> Enum.map(fn actor -> actor.inbox end) + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) end) - |> Enum.uniq() - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + + inboxes = inboxes -- priority_inboxes + + [{priority_inboxes, 0}, {inboxes, 1}] + |> Enum.each(fn {inboxes, priority} -> + inboxes |> Instances.filter_reachable() |> Enum.each(fn {inbox, unreachable_since} -> Pleroma.Web.Federator.Publisher.enqueue_one( -- cgit v1.2.3 From e2066994b1e64481e0e74350688d91c71d03e230 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Dec 2023 10:18:29 -0500 Subject: Fix Web Push notification delivery Finch does not automatically append header content-type: octet-stream for binary payloads. --- lib/pleroma/http/web_push.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/web_push.ex b/lib/pleroma/http/web_push.ex index ca399b6c8..888079c1e 100644 --- a/lib/pleroma/http/web_push.ex +++ b/lib/pleroma/http/web_push.ex @@ -6,7 +6,11 @@ defmodule Pleroma.HTTP.WebPush do @moduledoc false def post(url, payload, headers, options \\ []) do - list_headers = Map.to_list(headers) + list_headers = + headers + |> Map.to_list() + |> Kernel.++([{"content-type", "octet-stream"}]) + Pleroma.HTTP.post(url, payload, list_headers, options) end end -- cgit v1.2.3 From f43f33e3078385084136295d2a3320efa6cb4134 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Dec 2023 13:53:10 -0500 Subject: Return a 400 from a bad delivery attempt to the inbox This stops the backend from generating 500 errors from these events. --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3b2193ca3..b1a118160 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -273,12 +273,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do - with %User{} = recipient <- User.get_cached_by_nickname(nickname), - {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]), + with %User{is_active: true} = recipient <- User.get_cached_by_nickname(nickname), + {:ok, %User{is_active: true} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]), true <- Utils.recipient_in_message(recipient, actor, params), params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do Federator.incoming_ap_doc(params) json(conn, "ok") + else + _ -> + conn + |> put_status(:bad_request) + |> json("Invalid request.") end end -- cgit v1.2.3 From 9896b64f54d4568a7b42ddd11e647021a20c1378 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 28 Nov 2023 17:17:46 +0000 Subject: Elixir 1.15: Chase the Logger.warn deprecation --- lib/pleroma/application.ex | 2 +- lib/pleroma/application_requirements.ex | 4 ++-- lib/pleroma/config/deprecation_warnings.ex | 18 +++++++++--------- lib/pleroma/config/oban.ex | 2 +- lib/pleroma/config/transfer_task.ex | 6 +++--- lib/pleroma/emoji/loader.ex | 2 +- lib/pleroma/gun/conn.ex | 6 +++--- lib/pleroma/http/adapter_helper.ex | 8 ++++---- lib/pleroma/instances/instance.ex | 6 +++--- lib/pleroma/maintenance.ex | 2 +- lib/pleroma/migrators/support/base_migrator.ex | 6 +++--- lib/pleroma/reverse_proxy.ex | 2 +- lib/pleroma/telemetry/logger.ex | 2 +- lib/pleroma/upload/filter/analyze_metadata.ex | 4 ++-- lib/pleroma/user.ex | 2 +- lib/pleroma/user/backup.ex | 2 +- .../web/activity_pub/activity_pub_controller.ex | 2 +- lib/pleroma/web/activity_pub/mrf.ex | 2 +- lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex | 4 ++-- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++-- lib/pleroma/web/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 4 ++-- lib/pleroma/web/plugs/http_security_plug.ex | 2 +- lib/pleroma/web/plugs/rate_limiter.ex | 2 +- lib/pleroma/web/push.ex | 2 +- lib/pleroma/web/push/impl.ex | 2 +- lib/pleroma/web/rich_media/parser.ex | 2 +- lib/pleroma/web/web_finger.ex | 2 +- 29 files changed, 53 insertions(+), 53 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 52cd6e9a9..8fa6f3fae 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -138,7 +138,7 @@ defmodule Pleroma.Application do num else e -> - Logger.warn( + Logger.warning( "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6" ) diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 477eac35b..1dbfea3e2 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -34,7 +34,7 @@ defmodule Pleroma.ApplicationRequirements do defp check_welcome_message_config!(:ok) do if Pleroma.Config.get([:welcome, :email, :enabled], false) and not Pleroma.Emails.Mailer.enabled?() do - Logger.warn(""" + Logger.warning(""" To send welcome emails, you need to enable the mailer. Welcome emails will NOT be sent with the current config. @@ -53,7 +53,7 @@ defmodule Pleroma.ApplicationRequirements do def check_confirmation_accounts!(:ok) do if Pleroma.Config.get([:instance, :account_activation_required]) && not Pleroma.Emails.Mailer.enabled?() do - Logger.warn(""" + Logger.warning(""" Account activation is required, but the mailer is disabled. Users will NOT be able to confirm their accounts with this config. Either disable account activation or enable the mailer. diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index b53b15d95..1cd3241ea 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Config.DeprecationWarnings do filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) if Pleroma.Upload.Filter.Exiftool in filters do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -63,7 +63,7 @@ defmodule Pleroma.Config.DeprecationWarnings do |> Enum.any?(fn {_, v} -> Enum.any?(v, &is_binary/1) end) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the SimplePolicy configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -121,7 +121,7 @@ defmodule Pleroma.Config.DeprecationWarnings do has_strings = Config.get([:instance, :quarantined_instances]) |> Enum.any?(&is_binary/1) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the quarantined_instances configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -158,7 +158,7 @@ defmodule Pleroma.Config.DeprecationWarnings do has_strings = Config.get([:mrf, :transparency_exclusions]) |> Enum.any?(&is_binary/1) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the transparency_exclusions configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -193,7 +193,7 @@ defmodule Pleroma.Config.DeprecationWarnings do def check_hellthread_threshold do if Config.get([:mrf_hellthread, :threshold]) do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! You are using the old configuration mechanism for the hellthread filter. Please check config.md. """) @@ -274,7 +274,7 @@ defmodule Pleroma.Config.DeprecationWarnings do if warning == "" do :ok else - Logger.warn(warning_preface <> warning) + Logger.warning(warning_preface <> warning) :error end end @@ -284,7 +284,7 @@ defmodule Pleroma.Config.DeprecationWarnings do whitelist = Config.get([:media_proxy, :whitelist]) if Enum.any?(whitelist, &(not String.starts_with?(&1, "http"))) do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. """) @@ -299,7 +299,7 @@ defmodule Pleroma.Config.DeprecationWarnings do pool_config = Config.get(:connections_pool) if timeout = pool_config[:await_up_timeout] do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using old setting `config :pleroma, :connections_pool, await_up_timeout`. Please change to `config :pleroma, :connections_pool, connect_timeout` to ensure compatibility with future releases. """) @@ -331,7 +331,7 @@ defmodule Pleroma.Config.DeprecationWarnings do "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`" end) - Logger.warn(Enum.join([warning_preface | pool_warnings])) + Logger.warning(Enum.join([warning_preface | pool_warnings])) Config.put(:pools, updated_config) :error diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex index 483d2bb79..836f0c1a7 100644 --- a/lib/pleroma/config/oban.ex +++ b/lib/pleroma/config/oban.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Config.Oban do You are using old workers in Oban crontab settings, which were removed. Please, remove setting from crontab in your config file (prod.secret.exs): #{inspect(setting)} """ - |> Logger.warn() + |> Logger.warning() List.delete(acc, setting) else diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 6fd05b0e0..91885347f 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -144,7 +144,7 @@ defmodule Pleroma.Config.TransferTask do error_msg = "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{inspect(value)} error: #{inspect(error)}" - Logger.warn(error_msg) + Logger.warning(error_msg) nil end @@ -178,12 +178,12 @@ defmodule Pleroma.Config.TransferTask do :ok = Application.start(app) else nil -> - Logger.warn("#{app} is not started.") + Logger.warning("#{app} is not started.") error -> error |> inspect() - |> Logger.warn() + |> Logger.warning() end end diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index 97d4b8f70..eb6f6816b 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Emoji.Loader do Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") if not Enum.empty?(files) do - Logger.warn( + Logger.warning( "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{Enum.join(files, ", ")}" ) end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 7c5785def..804cd11c7 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) @@ -90,7 +90,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) @@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 252a6aba5..e9bb2023a 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -70,15 +70,15 @@ defmodule Pleroma.HTTP.AdapterHelper do {:ok, parse_host(host), port} else {_, _} -> - Logger.warn("Parsing port failed #{inspect(proxy)}") + Logger.warning("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} :error -> - Logger.warn("Parsing port failed #{inspect(proxy)}") + Logger.warning("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") + Logger.warning("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end @@ -88,7 +88,7 @@ defmodule Pleroma.HTTP.AdapterHelper do {:ok, type, parse_host(host), port} else _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") + Logger.warning("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 101e5dc88..c497a4fb7 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -173,7 +173,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") + Logger.warning("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") nil end @@ -201,7 +201,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn( + Logger.warning( "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" ) @@ -284,7 +284,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn( + Logger.warning( "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" ) diff --git a/lib/pleroma/maintenance.ex b/lib/pleroma/maintenance.ex index eb5a6ef42..1e39b03e6 100644 --- a/lib/pleroma/maintenance.ex +++ b/lib/pleroma/maintenance.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Maintenance do "full" -> Logger.info("Running VACUUM FULL.") - Logger.warn( + Logger.warning( "Re-packing your entire database may take a while and will consume extra disk space during the process." ) diff --git a/lib/pleroma/migrators/support/base_migrator.ex b/lib/pleroma/migrators/support/base_migrator.ex index 3bcd59fd0..ce88caac7 100644 --- a/lib/pleroma/migrators/support/base_migrator.ex +++ b/lib/pleroma/migrators/support/base_migrator.ex @@ -73,7 +73,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do data_migration.state == :manual or data_migration.name in manual_migrations -> message = "Data migration is in manual execution or manual fix mode." update_status(:manual, message) - Logger.warn("#{__MODULE__}: #{message}") + Logger.warning("#{__MODULE__}: #{message}") data_migration.state == :complete -> on_complete(data_migration) @@ -109,7 +109,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`. """ - Logger.warn("#{__MODULE__}: #{message}") + Logger.warning("#{__MODULE__}: #{message}") update_status(:manual, message) on_complete(data_migration()) @@ -125,7 +125,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do defp on_complete(data_migration) do if data_migration.feature_lock || feature_state() == :disabled do - Logger.warn( + Logger.warning( "#{__MODULE__}: migration complete but feature is locked; consider enabling." ) diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 2248c2713..880940d07 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -192,7 +192,7 @@ defmodule Pleroma.ReverseProxy do halt(conn) {:error, error, conn} -> - Logger.warn( + Logger.warning( "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}" ) diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 384c70fbc..92d395394 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Telemetry.Logger do %{key: key}, _ ) do - Logger.warn(fn -> + Logger.warning(fn -> "Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{inspect(reason)}" end) end diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index e510ae3e6..7ee643277 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {:ok, :filtered, upload} rescue e in ErlangError -> - Logger.warn("#{__MODULE__}: #{inspect(e)}") + Logger.warning("#{__MODULE__}: #{inspect(e)}") {:ok, :noop} end end @@ -46,7 +46,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {:ok, :filtered, upload} rescue e in ErlangError -> - Logger.warn("#{__MODULE__}: #{inspect(e)}") + Logger.warning("#{__MODULE__}: #{inspect(e)}") {:ok, :noop} end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ddeaf0624..5f98935b3 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1560,7 +1560,7 @@ defmodule Pleroma.User do unmute(muter, mutee) else {who, result} = error -> - Logger.warn( + Logger.warning( "User.unmute/2 failed. #{who}: #{result}, muter_id: #{muter_id}, mutee_id: #{mutee_id}" ) diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 04dc8e0ce..74e0ec073 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -293,7 +293,7 @@ defmodule Pleroma.User.Backup do acc + 1 else {:error, e} -> - Logger.warn( + Logger.warning( "Error processing backup item: #{inspect(e)}\n The item is: #{inspect(i)}" ) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3b2193ca3..3f76531c6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -475,7 +475,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json(message) e -> - Logger.warn(fn -> "AP C2S: #{inspect(e)}" end) + Logger.warning(fn -> "AP C2S: #{inspect(e)}" end) conn |> put_status(:bad_request) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 69054e25b..7f6dce925 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -203,7 +203,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do if Enum.all?(@required_description_keys, &Map.has_key?(description, &1)) do [description | acc] else - Logger.warn( + Logger.warning( "#{policy} config description doesn't have one or all required keys #{inspect(@required_description_keys)}" ) diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 5b6adbb4b..5a4a97626 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do try_follow(follower, message) else nil -> - Logger.warn( + Logger.warning( "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname account does not exist, or the account is not correctly configured as a bot." ) diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index f66c379b5..28c2cf3b3 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do shortcode e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") + Logger.warning("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") nil end else @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do end else e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") + Logger.warning("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") nil end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 86d3ac60f..35f3aea03 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -156,7 +156,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.drop(["conversation", "inReplyToAtomUri"]) else e -> - Logger.warn("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") + Logger.warning("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") object end else @@ -182,7 +182,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object e -> - Logger.warn("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") + Logger.warning("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") object end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 82c7f70d2..dfc0a625d 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -550,7 +550,7 @@ defmodule Pleroma.Web.CommonAPI do remove_mute(user, activity) else {what, result} = error -> - Logger.warn( + Logger.warning( "CommonAPI.remove_mute/2 failed. #{what}: #{result}, user_id: #{user_id}, activity_id: #{activity_id}" ) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 0f394e951..dcda3e0e8 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -321,13 +321,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do format_asctime(date) else _e -> - Logger.warn("Date #{date} in wrong format, must be ISO 8601") + Logger.warning("Date #{date} in wrong format, must be ISO 8601") "" end end def date_to_asctime(date) do - Logger.warn("Date #{date} in wrong format, must be ISO 8601") + Logger.warning("Date #{date} in wrong format, must be ISO 8601") "" end diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 5093414c4..a27dcd0ab 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -201,7 +201,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do def warn_if_disabled do unless Config.get([:http_security, :enabled]) do - Logger.warn(" + Logger.warning(" .i;;;;i. iYcviii;vXY: .YXi .i1c. diff --git a/lib/pleroma/web/plugs/rate_limiter.ex b/lib/pleroma/web/plugs/rate_limiter.ex index 2080b06bd..aa79dbf6b 100644 --- a/lib/pleroma/web/plugs/rate_limiter.ex +++ b/lib/pleroma/web/plugs/rate_limiter.ex @@ -89,7 +89,7 @@ defmodule Pleroma.Web.Plugs.RateLimiter do end defp handle_disabled(conn) do - Logger.warn( + Logger.warning( "Rate limiter disabled due to forwarded IP not being found. Please ensure your reverse proxy is providing the X-Forwarded-For header or disable the RemoteIP plug/rate limiter." ) diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex index 9665b0b4a..0d43f402e 100644 --- a/lib/pleroma/web/push.ex +++ b/lib/pleroma/web/push.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.Push do def init do unless enabled() do - Logger.warn(""" + Logger.warning(""" VAPID key pair is not found. If you wish to enabled web push, please run mix web_push.gen.keypair diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 3c5f00764..36f44d8e8 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -57,7 +57,7 @@ defmodule Pleroma.Web.Push.Impl do end def perform(_) do - Logger.warn("Unknown notification type") + Logger.warning("Unknown notification type") {:error, :unknown_type} end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index dbe81eabb..c37c45963 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -75,7 +75,7 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp log_error(url, reason) do - Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) + Logger.warning(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) end end diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index f95dc2458..9466ad430 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -163,7 +163,7 @@ defmodule Pleroma.Web.WebFinger do get_template_from_xml(body) else error -> - Logger.warn("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}") + Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}") {:error, :lrdd_not_found} end end -- cgit v1.2.3 From 39d3df86c8e2ee05d409865ebc866c543a604ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 21 Dec 2023 00:10:30 +0100 Subject: Use consistent terminology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 +- lib/pleroma/web/web_finger.ex | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a0d4aca66..523b362d6 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2252,7 +2252,7 @@ defmodule Pleroma.User do if String.contains?(user.nickname, "@") do user.nickname else - host = Pleroma.Web.WebFinger.domain() + host = Pleroma.Web.WebFinger.host() user.nickname <> "@" <> host end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index aef02a418..ba7836e51 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do instance = Config.get(:instance) %{ - uri: Pleroma.Web.WebFinger.domain(), + uri: Pleroma.Web.WebFinger.host(), title: Keyword.get(instance, :name), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index b28fad8d1..c6840f10b 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Web.WebFinger do def represent_user(user, "JSON") do %{ - "subject" => "acct:#{user.nickname}@#{domain()}", + "subject" => "acct:#{user.nickname}@#{host()}", "aliases" => gather_aliases(user), "links" => gather_links(user) } @@ -90,13 +90,13 @@ defmodule Pleroma.Web.WebFinger do :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{domain()}"} + {:Subject, "acct:#{user.nickname}@#{host()}"} ] ++ aliases ++ links } |> XmlBuilder.to_doc() end - def domain do + def host do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end -- cgit v1.2.3 From 4f2fb8dc5e2b4ca88d42565ccd811e1028e8c720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 21 Dec 2023 00:13:13 +0100 Subject: Use consistent terminology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 4 ++-- lib/pleroma/web/web_finger.ex | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index e03025a00..f95b5360a 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do common_information(instance) |> Map.merge(%{ - uri: Pleroma.Web.WebFinger.domain(), + uri: Pleroma.Web.WebFinger.host(), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), email: Keyword.get(instance, :email), @@ -49,7 +49,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do common_information(instance) |> Map.merge(%{ - domain: Pleroma.Web.WebFinger.domain(), + domain: Pleroma.Web.WebFinger.host(), source_url: Pleroma.Application.repository(), description: Keyword.get(instance, :short_description), usage: %{users: %{active_month: Pleroma.User.active_user_count()}}, diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index b28fad8d1..c6840f10b 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Web.WebFinger do def represent_user(user, "JSON") do %{ - "subject" => "acct:#{user.nickname}@#{domain()}", + "subject" => "acct:#{user.nickname}@#{host()}", "aliases" => gather_aliases(user), "links" => gather_links(user) } @@ -90,13 +90,13 @@ defmodule Pleroma.Web.WebFinger do :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{domain()}"} + {:Subject, "acct:#{user.nickname}@#{host()}"} ] ++ aliases ++ links } |> XmlBuilder.to_doc() end - def domain do + def host do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end -- cgit v1.2.3 From ea0ec5fbcf2e17d97f323bcb6872df48d5e42714 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 14:20:33 -0500 Subject: Remove Fetcher.fetch_object_from_id!/2 It was only being called once and can be replaced with a case statement. --- lib/pleroma/object.ex | 5 ++++- lib/pleroma/object/fetcher.ex | 26 +++++--------------------- 2 files changed, 9 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index fa5baf1a4..f2ba24abf 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -177,7 +177,10 @@ defmodule Pleroma.Object do ap_id Keyword.get(options, :fetch) -> - Fetcher.fetch_object_from_id!(ap_id, options) + case Fetcher.fetch_object_from_id(ap_id, options) do + {:ok, object} -> object + _ -> nil + end true -> get_cached_by_ap_id(ap_id) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index cc3772563..508ee7e62 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -79,9 +79,11 @@ defmodule Pleroma.Object.Fetcher do {:error, "Object containment failed."} {:transmogrifier, {:error, {:reject, e}}} -> + Logger.info("Rejected #{id} while fetching: #{inspect(e)}") {:reject, e} {:transmogrifier, {:reject, e}} -> + Logger.info("Rejected #{id} while fetching: #{inspect(e)}") {:reject, e} {:transmogrifier, _} = e -> @@ -97,10 +99,12 @@ defmodule Pleroma.Object.Fetcher do {:ok, object} {:fetch, {:error, error}} -> + Logger.error("Error while fetching #{id}: #{inspect(error)}") {:error, error} e -> - e + Logger.error("Error while fetching #{id}: #{inspect(e)}") + {:error, e} end end @@ -117,26 +121,6 @@ defmodule Pleroma.Object.Fetcher do |> Maps.put_if_present("bcc", data["bcc"]) end - def fetch_object_from_id!(id, options \\ []) do - with {:ok, object} <- fetch_object_from_id(id, options) do - object - else - {:error, %Tesla.Mock.Error{}} -> - nil - - {:error, "Object has been deleted"} -> - nil - - {:reject, reason} -> - Logger.info("Rejected #{id} while fetching: #{inspect(reason)}") - nil - - e -> - Logger.error("Error while fetching #{id}: #{inspect(e)}") - nil - end - end - defp make_signature(id, date) do uri = URI.parse(id) -- cgit v1.2.3 From 603e9f6a92e8ea2d2d34d2af26ec2b12fe4b8543 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 14:22:04 -0500 Subject: Fix Transmogrifier tests These tests relied on the removed Fetcher.fetch_object_from_id!/2 function injecting the error tuple into a log message with the exact words "Object containment failed." We will keep this behavior by generating a similar log message, but perhaps this should do a better job of matching on the error tuple returned by Transmogrifier.handle_incoming/1 --- lib/pleroma/object/fetcher.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 508ee7e62..13fb50639 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -75,8 +75,9 @@ defmodule Pleroma.Object.Fetcher do {:allowed_depth, false} -> {:error, "Max thread distance exceeded."} - {:containment, _} -> - {:error, "Object containment failed."} + {:containment, e} -> + Logger.info("Error while fetching #{id}: Object containment failed. #{inspect(e)}") + {:error, e} {:transmogrifier, {:error, {:reject, e}}} -> Logger.info("Rejected #{id} while fetching: #{inspect(e)}") -- cgit v1.2.3 From d472bafec19cee269e7c943bafae7c805785acd7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 15:54:14 -0500 Subject: Mark instances as unreachable when returning a 403 from an object fetch This is a definite sign the instance is blocked and they are enforcing authorized_fetch --- lib/pleroma/object/fetcher.ex | 9 +++++++++ lib/pleroma/workers/remote_fetcher_worker.ex | 11 +++++++++++ 2 files changed, 20 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 13fb50639..a07b35d6f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -99,6 +99,15 @@ defmodule Pleroma.Object.Fetcher do {:fetch_object, %Object{} = object} -> {:ok, object} + {:fetch, {:error, {:ok, %Tesla.Env{status: 403}}}} -> + Instances.set_consistently_unreachable(id) + + Logger.error( + "Error while fetching #{id}: HTTP 403 likely due to instance block rejecting the signed fetch." + ) + + {:error, "Object fetch has been denied"} + {:fetch, {:error, error}} -> Logger.error("Error while fetching #{id}: #{inspect(error)}") {:error, error} diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index d2a77aa17..70d074c1c 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -10,6 +10,17 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do {:ok, _object} = Fetcher.fetch_object_from_id(id, depth: args["depth"]) + + case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do + {:ok, _object} -> + :ok + + {:error, reason = "Object fetch has been denied"} -> + {:cancel, reason} + + _ -> + :error + end end @impl Oban.Worker -- cgit v1.2.3 From 67dd81e8254576950fc20ed84d3da3a3d24c1706 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 16:05:28 -0500 Subject: Consolidate the HTTP status code checking into the private get_object/1 --- lib/pleroma/object/fetcher.ex | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index a07b35d6f..75f39fb6a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -99,15 +99,6 @@ defmodule Pleroma.Object.Fetcher do {:fetch_object, %Object{} = object} -> {:ok, object} - {:fetch, {:error, {:ok, %Tesla.Env{status: 403}}}} -> - Instances.set_consistently_unreachable(id) - - Logger.error( - "Error while fetching #{id}: HTTP 403 likely due to instance block rejecting the signed fetch." - ) - - {:error, "Object fetch has been denied"} - {:fetch, {:error, error}} -> Logger.error("Error while fetching #{id}: #{inspect(error)}") {:error, error} @@ -221,6 +212,10 @@ defmodule Pleroma.Object.Fetcher do {:error, {:content_type, nil}} end + {:ok, %{status: 403}} -> + Instances.set_consistently_unreachable(id) + {:error, "Object fetch has been denied"} + {:ok, %{status: code}} when code in [404, 410] -> {:error, "Object has been deleted"} -- cgit v1.2.3 From c6b38441f15a4f3bb925bc4ffc68ac8930bc29b2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 16:05:44 -0500 Subject: Cancel remote fetch jobs for deleted objects --- lib/pleroma/workers/remote_fetcher_worker.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 70d074c1c..1a7bd2586 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -18,6 +18,9 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:error, reason = "Object fetch has been denied"} -> {:cancel, reason} + {:error, reason = "Object has been deleted"} -> + {:cancel, reason} + _ -> :error end -- cgit v1.2.3 From 5f6966cd9ff9c728d97a8e4075cba738ceca0aeb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 16:23:41 -0500 Subject: Remove mistaken duplicate fetch --- lib/pleroma/workers/remote_fetcher_worker.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 1a7bd2586..ff0286e54 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do - {:ok, _object} = Fetcher.fetch_object_from_id(id, depth: args["depth"]) - case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do {:ok, _object} -> :ok -- cgit v1.2.3 From 9c0040124a9ea68cedca4959d32105ee6a6c3ee1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 16:28:05 -0500 Subject: Skip remote fetch jobs for unreachable instances --- lib/pleroma/workers/remote_fetcher_worker.ex | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index ff0286e54..d4fae33df 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -3,24 +3,29 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.RemoteFetcherWorker do + alias Pleroma.Instances alias Pleroma.Object.Fetcher use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do - case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do - {:ok, _object} -> - :ok + if Instances.reachable?(id) do + case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do + {:ok, _object} -> + :ok - {:error, reason = "Object fetch has been denied"} -> - {:cancel, reason} + {:error, reason = "Object fetch has been denied"} -> + {:cancel, reason} - {:error, reason = "Object has been deleted"} -> - {:cancel, reason} + {:error, reason = "Object has been deleted"} -> + {:cancel, reason} - _ -> - :error + _ -> + :error + end + else + {:cancel, "Unreachable instance"} end end -- cgit v1.2.3 From 73c4c6d7de6d33c68cf663e65df8525ce8eef4f5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 17:12:58 -0500 Subject: Revert "Mark instances as unreachable when returning a 403 from an object fetch" This reverts commit d472bafec19cee269e7c943bafae7c805785acd7. --- lib/pleroma/object/fetcher.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 75f39fb6a..f1a0a483b 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -213,7 +213,6 @@ defmodule Pleroma.Object.Fetcher do end {:ok, %{status: 403}} -> - Instances.set_consistently_unreachable(id) {:error, "Object fetch has been denied"} {:ok, %{status: code}} when code in [404, 410] -> -- cgit v1.2.3 From 7a58ddfa486048d17fa653662914e0f02d11fadb Mon Sep 17 00:00:00 2001 From: tusooa Date: Sun, 5 Nov 2023 18:49:31 -0500 Subject: Allow local user to have group actor type https://git.pleroma.social/pleroma/pleroma/-/issues/3205 --- lib/pleroma/constants.ex | 8 ++++++++ lib/pleroma/user.ex | 3 ++- lib/pleroma/web/mastodon_api/views/account_view.ex | 10 +++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 77bc4bfac..d814b4931 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -76,6 +76,14 @@ defmodule Pleroma.Constants do ] ) + const(allowed_user_actor_types, + do: [ + "Person", + "Service", + "Group" + ] + ) + # basic regex, just there to weed out potential mistakes # https://datatracker.ietf.org/doc/html/rfc2045#section-5.1 const(mime_regex, diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 10dafbe6f..0fd1b6365 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -39,6 +39,7 @@ defmodule Pleroma.User do alias Pleroma.Workers.BackgroundWorker require Logger + require Pleroma.Constants @type t :: %__MODULE__{} @type account_status :: @@ -579,7 +580,7 @@ defmodule Pleroma.User do |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) - |> validate_inclusion(:actor_type, ["Person", "Service"]) + |> validate_inclusion(:actor_type, Pleroma.Constants.allowed_user_actor_types()) |> put_fields() |> put_emoji() |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 237de3055..e7c555eb2 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -212,7 +212,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do do: user.follower_count, else: 0 - bot = user.actor_type == "Service" + bot = is_bot?(user) emojis = Enum.map(user.emoji, fn {shortcode, raw_url} -> @@ -468,4 +468,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp image_url(%{"url" => [%{"href" => href} | _]}), do: href defp image_url(_), do: nil + + defp is_bot?(user) do + # Because older and/or Mastodon clients may not recognize a Group actor properly, + # and currently the group actor can only boost things, we should let these clients + # think groups are bots. + # See https://git.pleroma.social/pleroma/pleroma-meta/-/issues/14 + user.actor_type == "Service" || user.actor_type == "Group" + end end -- cgit v1.2.3 From 5459bbc1efabc373e5619990be8d811df072e5e6 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 6 Nov 2023 19:59:05 -0500 Subject: Allow group actors to boost posts --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + lib/pleroma/web/activity_pub/side_effects.ex | 2 ++ lib/pleroma/web/activity_pub/utils.ex | 17 +++++++++++++++++ 3 files changed, 20 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 32d1a1037..3b0140d96 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -319,6 +319,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, _actor} <- update_last_status_at_if_public(actor, activity), _ <- notify_and_stream(activity), :ok <- maybe_schedule_poll_notifications(activity), + :ok <- maybe_handle_group_posts(activity), :ok <- maybe_federate(activity) do {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 10f268f05..59b19180d 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -233,6 +233,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + Utils.maybe_handle_group_posts(activity) + meta = meta |> add_notifications(notifications) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index b32f19740..e4edbd5ee 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -935,4 +935,21 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> where([a, object: o], fragment("(?)->>'type' = 'Answer'", o.data)) |> Repo.all() end + + def maybe_handle_group_posts(activity) do + mentions = + activity.data["to"] + |> Enum.filter(&(&1 != activity.actor)) + + mentioned_local_groups = + User.get_all_by_ap_id(mentions) + |> Enum.filter(&(&1.actor_type == "Group" and &1.local)) + + mentioned_local_groups + |> Enum.each(fn group -> + Pleroma.Web.CommonAPI.repeat(activity.id, group) + end) + + :ok + end end -- cgit v1.2.3 From e34a975dd946cc609638d85c20a57e2bfed6ebc7 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 7 Nov 2023 21:16:24 -0500 Subject: Do not boost if group is blocking poster --- lib/pleroma/web/activity_pub/utils.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index e4edbd5ee..e2fc2640d 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -937,13 +937,19 @@ defmodule Pleroma.Web.ActivityPub.Utils do end def maybe_handle_group_posts(activity) do + poster = User.get_cached_by_ap_id(activity.actor) + mentions = activity.data["to"] |> Enum.filter(&(&1 != activity.actor)) mentioned_local_groups = User.get_all_by_ap_id(mentions) - |> Enum.filter(&(&1.actor_type == "Group" and &1.local)) + |> Enum.filter(fn user -> + user.actor_type == "Group" and + user.local and + not User.blocks?(user, poster) + end) mentioned_local_groups |> Enum.each(fn group -> -- cgit v1.2.3 From b273025fd7612eb869c5a14a8cca81cbc45bad42 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 27 Dec 2023 12:21:05 -0500 Subject: Add pleroma:group_actors to instance features --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f95b5360a..e514d0e91 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -125,7 +125,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do if Config.get([:instance, :profile_directory]) do "profile_directory" end, - "pleroma:get:main/ostatus" + "pleroma:get:main/ostatus", + "pleroma:group_actors" ] |> Enum.filter(& &1) end -- cgit v1.2.3 From 1a337dcc18f3dabb97bf480f5569e8787e5ce2cf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 12:43:12 -0500 Subject: These functions in Pleroma.Instances should be defdelegates to Pleroma.Instances.Instance --- lib/pleroma/instances.ex | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 782948f83..b6d83f591 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -7,16 +7,15 @@ defmodule Pleroma.Instances do alias Pleroma.Instances.Instance - def filter_reachable(urls_or_hosts), do: Instance.filter_reachable(urls_or_hosts) + defdelegate filter_reachable(urls_or_hosts), to: Instance - def reachable?(url_or_host), do: Instance.reachable?(url_or_host) + defdelegate reachable?(url_or_host), to: Instance - def set_reachable(url_or_host), do: Instance.set_reachable(url_or_host) + defdelegate set_reachable(url_or_host), to: Instance - def set_unreachable(url_or_host, unreachable_since \\ nil), - do: Instance.set_unreachable(url_or_host, unreachable_since) + defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance - def get_consistently_unreachable, do: Instance.get_consistently_unreachable() + defdelegate get_consistently_unreachable, to: Instance def set_consistently_unreachable(url_or_host), do: set_unreachable(url_or_host, reachability_datetime_threshold()) -- cgit v1.2.3 From f6fee39e42e633ff4298291dca0db656c92dad81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 23 Dec 2023 15:51:20 +0100 Subject: Add changelog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec.ex | 1 + .../web/api_spec/operations/instance_operation.ex | 2 +- lib/pleroma/web/mastodon_api/views/instance_view.ex | 18 +++++++++--------- 3 files changed, 11 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 163226ce5..0eedf4aea 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -97,6 +97,7 @@ defmodule Pleroma.Web.ApiSpec do "Frontend managment", "Instance configuration", "Instance documents", + "Instance rule managment", "Invites", "MediaProxy cache", "OAuth application managment", diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index a22eb5bc9..452e56e45 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -48,7 +48,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do def rules_operation do %Operation{ - tags: ["Instance"], + tags: ["Instance misc"], summary: "Retrieve list of instance rules", operationId: "InstanceController.rules", responses: %{ diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index fa3726d4a..62c6b9e2e 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -73,15 +73,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }) end - defp common_information(instance) do - %{ - languages: Keyword.get(instance, :languages, ["en"]), - rules: render(__MODULE__, "rules.json"), - title: Keyword.get(instance, :name), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})" - } - end - def render("rules.json", _) do Pleroma.Rule.query() |> Pleroma.Repo.all() @@ -95,6 +86,15 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do } end + defp common_information(instance) do + %{ + languages: Keyword.get(instance, :languages, ["en"]), + rules: render(__MODULE__, "rules.json"), + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})" + } + end + def features do [ "pleroma_api", -- cgit v1.2.3 From d4c77103d1fe5df9b2ea8bd3429a8fc0240645c1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 15:27:37 -0500 Subject: Fix detection of user follower collection being private We were overzealous with matching on a raw error from the object fetch that should have never been relied on like this. If we can't fetch successfully we should assume that the collection is private. Building a more expressive and universal error struct to match on may be something to consider. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 32d1a1037..39159b1c4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1697,9 +1697,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Fetcher.fetch_and_contain_remote_object_from_id(first) do {:ok, false} else - {:error, {:ok, %{status: code}}} when code in [401, 403] -> {:ok, true} - {:error, _} = e -> e - e -> {:error, e} + {:error, _} -> {:ok, true} end end -- cgit v1.2.3 From f53197c82a90533c9152d7d8ed57c2604a2d6685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 27 Dec 2023 23:52:46 +0100 Subject: Fix operation name typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec.ex | 6 +++--- .../web/api_spec/operations/admin/frontend_operation.ex | 4 ++-- .../web/api_spec/operations/admin/o_auth_app_operation.ex | 8 ++++---- lib/pleroma/web/api_spec/operations/admin/report_operation.ex | 10 +++++----- 4 files changed, 14 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 163226ce5..8a759185b 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -94,14 +94,14 @@ defmodule Pleroma.Web.ApiSpec do "tags" => [ "Chat administration", "Emoji pack administration", - "Frontend managment", + "Frontend management", "Instance configuration", "Instance documents", "Invites", "MediaProxy cache", - "OAuth application managment", + "OAuth application management", "Relays", - "Report managment", + "Report management", "Status administration", "User administration", "Announcement management" diff --git a/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex b/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex index 3e85c44d2..e17881b49 100644 --- a/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do def index_operation do %Operation{ - tags: ["Frontend managment"], + tags: ["Frontend management"], summary: "Retrieve a list of available frontends", operationId: "AdminAPI.FrontendController.index", security: [%{"oAuth" => ["admin:read"]}], @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do def install_operation do %Operation{ - tags: ["Frontend managment"], + tags: ["Frontend management"], summary: "Install a frontend", operationId: "AdminAPI.FrontendController.install", security: [%{"oAuth" => ["admin:read"]}], diff --git a/lib/pleroma/web/api_spec/operations/admin/o_auth_app_operation.ex b/lib/pleroma/web/api_spec/operations/admin/o_auth_app_operation.ex index 1a05aff6a..2b2496c26 100644 --- a/lib/pleroma/web/api_spec/operations/admin/o_auth_app_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/o_auth_app_operation.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do def index_operation do %Operation{ summary: "Retrieve a list of OAuth applications", - tags: ["OAuth application managment"], + tags: ["OAuth application management"], operationId: "AdminAPI.OAuthAppController.index", security: [%{"oAuth" => ["admin:write"]}], parameters: [ @@ -69,7 +69,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do def create_operation do %Operation{ - tags: ["OAuth application managment"], + tags: ["OAuth application management"], summary: "Create an OAuth application", operationId: "AdminAPI.OAuthAppController.create", requestBody: request_body("Parameters", create_request()), @@ -84,7 +84,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do def update_operation do %Operation{ - tags: ["OAuth application managment"], + tags: ["OAuth application management"], summary: "Update OAuth application", operationId: "AdminAPI.OAuthAppController.update", parameters: [id_param() | admin_api_params()], @@ -102,7 +102,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do def delete_operation do %Operation{ - tags: ["OAuth application managment"], + tags: ["OAuth application management"], summary: "Delete OAuth application", operationId: "AdminAPI.OAuthAppController.delete", parameters: [id_param() | admin_api_params()], diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index 312e091a5..d7a74f665 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def index_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Retrieve a list of reports", operationId: "AdminAPI.ReportController.index", security: [%{"oAuth" => ["admin:read:reports"]}], @@ -69,7 +69,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def show_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Retrieve a report", operationId: "AdminAPI.ReportController.show", parameters: [id_param() | admin_api_params()], @@ -83,7 +83,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def update_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Change state of specified reports", operationId: "AdminAPI.ReportController.update", security: [%{"oAuth" => ["admin:write:reports"]}], @@ -99,7 +99,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def notes_create_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Add a note to the report", operationId: "AdminAPI.ReportController.notes_create", parameters: [id_param() | admin_api_params()], @@ -120,7 +120,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def notes_delete_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Delete note attached to the report", operationId: "AdminAPI.ReportController.notes_delete", parameters: [ -- cgit v1.2.3 From 017e35fbf128d47c033275a70b76b72f24d7c754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 28 Dec 2023 00:15:32 +0100 Subject: Fix some more typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/mix/tasks/pleroma/database.ex | 2 +- lib/mix/tasks/pleroma/digest.ex | 2 +- lib/mix/tasks/pleroma/ecto/rollback.ex | 2 +- lib/mix/tasks/pleroma/instance.ex | 2 +- lib/pleroma/captcha/kocaptcha.ex | 2 +- lib/pleroma/config/deprecation_warnings.ex | 8 ++++---- lib/pleroma/docs/generator.ex | 2 +- lib/pleroma/search/search_backend.ex | 2 +- lib/pleroma/user/query.ex | 2 +- lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex | 2 +- lib/pleroma/web/api_spec.ex | 2 +- lib/pleroma/web/api_spec/operations/account_operation.ex | 2 +- lib/pleroma/web/api_spec/operations/instance_operation.ex | 4 ++-- lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex | 2 +- lib/pleroma/web/api_spec/operations/status_operation.ex | 4 ++-- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 6 +++--- lib/pleroma/web/api_spec/schemas/attachment.ex | 2 +- lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 2 +- lib/pleroma/web/o_auth/o_auth_controller.ex | 2 +- lib/pleroma/web/plugs/cache.ex | 2 +- lib/pleroma/web/plugs/uploaded_media.ex | 2 +- lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/streamer.ex | 2 +- lib/pleroma/web/twitter_api/controllers/password_controller.ex | 2 +- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 4 ++-- 25 files changed, 33 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ed560c177..93ee57dc3 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -193,7 +193,7 @@ defmodule Mix.Tasks.Pleroma.Database do "ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';" ) - # non-exist config will not raise excpetion but only give >0 messages + # non-exist config will not raise exception but only give >0 messages if length(msg) > 0 do shell_info("Error: #{inspect(msg, pretty: true)}") else diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex index aea9c8ac5..53cac0b94 100644 --- a/lib/mix/tasks/pleroma/digest.ex +++ b/lib/mix/tasks/pleroma/digest.ex @@ -30,7 +30,7 @@ defmodule Mix.Tasks.Pleroma.Digest do shell_info("Digest email have been sent to #{nickname} (#{user.email})") else _ -> - shell_info("Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}") + shell_info("Couldn't find any mentions for #{nickname} since #{last_digest_emailed_at}") end end end diff --git a/lib/mix/tasks/pleroma/ecto/rollback.ex b/lib/mix/tasks/pleroma/ecto/rollback.ex index 3d78eaec4..121890f39 100644 --- a/lib/mix/tasks/pleroma/ecto/rollback.ex +++ b/lib/mix/tasks/pleroma/ecto/rollback.ex @@ -61,7 +61,7 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do Logger.configure(level: :info) if opts[:env] == "test" do - Logger.info("Rollback succesfully") + Logger.info("Rollback successfully") else {:ok, _, _} = Ecto.Migrator.with_repo(Pleroma.Repo, &Ecto.Migrator.run(&1, path, :down, opts)) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 5d8b254a2..fd1809a42 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -292,7 +292,7 @@ defmodule Mix.Tasks.Pleroma.Instance do if db_configurable? do shell_info( - " Please transfer your config to the database after running database migrations. Refer to \"Transfering the config to/from the database\" section of the docs for more information." + " Please transfer your config to the database after running database migrations. Refer to \"Transferring the config to/from the database\" section of the docs for more information." ) end else diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index e786e28b9..c4987d4fd 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Captcha.Kocaptcha do @impl Service def validate(_token, captcha, answer_data) do - # Here the token is unsed, because the unencrypted captcha answer is just passed to method + # Here the token is unused, because the unencrypted captcha answer is just passed to method if not is_nil(captcha) and :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(answer_data), do: :ok, diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 1cd3241ea..a77923264 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -172,7 +172,7 @@ defmodule Pleroma.Config.DeprecationWarnings do ``` config :pleroma, :mrf, - transparency_exclusions: [{"instance.tld", "Reason to exlude transparency"}] + transparency_exclusions: [{"instance.tld", "Reason to exclude transparency"}] ``` """) @@ -213,7 +213,7 @@ defmodule Pleroma.Config.DeprecationWarnings do check_gun_pool_options(), check_activity_expiration_config(), check_remote_ip_plug_name(), - check_uploders_s3_public_endpoint(), + check_uploaders_s3_public_endpoint(), check_old_chat_shoutbox(), check_quarantined_instances_tuples(), check_transparency_exclusions_tuples(), @@ -372,8 +372,8 @@ defmodule Pleroma.Config.DeprecationWarnings do ) end - @spec check_uploders_s3_public_endpoint() :: :ok | nil - def check_uploders_s3_public_endpoint do + @spec check_uploaders_s3_public_endpoint() :: :ok | nil + def check_uploaders_s3_public_endpoint do s3_config = Pleroma.Config.get([Pleroma.Uploaders.S3]) use_old_config = Keyword.has_key?(s3_config, :public_endpoint) diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 456a8fd54..93b19484f 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Docs.Generator do :code.all_loaded() |> Enum.filter(fn {module, _} -> # This shouldn't be needed as all modules are expected to have module_info/1, - # but in test enviroments some transient modules `:elixir_compiler_XX` + # but in test environments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). Code.ensure_loaded(module) diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index a42e2f5f6..46be84551 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Search.SearchBackend do Remove the object from the index. Just the object, as opposed to the whole activity, is passed, since the object - is what contains the actual content and there is no need for fitlering when removing + is what contains the actual content and there is no need for filtering when removing from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: {:ok, any()} | {:error, any()} diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 3e090cac0..874e8ec2b 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -22,7 +22,7 @@ defmodule Pleroma.User.Query do - pass non empty string - e.g. Pleroma.User.Query.build(%{email: "email@example.com"}) - *contains criteria* - - add field to @containns_criteria list + - add field to @contains_criteria list - pass values list - e.g. Pleroma.User.Query.build(%{ap_id: ["http://ap_id1", "http://ap_id2"]}) """ diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index b73fd974c..d13d980cc 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do alias Pleroma.Object @moduledoc """ - Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #) + Reject, TWKN-remove or Set-Sensitive messages with specific hashtags (without the leading #) Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. """ diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 8a759185b..3588608f2 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.ApiSpec do - [Mastodon API documentation](https://docs.joinmastodon.org/client/intro/) - [Differences in Mastodon API responses from vanilla Mastodon](https://docs-develop.pleroma.social/backend/development/API/differences_in_mastoapi_responses/) - Please report such occurences on our [issue tracker](https://git.pleroma.social/pleroma/pleroma/-/issues). Feel free to submit API questions or proposals there too! + Please report such occurrences on our [issue tracker](https://git.pleroma.social/pleroma/pleroma/-/issues). Feel free to submit API questions or proposals there too! """, # Strip environment from the version version: Application.spec(:pleroma, :vsn) |> to_string() |> String.replace(~r/\+.*$/, ""), diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index f2897a3a3..75cea2184 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -347,7 +347,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do summary: "Endorse", operationId: "AccountController.endorse", security: [%{"oAuth" => ["follow", "write:accounts"]}], - description: "Addds the given account to endorsed accounts list.", + description: "Adds the given account to endorsed accounts list.", parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}], responses: %{ 200 => Operation.response("Relationship", "application/json", AccountRelationship), diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 8e395bde8..708b74b12 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -101,7 +101,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do languages: %Schema{ type: :array, items: %Schema{type: :string}, - description: "Primary langauges of the website and its staff" + description: "Primary languages of the website and its staff" }, registrations: %Schema{type: :boolean, description: "Whether registrations are enabled"}, # Extra (not present in Mastodon): @@ -252,7 +252,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do languages: %Schema{ type: :array, items: %Schema{type: :string}, - description: "Primary langauges of the website and its staff" + description: "Primary languages of the website and its staff" }, registrations: %Schema{ type: :object, diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index 141b60533..f595583b6 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do security: [%{"oAuth" => ["write"]}], operationId: "PleromaAPI.ScrobbleController.create", deprecated: true, - requestBody: request_body("Parameters", create_request(), requried: true), + requestBody: request_body("Parameters", create_request(), required: true), responses: %{ 200 => Operation.response("Scrobble", "application/json", scrobble()) } diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index c133a3aac..f52372c18 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -534,7 +534,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do format: :"date-time", nullable: true, description: - "ISO 8601 Datetime at which to schedule a status. Providing this paramter will cause ScheduledStatus to be returned instead of Status. Must be at least 5 minutes in the future." + "ISO 8601 Datetime at which to schedule a status. Providing this parameter will cause ScheduledStatus to be returned instead of Status. Must be at least 5 minutes in the future." }, language: %Schema{ type: :string, @@ -546,7 +546,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do allOf: [BooleanLike], nullable: true, description: - "If set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example" + "If set to `true` the post won't be actually posted, but the status entity would still be rendered back. This could be useful for previewing rich text/custom emoji, for example" }, content_type: %Schema{ type: :string, diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 084329ad7..87af0d526 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -87,7 +87,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do defp change_password_request do %Schema{ title: "ChangePasswordRequest", - description: "POST body for changing the account's passowrd", + description: "POST body for changing the account's password", type: :object, required: [:password, :new_password, :new_password_confirmation], properties: %{ @@ -136,12 +136,12 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do } end - def update_notificaton_settings_operation do + def update_notification_settings_operation do %Operation{ tags: ["Settings"], summary: "Update Notification Settings", security: [%{"oAuth" => ["write:accounts"]}], - operationId: "UtilController.update_notificaton_settings", + operationId: "UtilController.update_notification_settings", parameters: [ Operation.parameter( :block_from_strangers, diff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex index 48634a14f..2871b5f99 100644 --- a/lib/pleroma/web/api_spec/schemas/attachment.ex +++ b/lib/pleroma/web/api_spec/schemas/attachment.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do title: "Attachment", description: "Represents a file or media attachment that can be added to a status.", type: :object, - requried: [:id, :url, :preview_url], + required: [:id, :url, :preview_url], properties: %{ id: %Schema{type: :string, description: "The ID of the attachment in the database."}, url: %Schema{ diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 9a4b56301..1b5de4b45 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -235,7 +235,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do # So we first build the normal local changeset, then apply it to the # user data, but don't persist it. With this, we generate the object # data for our update activity. We feed this and the changeset as meta - # inforation into the pipeline, where they will be properly updated and + # information into the pipeline, where they will be properly updated and # federated. with changeset <- User.update_changeset(user, user_params), {:ok, unpersisted_user} <- Ecto.Changeset.apply_action(changeset, :update), diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index c1fb4f378..2bbe5d5fa 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -310,7 +310,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do after_token_exchange(conn, %{token: token}) else _error -> - handle_token_exchange_error(conn, :invalid_credentails) + handle_token_exchange_error(conn, :invalid_credentials) end end diff --git a/lib/pleroma/web/plugs/cache.ex b/lib/pleroma/web/plugs/cache.ex index 667477857..5a7e86ef7 100644 --- a/lib/pleroma/web/plugs/cache.ex +++ b/lib/pleroma/web/plugs/cache.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.Plugs.Cache do - `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`. - `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`. - - `tracking_fun`: A function that is called on successfull responses, no matter if the request is cached or not. It should accept a conn as the first argument and the value assigned to `tracking_fun_data` as the second. + - `tracking_fun`: A function that is called on successful responses, no matter if the request is cached or not. It should accept a conn as the first argument and the value assigned to `tracking_fun_data` as the second. Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct: diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index 8b3bc9acb..f1076da1b 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -105,7 +105,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do end defp get_media(conn, unknown, _, _) do - Logger.error("#{__MODULE__}: Unknown get startegy: #{inspect(unknown)}") + Logger.error("#{__MODULE__}: Unknown get strategy: #{inspect(unknown)}") conn |> send_resp(:internal_server_error, dgettext("errors", "Internal Error")) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 22c85e34b..00268b121 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -481,7 +481,7 @@ defmodule Pleroma.Web.Router do post("/change_email", UtilController, :change_email) post("/change_password", UtilController, :change_password) post("/delete_account", UtilController, :delete_account) - put("/notification_settings", UtilController, :update_notificaton_settings) + put("/notification_settings", UtilController, :update_notification_settings) post("/disable_account", UtilController, :disable_account) post("/move_account", UtilController, :move_account) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 48ca82421..aaee79d8e 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -396,7 +396,7 @@ defmodule Pleroma.Web.Streamer do end end - # In test environement, only return true if the registry is started. + # In test environment, only return true if the registry is started. # In benchmark environment, returns false. # In any other environment, always returns true. cond do diff --git a/lib/pleroma/web/twitter_api/controllers/password_controller.ex b/lib/pleroma/web/twitter_api/controllers/password_controller.ex index 31b7dd728..e5482de9d 100644 --- a/lib/pleroma/web/twitter_api/controllers/password_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/password_controller.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.TwitterAPI.PasswordController do @moduledoc """ - The module containts functions for reset password. + The module contains functions for password reset. """ use Pleroma.Web, :controller diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index ca8a98960..50f8f803f 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -35,7 +35,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do :change_email, :change_password, :delete_account, - :update_notificaton_settings, + :update_notification_settings, :disable_account, :move_account, :add_alias, @@ -181,7 +181,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do json(conn, emoji) end - def update_notificaton_settings(%{assigns: %{user: user}} = conn, params) do + def update_notification_settings(%{assigns: %{user: user}} = conn, params) do with {:ok, _} <- User.update_notification_settings(user, params) do json(conn, %{status: "success"}) end -- cgit v1.2.3 From 6c9929b809290d9caca6a7a40eb0f59af6dd558d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 20:18:14 -0500 Subject: Set Logger level to error --- lib/pleroma/object/fetcher.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index f1a0a483b..9ba0ac0ca 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -76,15 +76,15 @@ defmodule Pleroma.Object.Fetcher do {:error, "Max thread distance exceeded."} {:containment, e} -> - Logger.info("Error while fetching #{id}: Object containment failed. #{inspect(e)}") + Logger.error("Error while fetching #{id}: Object containment failed. #{inspect(e)}") {:error, e} {:transmogrifier, {:error, {:reject, e}}} -> - Logger.info("Rejected #{id} while fetching: #{inspect(e)}") + Logger.error("Rejected #{id} while fetching: #{inspect(e)}") {:reject, e} {:transmogrifier, {:reject, e}} -> - Logger.info("Rejected #{id} while fetching: #{inspect(e)}") + Logger.error("Rejected #{id} while fetching: #{inspect(e)}") {:reject, e} {:transmogrifier, _} = e -> -- cgit v1.2.3 From becb070603bc65d5302698a2b6cf5f89b3d5c1f0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 20:47:18 -0500 Subject: Conslidate log messages for object fetcher failures and leverage Logger.metadata --- lib/pleroma/object/fetcher.ex | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 9ba0ac0ca..72420519a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -72,23 +72,25 @@ defmodule Pleroma.Object.Fetcher do {:object, data, Object.normalize(activity, fetch: false)} do {:ok, object} else - {:allowed_depth, false} -> + {:allowed_depth, false} = e -> + log_fetch_error(id, e) {:error, "Max thread distance exceeded."} - {:containment, e} -> - Logger.error("Error while fetching #{id}: Object containment failed. #{inspect(e)}") - {:error, e} + {:containment, reason} = e -> + log_fetch_error(id, e) + {:error, reason} - {:transmogrifier, {:error, {:reject, e}}} -> - Logger.error("Rejected #{id} while fetching: #{inspect(e)}") - {:reject, e} + {:transmogrifier, {:error, {:reject, reason}}} = e -> + log_fetch_error(id, e) + {:reject, reason} - {:transmogrifier, {:reject, e}} -> - Logger.error("Rejected #{id} while fetching: #{inspect(e)}") - {:reject, e} + {:transmogrifier, {:reject, reason}} = e -> + log_fetch_error(id, e) + {:reject, reason} - {:transmogrifier, _} = e -> - {:error, e} + {:transmogrifier, reason} = e -> + log_fetch_error(id, e) + {:error, reason} {:object, data, nil} -> reinject_object(%Object{}, data) @@ -99,16 +101,21 @@ defmodule Pleroma.Object.Fetcher do {:fetch_object, %Object{} = object} -> {:ok, object} - {:fetch, {:error, error}} -> - Logger.error("Error while fetching #{id}: #{inspect(error)}") - {:error, error} + {:fetch, {:error, reason}} = e -> + log_fetch_error(id, e) + {:error, reason} e -> - Logger.error("Error while fetching #{id}: #{inspect(e)}") + log_fetch_error(id, e) {:error, e} end end + defp log_fetch_error(id, error) do + Logger.metadata([object: id]) + Logger.error("Object rejected while fetching #{id} #{inspect(error)}") + end + defp prepare_activity_params(data) do %{ "type" => "Create", -- cgit v1.2.3 From 882267e3eceaa61cff16a9e0dee231d54a80d927 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 21:39:17 -0500 Subject: Remove duplicate log messages from Transmogrifier Object fetch errors are logged in the fetcher module --- lib/pleroma/web/activity_pub/transmogrifier.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 35f3aea03..80f64cf1e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -23,7 +23,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do import Ecto.Query - require Logger require Pleroma.Constants @doc """ @@ -155,8 +154,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("context", replied_object.data["context"] || object["conversation"]) |> Map.drop(["conversation", "inReplyToAtomUri"]) else - e -> - Logger.warning("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") + _ -> object end else @@ -181,8 +179,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:quoting?, _} -> object - e -> - Logger.warning("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") + _ -> object end end @@ -852,8 +849,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do relative_object do Map.put(data, "object", external_url) else - {:fetch, e} -> - Logger.error("Couldn't fetch #{object} #{inspect(e)}") + {:fetch, _} -> data _ -> -- cgit v1.2.3 From 287f2c9719310f96ce37b445f26e069f804c34b2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 21:55:07 -0500 Subject: Formatting --- lib/pleroma/object/fetcher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 72420519a..5bc1cfd5e 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -112,7 +112,7 @@ defmodule Pleroma.Object.Fetcher do end defp log_fetch_error(id, error) do - Logger.metadata([object: id]) + Logger.metadata(object: id) Logger.error("Object rejected while fetching #{id} #{inspect(error)}") end -- cgit v1.2.3 From a2708f7fe31c6009b0c9954e5de3b74c45f9818f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 21:57:47 -0500 Subject: Leverage existing atoms as return errors for the object fetcher --- lib/pleroma/object/fetcher.ex | 4 ++-- lib/pleroma/workers/remote_fetcher_worker.ex | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 5bc1cfd5e..0c90e0eaa 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -220,10 +220,10 @@ defmodule Pleroma.Object.Fetcher do end {:ok, %{status: 403}} -> - {:error, "Object fetch has been denied"} + {:error, :forbidden} {:ok, %{status: code}} when code in [404, 410] -> - {:error, "Object has been deleted"} + {:error, :not_found} {:error, e} -> {:error, e} diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index d4fae33df..dca4530d7 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -15,11 +15,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:ok, _object} -> :ok - {:error, reason = "Object fetch has been denied"} -> - {:cancel, reason} + {:error, :forbidden} -> + {:cancel, :forbidden} - {:error, reason = "Object has been deleted"} -> - {:cancel, reason} + {:error, :not_found} -> + {:cancel, :not_found} _ -> :error -- cgit v1.2.3 From ad0a5deb67f454b0529a4faf72399cd9ecc9c0e6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 22:28:41 -0500 Subject: Prevent requeuing Remote Fetcher jobs that exceed thread depth --- lib/pleroma/object/fetcher.ex | 2 +- lib/pleroma/workers/remote_fetcher_worker.ex | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 0c90e0eaa..972dfaf28 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -74,7 +74,7 @@ defmodule Pleroma.Object.Fetcher do else {:allowed_depth, false} = e -> log_fetch_error(id, e) - {:error, "Max thread distance exceeded."} + {:error, :allowed_depth} {:containment, reason} = e -> log_fetch_error(id, e) diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index dca4530d7..fef8a26b2 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -21,6 +21,9 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:error, :not_found} -> {:cancel, :not_found} + {:error, :allowed_depth} -> + {:cancel, :allowed_depth} + _ -> :error end -- cgit v1.2.3 From 1d816222e00741324fe3a068fb69f16675067a56 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 11:15:43 -0500 Subject: Remove support for multiple federation publisher modules This also unravels some needless indirection. --- lib/pleroma/web/federator.ex | 2 +- lib/pleroma/web/federator/publisher.ex | 46 ---------------------------------- lib/pleroma/web/nodeinfo/nodeinfo.ex | 2 +- lib/pleroma/web/web_finger.ex | 2 +- 4 files changed, 3 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 8621d984c..6deebb963 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -6,9 +6,9 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Activity alias Pleroma.Object.Containment alias Pleroma.User + alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Federator.Publisher alias Pleroma.Workers.PublisherWorker alias Pleroma.Workers.ReceiverWorker diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 8c6547208..03509bbaf 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator.Publisher do - alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.User alias Pleroma.Workers.PublisherWorker @@ -38,50 +36,6 @@ defmodule Pleroma.Web.Federator.Publisher do ) end - @doc """ - Relays an activity to all specified peers. - """ - @callback publish(User.t(), Activity.t()) :: :ok | {:error, any()} - - @spec publish(User.t(), Activity.t()) :: :ok - def publish(%User{} = user, %Activity{} = activity) do - Config.get([:instance, :federation_publisher_modules]) - |> Enum.each(fn module -> - if module.is_representable?(activity) do - Logger.debug("Publishing #{activity.data["id"]} using #{inspect(module)}") - module.publish(user, activity) - end - end) - - :ok - end - - @doc """ - Gathers links used by an outgoing federation module for WebFinger output. - """ - @callback gather_webfinger_links(User.t()) :: list() - - @spec gather_webfinger_links(User.t()) :: list() - def gather_webfinger_links(%User{} = user) do - Config.get([:instance, :federation_publisher_modules]) - |> Enum.reduce([], fn module, links -> - links ++ module.gather_webfinger_links(user) - end) - end - - @doc """ - Gathers nodeinfo protocol names supported by the federation module. - """ - @callback gather_nodeinfo_protocol_names() :: list() - - @spec gather_nodeinfo_protocol_names() :: list() - def gather_nodeinfo_protocol_names do - Config.get([:instance, :federation_publisher_modules]) - |> Enum.reduce([], fn module, links -> - links ++ module.gather_nodeinfo_protocol_names() - end) - end - @doc """ Gathers a set of remote users given an IR envelope. """ diff --git a/lib/pleroma/web/nodeinfo/nodeinfo.ex b/lib/pleroma/web/nodeinfo/nodeinfo.ex index 9e27ac26c..4d5a9a57f 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Nodeinfo.Nodeinfo do alias Pleroma.Config alias Pleroma.Stats alias Pleroma.User - alias Pleroma.Web.Federator.Publisher + alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.MastodonAPI.InstanceView # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 0684a770c..26fb8af84 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.HTTP alias Pleroma.User + alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.Endpoint - alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.XML alias Pleroma.XmlBuilder require Jason -- cgit v1.2.3 From 3acfdb6f8a8e242b87d7f81255c1c2ff8cd3171d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 14:53:09 -0500 Subject: Retire the Pleroma.Web.Federator.Publisher module --- lib/pleroma/web/activity_pub/publisher.ex | 48 ++++++++++++++++++++--- lib/pleroma/web/federator/publisher.ex | 64 ------------------------------- lib/pleroma/workers/publisher_worker.ex | 4 +- 3 files changed, 44 insertions(+), 72 deletions(-) delete mode 100644 lib/pleroma/web/federator/publisher.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index a580994b1..bb8209232 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -13,19 +13,56 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Workers.PublisherWorker require Pleroma.Constants import Pleroma.Web.ActivityPub.Visibility - @behaviour Pleroma.Web.Federator.Publisher - require Logger @moduledoc """ ActivityPub outgoing federation module. """ + @doc """ + Enqueue publishing a single activity. + """ + @spec enqueue_one(Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} + def enqueue_one(%{} = params, worker_args \\ []) do + PublisherWorker.enqueue( + "publish_one", + %{"params" => params}, + worker_args + ) + end + + @doc """ + Gathers a set of remote users given an IR envelope. + """ + def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do + cc = Map.get(data, "cc", []) + + bcc = + data + |> Map.get("bcc", []) + |> Enum.reduce([], fn ap_id, bcc -> + case Pleroma.List.get_by_ap_id(ap_id) do + %Pleroma.List{user_id: ^user_id} = list -> + {:ok, following} = Pleroma.List.get_following(list) + bcc ++ Enum.map(following, & &1.ap_id) + + _ -> + bcc + end + end) + + [to, cc, bcc] + |> Enum.concat() + |> Enum.map(&User.get_cached_by_ap_id/1) + |> Enum.filter(fn user -> user && !user.local end) + end + @doc """ Determine if an activity can be represented by running it through Transmogrifier. """ @@ -138,7 +175,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [] end - mentioned = Pleroma.Web.Federator.Publisher.remote_users(actor, activity) + mentioned = remote_users(actor, activity) non_mentioned = (followers ++ fetchers) -- mentioned [mentioned, non_mentioned] @@ -223,7 +260,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do |> Map.put("cc", cc) |> Jason.encode!() - Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ + __MODULE__.enqueue_one(%{ inbox: inbox, json: json, actor_id: actor.id, @@ -262,8 +299,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do inboxes |> Instances.filter_reachable() |> Enum.each(fn {inbox, unreachable_since} -> - Pleroma.Web.Federator.Publisher.enqueue_one( - __MODULE__, + __MODULE__.enqueue_one( %{ inbox: inbox, json: json, diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex deleted file mode 100644 index 03509bbaf..000000000 --- a/lib/pleroma/web/federator/publisher.ex +++ /dev/null @@ -1,64 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Federator.Publisher do - alias Pleroma.User - alias Pleroma.Workers.PublisherWorker - - require Logger - - @moduledoc """ - Defines the contract used by federation implementations to publish messages to - their peers. - """ - - @doc """ - Determine whether an activity can be relayed using the federation module. - """ - @callback is_representable?(Pleroma.Activity.t()) :: boolean() - - @doc """ - Relays an activity to a specified peer, determined by the parameters. The - parameters used are controlled by the federation module. - """ - @callback publish_one(Map.t()) :: {:ok, Map.t()} | {:error, any()} - - @doc """ - Enqueue publishing a single activity. - """ - @spec enqueue_one(module(), Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} - def enqueue_one(module, %{} = params, worker_args \\ []) do - PublisherWorker.enqueue( - "publish_one", - %{"module" => to_string(module), "params" => params}, - worker_args - ) - end - - @doc """ - Gathers a set of remote users given an IR envelope. - """ - def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do - cc = Map.get(data, "cc", []) - - bcc = - data - |> Map.get("bcc", []) - |> Enum.reduce([], fn ap_id, bcc -> - case Pleroma.List.get_by_ap_id(ap_id) do - %Pleroma.List{user_id: ^user_id} = list -> - {:ok, following} = Pleroma.List.get_following(list) - bcc ++ Enum.map(following, & &1.ap_id) - - _ -> - bcc - end - end) - - [to, cc, bcc] - |> Enum.concat() - |> Enum.map(&User.get_cached_by_ap_id/1) - |> Enum.filter(fn user -> user && !user.local end) - end -end diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index 598ae3779..63fcf4ac2 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -18,9 +18,9 @@ defmodule Pleroma.Workers.PublisherWorker do Federator.perform(:publish, activity) end - def perform(%Job{args: %{"op" => "publish_one", "module" => module_name, "params" => params}}) do + def perform(%Job{args: %{"op" => "publish_one", "params" => params}}) do params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end) - Federator.perform(:publish_one, String.to_atom(module_name), params) + Federator.perform(:publish_one, params) end @impl Oban.Worker -- cgit v1.2.3 From 08ba9a15b2b5fe6896197ab947bc59d57f4845c8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 22:51:47 -0500 Subject: Fix the Federator perform/2 Oban callback --- lib/pleroma/web/federator.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 6deebb963..1f2c3835a 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -68,10 +68,8 @@ defmodule Pleroma.Web.Federator do # Job Worker Callbacks - @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} - def perform(:publish_one, module, params) do - apply(module, :publish_one, [params]) - end + @spec perform(atom(), any()) :: {:ok, any()} | {:error, any()} + def perform(:publish_one, params), do: Publisher.publish_one(params) def perform(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) -- cgit v1.2.3 From efd50759d587977a52db01ef6ececc961c736604 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 22:59:27 -0500 Subject: Log errors when publishing activities --- lib/pleroma/web/activity_pub/publisher.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index bb8209232..e0dd2e7c9 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -117,8 +117,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do result else - {_post_result, response} -> + {_post_result, response} = e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) + Logger.error("Failed to publish activity #{id} #{inspect(e)}") {:error, response} end end -- cgit v1.2.3 From aa070c7dafbceb33b9656f54aa552672497942f6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 23:09:33 -0500 Subject: Handle 401s as I have observed it in the wild --- lib/pleroma/object/fetcher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 972dfaf28..af5642af4 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -219,7 +219,7 @@ defmodule Pleroma.Object.Fetcher do {:error, {:content_type, nil}} end - {:ok, %{status: 403}} -> + {:ok, %{status: code}} when code in [401, 403] -> {:error, :forbidden} {:ok, %{status: code}} when code in [404, 410] -> -- cgit v1.2.3 From 39dc6c65ef7a95412a985a3edce019914af12df8 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 29 Dec 2023 05:23:01 +0100 Subject: ChatMessage: Tolerate attachment field set to an empty array Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3224 --- .../web/activity_pub/object_validators/chat_message_validator.ex | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index efae48cae..09e25be89 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -57,6 +57,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do |> Map.put("attachment", attachment) end + def fix_attachment(%{"attachment" => attachment} = data) when attachment == [] do + data + |> Map.drop(["attachment"]) + end + def fix_attachment(data), do: data def changeset(struct, data) do -- cgit v1.2.3 From f17f92105bff555d2d372ff2ec053fe40fa1b41b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 23:52:05 -0500 Subject: Oban jobs should be discarded on permanent errors --- lib/pleroma/workers/remote_fetcher_worker.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index fef8a26b2..7919969aa 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -16,19 +16,19 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do :ok {:error, :forbidden} -> - {:cancel, :forbidden} + {:discard, :forbidden} {:error, :not_found} -> - {:cancel, :not_found} + {:discard, :not_found} {:error, :allowed_depth} -> - {:cancel, :allowed_depth} + {:discard, :allowed_depth} _ -> :error end else - {:cancel, "Unreachable instance"} + {:discard, "Unreachable instance"} end end -- cgit v1.2.3 From 77949d4590b2a82ef6bb4c79f0777962991e28b1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 00:25:11 -0500 Subject: Make the Publisher log error less noisy --- lib/pleroma/web/activity_pub/publisher.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e0dd2e7c9..103b00f11 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -119,7 +119,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do else {_post_result, response} = e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) - Logger.error("Failed to publish activity #{id} #{inspect(e)}") + Logger.metadata(activity: id, inbox: inbox, status: code) + Logger.error("Publisher failed to inbox #{inbox} with status #{code}") {:error, response} end end -- cgit v1.2.3 From 7ebca7ecfa93f41d9eac2dcefa4b2e55f1b0c4ac Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 00:25:33 -0500 Subject: Activity publishing failures will prevent the job from retrying if the publishing request returns a 403 or 410 --- lib/pleroma/web/activity_pub/publisher.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 103b00f11..1ce920e81 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -117,11 +117,16 @@ defmodule Pleroma.Web.ActivityPub.Publisher do result else - {_post_result, response} = e -> + {_post_result, %{status: code} = response} = e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) Logger.metadata(activity: id, inbox: inbox, status: code) Logger.error("Publisher failed to inbox #{inbox} with status #{code}") - {:error, response} + + case response do + %{status: 403} -> {:discard, :forbidden} + %{status: 410} -> {:discard, :not_found} + _ -> {:error, response} + end end end -- cgit v1.2.3 From 141702538b7ebdc501425b09928332d345c2b577 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 00:31:05 -0500 Subject: Discard on a 404 as well --- lib/pleroma/web/activity_pub/publisher.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 1ce920e81..282cc672a 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -124,6 +124,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do case response do %{status: 403} -> {:discard, :forbidden} + %{status: 404} -> {:discard, :not_found} %{status: 410} -> {:discard, :not_found} _ -> {:error, response} end -- cgit v1.2.3 From 2950397d476b0fd015b28182572927539b88e8fb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 00:50:50 -0500 Subject: Fix following redirects with Finch --- lib/pleroma/http.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index d41061538..eec61cf14 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -106,6 +106,10 @@ defmodule Pleroma.HTTP do [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] end + defp adapter_middlewares({Tesla.Adapter.Finch, _}) do + [Tesla.Middleware.FollowRedirects] + end + defp adapter_middlewares(_) do if Pleroma.Config.get(:env) == :test do # Emulate redirects in test env, which are handled by adapters in other environments -- cgit v1.2.3 From 4afe211e50466606c898e3fbd08b617413ef2e16 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 01:04:05 -0500 Subject: Return the full tuple from Tesla --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 282cc672a..e44e6eac1 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -126,7 +126,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do %{status: 403} -> {:discard, :forbidden} %{status: 404} -> {:discard, :not_found} %{status: 410} -> {:discard, :not_found} - _ -> {:error, response} + _ -> {:error, e} end end end -- cgit v1.2.3 From 833117f5738c286864d525aea9a87d1a7c193ff3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 12:18:23 -0500 Subject: Fix tests Need to handle the edge case of no valid HTTP response which has no status code --- lib/pleroma/web/activity_pub/publisher.ex | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e44e6eac1..cc47b3d27 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -128,6 +128,12 @@ defmodule Pleroma.Web.ActivityPub.Publisher do %{status: 410} -> {:discard, :not_found} _ -> {:error, e} end + + e -> + unless params[:unreachable_since], do: Instances.set_unreachable(inbox) + Logger.metadata(activity: id, inbox: inbox) + Logger.error("Publisher failed to inbox #{inbox} #{inspect(e)}") + {:error, e} end end -- cgit v1.2.3 From 50edef5bc13d0407aaaf26c951ce7a4a8cd4db58 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 14:12:44 -0500 Subject: Change QTFastStart to recover gracefully if it encounters an error during bitstring matching This fixes issues with internal errors when trying to serve the video --- lib/pleroma/helpers/qt_fast_start.ex | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/helpers/qt_fast_start.ex b/lib/pleroma/helpers/qt_fast_start.ex index 5711c7162..919fec84a 100644 --- a/lib/pleroma/helpers/qt_fast_start.ex +++ b/lib/pleroma/helpers/qt_fast_start.ex @@ -40,16 +40,21 @@ defmodule Pleroma.Helpers.QtFastStart do got_mdat, acc ) do - full_size = (size - 8) * 8 - <> = rest - - acc = [ - {fourcc, pos, pos + size, size, - <>} - | acc - ] - - fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc) + try do + full_size = (size - 8) * 8 + <> = rest + + acc = [ + {fourcc, pos, pos + size, size, + <>} + | acc + ] + + fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc) + rescue + _ -> + :abort + end end defp fix(<<>>, _pos, _, _, acc) do -- cgit v1.2.3 From 1b5168ae021fa9d6b154b25444dd96c812083ce7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 28 Nov 2023 20:46:41 +0000 Subject: Phoenix detects the webfinger requests with content-type application/jrd+json as "jrd" now --- lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/web_finger/web_finger_controller.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 00268b121..8ba845364 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -182,7 +182,7 @@ defmodule Pleroma.Web.Router do end pipeline :well_known do - plug(:accepts, ["json", "jrd+json", "xml", "xrd+xml"]) + plug(:accepts, ["json", "jrd", "jrd+json", "xml", "xrd+xml"]) end pipeline :config do diff --git a/lib/pleroma/web/web_finger/web_finger_controller.ex b/lib/pleroma/web/web_finger/web_finger_controller.ex index 9e5efb77f..021df9bc5 100644 --- a/lib/pleroma/web/web_finger/web_finger_controller.ex +++ b/lib/pleroma/web/web_finger/web_finger_controller.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerController do end def webfinger(%{assigns: %{format: format}} = conn, %{"resource" => resource}) - when format in ["json", "jrd+json"] do + when format in ["jrd", "json", "jrd+json"] do with {:ok, response} <- WebFinger.webfinger(resource, "JSON") do json(conn, response) else -- cgit v1.2.3 From 6af49270a9b1ddbdf8836139597b66695d8e1606 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 2 Jan 2024 15:12:33 +0100 Subject: MRF: Log sensible error for subdomains_regex --- lib/pleroma/web/activity_pub/mrf.ex | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 7f6dce925..1071f8e6e 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors +# Copyright © 2017-2023 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF do @@ -139,7 +139,16 @@ defmodule Pleroma.Web.ActivityPub.MRF do @spec subdomains_regex([String.t()]) :: [Regex.t()] def subdomains_regex(domains) when is_list(domains) do - for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$)i + for domain <- domains do + try do + target = String.replace(domain, "*.", "(.*\\.)*") + ~r<^#{target}$>i + rescue + e -> + Logger.error("MRF: Invalid subdomain Regex: #{domain}") + reraise e, __STACKTRACE__ + end + end end @spec subdomain_match?([Regex.t()], String.t()) :: boolean() -- cgit v1.2.3 From 32d8e0d496265f2ebe15199bede63a1f57f043cd Mon Sep 17 00:00:00 2001 From: Alexander Tumin Date: Thu, 4 Jan 2024 16:41:27 +0300 Subject: Fix authentication check on account rendering when bio is defined --- lib/pleroma/web/mastodon_api/views/account_view.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index e7c555eb2..df8fdc8b8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -194,6 +194,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end defp do_render("show.json", %{user: user} = opts) do + self = opts[:for] == user + user = User.sanitize_html(user, User.html_filter_policy(opts[:for])) display_name = user.name || user.nickname @@ -203,12 +205,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true) following_count = - if !user.hide_follows_count or !user.hide_follows or opts[:for] == user, + if !user.hide_follows_count or !user.hide_follows or self, do: user.following_count, else: 0 followers_count = - if !user.hide_followers_count or !user.hide_followers or opts[:for] == user, + if !user.hide_followers_count or !user.hide_followers or self, do: user.follower_count, else: 0 -- cgit v1.2.3 From 69e4ebbb8ea5d273c4a82ee8dcc275da8ee1dace Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 7 Jan 2024 15:28:40 +0300 Subject: Make remote emoji packs API use specifically the V1 URL Akkoma does not understand it without V1, and it works either way with normal pleroma, so no reason to not do this --- lib/pleroma/emoji/pack.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 6e58f8898..49bb93799 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -209,7 +209,9 @@ defmodule Pleroma.Emoji.Pack do with :ok <- validate_shareable_packs_available(uri) do uri - |> URI.merge("/api/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}") + |> URI.merge( + "/api/v1/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}" + ) |> http_get() end end @@ -250,7 +252,7 @@ defmodule Pleroma.Emoji.Pack do with :ok <- validate_shareable_packs_available(uri), {:ok, remote_pack} <- - uri |> URI.merge("/api/pleroma/emoji/pack?name=#{name}") |> http_get(), + uri |> URI.merge("/api/v1/pleroma/emoji/pack?name=#{name}") |> http_get(), {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name), {:ok, archive} <- download_archive(url, sha), pack <- copy_as(remote_pack, as || name), @@ -592,7 +594,7 @@ defmodule Pleroma.Emoji.Pack do {:ok, %{ sha: sha, - url: URI.merge(uri, "/api/pleroma/emoji/packs/archive?name=#{name}") |> to_string() + url: URI.merge(uri, "/api/v1/pleroma/emoji/packs/archive?name=#{name}") |> to_string() }} %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> -- cgit v1.2.3 From 29158681f9f6d3f16bcab011a3338a60f11afddc Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 7 Jan 2024 17:05:30 +0300 Subject: Fetch count before downloading the pack and use that as page size --- lib/pleroma/emoji/pack.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 49bb93799..85f7e1877 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -251,8 +251,12 @@ defmodule Pleroma.Emoji.Pack do uri = url |> String.trim() |> URI.parse() with :ok <- validate_shareable_packs_available(uri), + {:ok, %{"files_count" => files_count}} <- + uri |> URI.merge("/api/v1/pleroma/emoji/pack?name=#{name}&page_size=0") |> http_get(), {:ok, remote_pack} <- - uri |> URI.merge("/api/v1/pleroma/emoji/pack?name=#{name}") |> http_get(), + uri + |> URI.merge("/api/v1/pleroma/emoji/pack?name=#{name}&page_size=#{files_count}") + |> http_get(), {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name), {:ok, archive} <- download_archive(url, sha), pack <- copy_as(remote_pack, as || name), -- cgit v1.2.3 From 4ca65c6182e9a575de3f360a3dac2861ad4e9960 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 7 Jan 2024 19:36:13 +0100 Subject: MRF.StealEmojiPolicy: Properly add fallback extension to filenames missing one --- lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 28c2cf3b3..237dfefa5 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -34,7 +34,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do |> Path.basename() |> Path.extname() - file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png")) + extension = if extension == "", do: ".png", else: extension + + file_path = Path.join(emoji_dir_path, shortcode <> extension) case File.write(file_path, response.body) do :ok -> -- cgit v1.2.3 From 7651198508cc40adfab27969f196e4496bd441da Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 11 Jan 2024 07:13:13 +0100 Subject: Support objects with a nil contentMap (firefish) Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3231 --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 35f3aea03..68f3e1399 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -339,6 +339,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_tag(object), do: object + def fix_content_map(%{"contentMap" => nil} = object) do + Map.drop(object, ["contentMap"]) + end + # content map usually only has one language so this will do for now. def fix_content_map(%{"contentMap" => content_map} = object) do content_groups = Map.to_list(content_map) -- cgit v1.2.3 From 3c30eadd5ede822d6310d3ce6534d26d7caf41f5 Mon Sep 17 00:00:00 2001 From: Mint Date: Thu, 11 Jan 2024 20:38:12 +0300 Subject: Fix duplicate inbox deliveries --- lib/pleroma/web/activity_pub/publisher.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index cc47b3d27..fb7c6f005 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -255,7 +255,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [priority_recipients, recipients] |> Enum.map(fn recipients -> recipients - |> Enum.map(fn actor -> actor.inbox end) + |> Enum.map(fn %User{} = user -> + determine_inbox(activity, user) + end) + |> Enum.uniq() |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) |> Instances.filter_reachable() end) @@ -302,7 +305,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do recipients(actor, activity) |> Enum.map(fn recipients -> recipients - |> Enum.map(fn actor -> actor.inbox end) + |> Enum.map(fn %User{} = user -> + determine_inbox(activity, user) + end) + |> Enum.uniq() |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) end) -- cgit v1.2.3 From 12c052551bcd6b7871ccde5b9228315b89f45e01 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 14 Jan 2024 13:23:17 -0500 Subject: Allow the Remote Fetcher to attempt fetching an unreachable instance --- lib/pleroma/workers/remote_fetcher_worker.ex | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 7919969aa..d526a99cb 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -3,32 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.RemoteFetcherWorker do - alias Pleroma.Instances alias Pleroma.Object.Fetcher use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do - if Instances.reachable?(id) do - case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do - {:ok, _object} -> - :ok + case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do + {:ok, _object} -> + :ok - {:error, :forbidden} -> - {:discard, :forbidden} + {:error, :forbidden} -> + {:discard, :forbidden} - {:error, :not_found} -> - {:discard, :not_found} + {:error, :not_found} -> + {:discard, :not_found} - {:error, :allowed_depth} -> - {:discard, :allowed_depth} + {:error, :allowed_depth} -> + {:discard, :allowed_depth} - _ -> - :error - end - else - {:discard, "Unreachable instance"} + _ -> + :error end end -- cgit v1.2.3 From b39403a48fdb861b905bea16febba6d1660bb8df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 17 Jan 2024 17:12:40 +0100 Subject: Update API docs for my changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index e514d0e91..7f73917d6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -40,6 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), + chat_limit: Keyword.get(instance, :chat_limit), pleroma: pleroma_configuration(instance) }) end @@ -222,6 +223,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do banner_upload_limit: Keyword.get(instance, :banner_upload_limit), background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), + chat_limit: Keyword.get(instance, :chat_limit), description_limit: Keyword.get(instance, :description_limit), shout_limit: Config.get([:shout, :limit]) }) -- cgit v1.2.3 From a5f64ffd0cba5e183744cf46506656ba61d36eda Mon Sep 17 00:00:00 2001 From: Haelwenn Date: Fri, 19 Jan 2024 21:28:15 +0000 Subject: =?UTF-8?q?Apply=20lanodan=E2=80=99s=20suggestion=20to=201=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lib/pleroma/web/api_spec/schemas/poll.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex index cb2ffdc68..eeb1a5490 100644 --- a/lib/pleroma/web/api_spec/schemas/poll.ex +++ b/lib/pleroma/web/api_spec/schemas/poll.ex @@ -60,7 +60,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do pleroma: %Schema{ type: :object, properties: %{ - non_anonymous: %Schema{type: :boolean, description: "Is the voters collection public?"} + non_anonymous: %Schema{type: :boolean, description: "Can voters be publicly identified?"} } } }, -- cgit v1.2.3 From 2a28377be06d286a13c69d6735734d5cfd503309 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 15:55:33 -0500 Subject: Fix mix task pleroma.instance dialyzer error lib/mix/tasks/pleroma/instance.ex:356:pattern_match_cov The pattern :variable_ can never match, because previous clauses completely cover the type %{ :anonymize => boolean(), :dedupe => boolean(), :read_description => boolean(), :strip_location => boolean() }. --- lib/mix/tasks/pleroma/instance.ex | 2 -- .../web/twitter_api/controllers/.util_controller.ex.swp | Bin 0 -> 4096 bytes 2 files changed, 2 deletions(-) create mode 100644 lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index fd1809a42..0dc30549c 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -352,6 +352,4 @@ defmodule Mix.Tasks.Pleroma.Instance do enabled_filters end - - defp upload_filters(_), do: [] end diff --git a/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp b/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp new file mode 100644 index 000000000..822db48a7 Binary files /dev/null and b/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp differ -- cgit v1.2.3 From df1d390a49f3f632fbf99b0ffc4724ba03885e7e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 16:01:30 -0500 Subject: Pleroma.Activity.Queries: fix dialyzer error lib/pleroma/activity/queries.ex:12:unknown_type Unknown type: Activity.t/0. --- lib/pleroma/activity/queries.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index 81c44ac05..d770b9ff3 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Activity.Queries do import Ecto.Query, only: [from: 2, where: 3] - @type query :: Ecto.Queryable.t() | Activity.t() + @type query :: Ecto.Queryable.t() | Pleroma.Activity.t() alias Pleroma.Activity alias Pleroma.User -- cgit v1.2.3 From b16a01ba9d59ff1b5b64f799874dc646fd87aee9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 16:08:38 -0500 Subject: Pleroma.ApplicationRequirements: fix dialyzer errors lib/pleroma/application_requirements.ex:19:unknown_type Unknown type: Pleroma.ApplicationRequirements.VerifyError.t/0. lib/pleroma/application_requirements.ex:199:pattern_match_cov The pattern variable_result can never match, because previous clauses completely cover the type :ok. --- lib/pleroma/application_requirements.ex | 7 ++++--- .../web/twitter_api/controllers/.util_controller.ex.swp | Bin 4096 -> 0 bytes 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp (limited to 'lib') diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 1dbfea3e2..819245481 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -7,7 +7,10 @@ defmodule Pleroma.ApplicationRequirements do The module represents the collection of validations to runs before start server. """ - defmodule VerifyError, do: defexception([:message]) + defmodule VerifyError do + defexception([:message]) + @type t :: %__MODULE__{} + end alias Pleroma.Config alias Pleroma.Helpers.MediaHelper @@ -193,8 +196,6 @@ defmodule Pleroma.ApplicationRequirements do end end - defp check_system_commands!(result), do: result - defp check_repo_pool_size!(:ok) do if Pleroma.Config.get([Pleroma.Repo, :pool_size], 10) != 10 and not Pleroma.Config.get([:dangerzone, :override_repo_pool_size], false) do diff --git a/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp b/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp deleted file mode 100644 index 822db48a7..000000000 Binary files a/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp and /dev/null differ -- cgit v1.2.3 From dc8045d7664327a2ada9c2a14ec46a0e84bfd6bc Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 16:36:01 -0500 Subject: FlakeId.Ecto.CompatType.t() does not exist This type is not exported and usable. FlakeId intends to return the type as :uuid, so we replace it in the typespecs with Ecto.UUID.t() which assuages the dialyzer errors e.g., lib/pleroma/bookmark.ex:25:unknown_type Unknown type: FlakeId.Ecto.CompatType.t/0. --- lib/pleroma/bookmark.ex | 6 +++--- lib/pleroma/chat.ex | 12 ++++++------ lib/pleroma/report_note.ex | 4 ++-- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index 187749e86..5119701f5 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Bookmark do timestamps() end - @spec create(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) :: + @spec create(Ecto.UUID.t(), Ecto.UUID.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()} def create(user_id, activity_id) do attrs = %{ @@ -37,7 +37,7 @@ defmodule Pleroma.Bookmark do |> Repo.insert() end - @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t() + @spec for_user_query(Ecto.UUID.t()) :: Ecto.Query.t() def for_user_query(user_id) do Bookmark |> where(user_id: ^user_id) @@ -52,7 +52,7 @@ defmodule Pleroma.Bookmark do |> Repo.one() end - @spec destroy(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) :: + @spec destroy(Ecto.UUID.t(), Ecto.UUID.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()} def destroy(user_id, activity_id) do from(b in Bookmark, diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex index fe32ec08c..5c4dbc1ff 100644 --- a/lib/pleroma/chat.ex +++ b/lib/pleroma/chat.ex @@ -42,7 +42,7 @@ defmodule Pleroma.Chat do |> unique_constraint(:user_id, name: :chats_user_id_recipient_index) end - @spec get_by_user_and_id(User.t(), FlakeId.Ecto.CompatType.t()) :: + @spec get_by_user_and_id(User.t(), Ecto.UUID.t()) :: {:ok, t()} | {:error, :not_found} def get_by_user_and_id(%User{id: user_id}, id) do from(c in __MODULE__, @@ -52,17 +52,17 @@ defmodule Pleroma.Chat do |> Repo.find_resource() end - @spec get_by_id(FlakeId.Ecto.CompatType.t()) :: t() | nil + @spec get_by_id(Ecto.UUID.t()) :: t() | nil def get_by_id(id) do Repo.get(__MODULE__, id) end - @spec get(FlakeId.Ecto.CompatType.t(), String.t()) :: t() | nil + @spec get(Ecto.UUID.t(), String.t()) :: t() | nil def get(user_id, recipient) do Repo.get_by(__MODULE__, user_id: user_id, recipient: recipient) end - @spec get_or_create(FlakeId.Ecto.CompatType.t(), String.t()) :: + @spec get_or_create(Ecto.UUID.t(), String.t()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} def get_or_create(user_id, recipient) do %__MODULE__{} @@ -75,7 +75,7 @@ defmodule Pleroma.Chat do ) end - @spec bump_or_create(FlakeId.Ecto.CompatType.t(), String.t()) :: + @spec bump_or_create(Ecto.UUID.t(), String.t()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} def bump_or_create(user_id, recipient) do %__MODULE__{} @@ -87,7 +87,7 @@ defmodule Pleroma.Chat do ) end - @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t() + @spec for_user_query(Ecto.UUID.t()) :: Ecto.Query.t() def for_user_query(user_id) do from(c in Chat, where: c.user_id == ^user_id, diff --git a/lib/pleroma/report_note.ex b/lib/pleroma/report_note.ex index f2ad76fa8..1a45b8e99 100644 --- a/lib/pleroma/report_note.ex +++ b/lib/pleroma/report_note.ex @@ -23,7 +23,7 @@ defmodule Pleroma.ReportNote do timestamps() end - @spec create(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t(), String.t()) :: + @spec create(Ecto.UUID.t(), Ecto.UUID.t(), String.t()) :: {:ok, ReportNote.t()} | {:error, Changeset.t()} def create(user_id, activity_id, content) do attrs = %{ @@ -38,7 +38,7 @@ defmodule Pleroma.ReportNote do |> Repo.insert() end - @spec destroy(FlakeId.Ecto.CompatType.t()) :: + @spec destroy(Ecto.UUID.t()) :: {:ok, ReportNote.t()} | {:error, Changeset.t()} def destroy(id) do from(r in ReportNote, where: r.id == ^id) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3980ef7b8..a12438f56 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -499,7 +499,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end @spec fetch_latest_direct_activity_id_for_context(String.t(), keyword() | map()) :: - FlakeId.Ecto.CompatType.t() | nil + Ecto.UUID.t() | nil def fetch_latest_direct_activity_id_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(Map.merge(%{skip_preload: true}, opts)) -- cgit v1.2.3 From 0b7d2142111fa6ff6d0d9a462322f4bc88f15175 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 16:47:50 -0500 Subject: Fix invalid typespec references to Ecto.Changeset.t() --- lib/pleroma/bookmark.ex | 4 ++-- lib/pleroma/config_db.ex | 6 +++--- lib/pleroma/report_note.ex | 4 ++-- lib/pleroma/user.ex | 16 ++++++++-------- lib/pleroma/user_invite_token.ex | 2 +- lib/pleroma/web/o_auth/authorization.ex | 8 ++++---- lib/pleroma/web/o_auth/token.ex | 4 ++-- 7 files changed, 22 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index 5119701f5..b83d72446 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Bookmark do end @spec create(Ecto.UUID.t(), Ecto.UUID.t()) :: - {:ok, Bookmark.t()} | {:error, Changeset.t()} + {:ok, Bookmark.t()} | {:error, Ecto.Changeset.t()} def create(user_id, activity_id) do attrs = %{ user_id: user_id, @@ -53,7 +53,7 @@ defmodule Pleroma.Bookmark do end @spec destroy(Ecto.UUID.t(), Ecto.UUID.t()) :: - {:ok, Bookmark.t()} | {:error, Changeset.t()} + {:ok, Bookmark.t()} | {:error, Ecto.Changeset.t()} def destroy(user_id, activity_id) do from(b in Bookmark, where: b.user_id == ^user_id, diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index 846cede04..e28fcb124 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -54,7 +54,7 @@ defmodule Pleroma.ConfigDB do @spec get_by_params(map()) :: ConfigDB.t() | nil def get_by_params(%{group: _, key: _} = params), do: Repo.get_by(ConfigDB, params) - @spec changeset(ConfigDB.t(), map()) :: Changeset.t() + @spec changeset(ConfigDB.t(), map()) :: Ecto.Changeset.t() def changeset(config, params \\ %{}) do config |> cast(params, [:key, :group, :value]) @@ -138,7 +138,7 @@ defmodule Pleroma.ConfigDB do end end - @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} + @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Ecto.Changeset.t()} def update_or_create(params) do params = Map.put(params, :value, to_elixir_types(params[:value])) search_opts = Map.take(params, [:group, :key]) @@ -175,7 +175,7 @@ defmodule Pleroma.ConfigDB do end) end - @spec delete(ConfigDB.t() | map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} + @spec delete(ConfigDB.t() | map()) :: {:ok, ConfigDB.t()} | {:error, Ecto.Changeset.t()} def delete(%ConfigDB{} = config), do: Repo.delete(config) def delete(params) do diff --git a/lib/pleroma/report_note.ex b/lib/pleroma/report_note.ex index 1a45b8e99..f59e5451b 100644 --- a/lib/pleroma/report_note.ex +++ b/lib/pleroma/report_note.ex @@ -24,7 +24,7 @@ defmodule Pleroma.ReportNote do end @spec create(Ecto.UUID.t(), Ecto.UUID.t(), String.t()) :: - {:ok, ReportNote.t()} | {:error, Changeset.t()} + {:ok, ReportNote.t()} | {:error, Ecto.Changeset.t()} def create(user_id, activity_id, content) do attrs = %{ user_id: user_id, @@ -39,7 +39,7 @@ defmodule Pleroma.ReportNote do end @spec destroy(Ecto.UUID.t()) :: - {:ok, ReportNote.t()} | {:error, Changeset.t()} + {:ok, ReportNote.t()} | {:error, Ecto.Changeset.t()} def destroy(id) do from(r in ReportNote, where: r.id == ^id) |> Repo.one() diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0fd1b6365..45288bb44 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -672,7 +672,7 @@ defmodule Pleroma.User do |> validate_inclusion(:actor_type, ["Person", "Service"]) end - @spec update_as_admin(User.t(), map()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec update_as_admin(User.t(), map()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def update_as_admin(user, params) do params = Map.put(params, "password_confirmation", params["password"]) changeset = update_as_admin_changeset(user, params) @@ -693,7 +693,7 @@ defmodule Pleroma.User do |> put_change(:password_reset_pending, false) end - @spec reset_password(User.t(), map()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec reset_password(User.t(), map()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def reset_password(%User{} = user, params) do reset_password(user, user, params) end @@ -1783,14 +1783,14 @@ defmodule Pleroma.User do BackgroundWorker.enqueue("user_activation", %{"user_id" => user.id, "status" => status}) end - @spec set_activation([User.t()], boolean()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec set_activation([User.t()], boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def set_activation(users, status) when is_list(users) do Repo.transaction(fn -> for user <- users, do: set_activation(user, status) end) end - @spec set_activation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec set_activation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def set_activation(%User{} = user, status) do with {:ok, user} <- set_activation_status(user, status) do user @@ -1868,7 +1868,7 @@ defmodule Pleroma.User do |> update_and_set_cache() end - @spec purge_user_changeset(User.t()) :: Changeset.t() + @spec purge_user_changeset(User.t()) :: Ecto.Changeset.t() def purge_user_changeset(user) do # "Right to be forgotten" # https://gdpr.eu/right-to-be-forgotten/ @@ -2359,7 +2359,7 @@ defmodule Pleroma.User do updated_user end - @spec set_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec set_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def set_confirmation(%User{} = user, bool) do user |> confirmation_changeset(set_confirmation: bool) @@ -2537,7 +2537,7 @@ defmodule Pleroma.User do |> update_and_set_cache() end - @spec confirmation_changeset(User.t(), keyword()) :: Changeset.t() + @spec confirmation_changeset(User.t(), keyword()) :: Ecto.Changeset.t() def confirmation_changeset(user, set_confirmation: confirmed?) do params = if confirmed? do @@ -2555,7 +2555,7 @@ defmodule Pleroma.User do cast(user, params, [:is_confirmed, :confirmation_token]) end - @spec approval_changeset(User.t(), keyword()) :: Changeset.t() + @spec approval_changeset(User.t(), keyword()) :: Ecto.Changeset.t() def approval_changeset(user, set_approval: approved?) do cast(user, %{is_approved: approved?}, [:is_approved]) end diff --git a/lib/pleroma/user_invite_token.ex b/lib/pleroma/user_invite_token.ex index b242a8848..4bfb3a6a7 100644 --- a/lib/pleroma/user_invite_token.ex +++ b/lib/pleroma/user_invite_token.ex @@ -64,7 +64,7 @@ defmodule Pleroma.UserInviteToken do end @spec update_invite(UserInviteToken.t(), map()) :: - {:ok, UserInviteToken.t()} | {:error, Changeset.t()} + {:ok, UserInviteToken.t()} | {:error, Ecto.Changeset.t()} def update_invite(invite, changes) do change(invite, changes) |> Repo.update() end diff --git a/lib/pleroma/web/o_auth/authorization.ex b/lib/pleroma/web/o_auth/authorization.ex index 593d2d66f..3340b4b12 100644 --- a/lib/pleroma/web/o_auth/authorization.ex +++ b/lib/pleroma/web/o_auth/authorization.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.OAuth.Authorization do end @spec create_authorization(App.t(), User.t() | %{}, [String.t()] | nil) :: - {:ok, Authorization.t()} | {:error, Changeset.t()} + {:ok, Authorization.t()} | {:error, Ecto.Changeset.t()} def create_authorization(%App{} = app, %User{} = user, scopes \\ nil) do %{ scopes: scopes || app.scopes, @@ -39,7 +39,7 @@ defmodule Pleroma.Web.OAuth.Authorization do |> Repo.insert() end - @spec create_changeset(map()) :: Changeset.t() + @spec create_changeset(map()) :: Ecto.Changeset.t() def create_changeset(attrs \\ %{}) do %Authorization{} |> cast(attrs, [:user_id, :app_id, :scopes, :valid_until]) @@ -58,7 +58,7 @@ defmodule Pleroma.Web.OAuth.Authorization do put_change(changeset, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), lifespan)) end - @spec use_changeset(Authtorizatiton.t(), map()) :: Changeset.t() + @spec use_changeset(Authtorizatiton.t(), map()) :: Ecto.Changeset.t() def use_changeset(%Authorization{} = auth, params) do auth |> cast(params, [:used]) @@ -66,7 +66,7 @@ defmodule Pleroma.Web.OAuth.Authorization do end @spec use_token(Authorization.t()) :: - {:ok, Authorization.t()} | {:error, Changeset.t()} | {:error, String.t()} + {:ok, Authorization.t()} | {:error, Ecto.Changeset.t()} | {:error, String.t()} def use_token(%Authorization{used: false, valid_until: valid_until} = auth) do if NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) < 0 do Repo.update(use_changeset(auth, %{used: true})) diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 26de7bb10..4369b3164 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Web.OAuth.Token do |> Repo.find_resource() end - @spec exchange_token(App.t(), Authorization.t()) :: {:ok, Token.t()} | {:error, Changeset.t()} + @spec exchange_token(App.t(), Authorization.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()} def exchange_token(app, auth) do with {:ok, auth} <- Authorization.use_token(auth), true <- auth.app_id == app.id do @@ -95,7 +95,7 @@ defmodule Pleroma.Web.OAuth.Token do |> validate_required([:valid_until]) end - @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()} + @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Ecto.Changeset.t()} def create(%App{} = app, %User{} = user, attrs \\ %{}) do with {:ok, token} <- do_create(app, user, attrs) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do -- cgit v1.2.3 From 8ed506a370f0b4a7743353cd3c96c9dacd290765 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:05:55 -0500 Subject: Fix invalid type lib/pleroma/docs/json.ex:21:unknown_type Unknown type: Map.t/0. --- lib/pleroma/docs/json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex index 05f46f39b..f69854935 100644 --- a/lib/pleroma/docs/json.ex +++ b/lib/pleroma/docs/json.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Docs.JSON do :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(descriptions)) end - @spec compiled_descriptions :: Map.t() + @spec compiled_descriptions :: map() def compiled_descriptions do :persistent_term.get(@term) end -- cgit v1.2.3 From 559aeb5dd0cb66e57fedce9ea20eff505786d0cb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:08:18 -0500 Subject: Add missing type Pleroma.Emoji.t() lib/pleroma/emoji/loader.ex:23:unknown_type Unknown type: Pleroma.Emoji.t/0. --- lib/pleroma/emoji.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 43a3447c3..48302f396 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -24,6 +24,8 @@ defmodule Pleroma.Emoji do defstruct [:code, :file, :tags, :safe_code, :safe_file] + @type t :: %__MODULE__{} + @doc "Build emoji struct" def build({code, file, tags}) do %__MODULE__{ -- cgit v1.2.3 From 593c7e26d4769196a3405b61147dd88aa42e6012 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:13:27 -0500 Subject: Fix invalid type lib/pleroma/migrators/hashtags_table_migrator.ex:103:unknown_type Unknown type: Map.t/0. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index dca4bfa6f..bd4dd2f1d 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -100,7 +100,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do |> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id)) end - @spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()} + @spec transfer_object_hashtags(map()) :: {:noop | :ok | :error, integer()} defp transfer_object_hashtags(object) do embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"] hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags}) -- cgit v1.2.3 From e3f52ee13fbda3b45b5d443ef9070e84cb7c1518 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:14:10 -0500 Subject: Fix invalid types lib/pleroma/web/streamer.ex:37:unknown_type Unknown type: Map.t/0. ________________________________________________________________________________ lib/pleroma/web/streamer.ex:63:unknown_type Unknown type: Map.t/0. --- lib/pleroma/web/streamer.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index aaee79d8e..0c9f04f82 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Web.Streamer do stream :: String.t(), User.t() | nil, Token.t() | nil, - Map.t() | nil + map() | nil ) :: {:ok, topic :: String.t()} | {:error, :bad_topic} | {:error, :unauthorized} def get_topic_and_add_socket(stream, user, oauth_token, params \\ %{}) do @@ -60,7 +60,7 @@ defmodule Pleroma.Web.Streamer do end @doc "Expand and authorizes a stream" - @spec get_topic(stream :: String.t() | nil, User.t() | nil, Token.t() | nil, Map.t()) :: + @spec get_topic(stream :: String.t() | nil, User.t() | nil, Token.t() | nil, map()) :: {:ok, topic :: String.t() | nil} | {:error, :bad_topic} def get_topic(stream, user, oauth_token, params \\ %{}) -- cgit v1.2.3 From 467a65af90e4ee7893cd40cefe4931f2738219d4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:14:56 -0500 Subject: Fix invalid types lib/pleroma/web/rich_media/parser/ttl.ex:6:unknown_type Unknown type: Integer.t/0. lib/pleroma/web/rich_media/parser/ttl.ex:6:unknown_type Unknown type: Map.t/0. --- lib/pleroma/web/rich_media/parser/ttl.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/parser/ttl.ex b/lib/pleroma/web/rich_media/parser/ttl.ex index 59d7f87ab..b51298bd8 100644 --- a/lib/pleroma/web/rich_media/parser/ttl.ex +++ b/lib/pleroma/web/rich_media/parser/ttl.ex @@ -3,5 +3,5 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser.TTL do - @callback ttl(Map.t(), String.t()) :: Integer.t() | nil + @callback ttl(map(), String.t()) :: integer() | nil end -- cgit v1.2.3 From 09ae0ab24a30eebbb4880ff44300ee1774bffe93 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:16:10 -0500 Subject: Fix invalid type lib/pleroma/web/rich_media/parser.ex:105:unknown_type Unknown type: Integer.t/0. --- lib/pleroma/web/rich_media/parser.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index c37c45963..17a6ad617 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -102,7 +102,7 @@ defmodule Pleroma.Web.RichMedia.Parser do ttl_setters: [MyModule] """ @spec set_ttl_based_on_image(map(), String.t()) :: - {:ok, Integer.t() | :noop} | {:error, :no_key} + {:ok, integer() | :noop} | {:error, :no_key} def set_ttl_based_on_image(data, url) do case get_ttl_from_image(data, url) do {:ok, ttl} when is_number(ttl) -> -- cgit v1.2.3 From 65dfaa6cb9f6aa5bf7f66b5f02d63141a93137ba Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:18:16 -0500 Subject: Fix invalid type due to late aliasing lib/pleroma/web/o_auth/token/query.ex:12:unknown_type Unknown type: Token.t/0. --- lib/pleroma/web/o_auth/token/query.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/o_auth/token/query.ex b/lib/pleroma/web/o_auth/token/query.ex index 4a4d2d3ef..6853ec8dd 100644 --- a/lib/pleroma/web/o_auth/token/query.ex +++ b/lib/pleroma/web/o_auth/token/query.ex @@ -9,10 +9,10 @@ defmodule Pleroma.Web.OAuth.Token.Query do import Ecto.Query, only: [from: 2] - @type query :: Ecto.Queryable.t() | Token.t() - alias Pleroma.Web.OAuth.Token + @type query :: Ecto.Queryable.t() | Token.t() + @spec get_by_refresh_token(query, String.t()) :: query def get_by_refresh_token(query \\ Token, refresh_token) do from(q in query, where: q.refresh_token == ^refresh_token) -- cgit v1.2.3 From e5120a27031c6423504cb15ea1ccd3ef9d47e6e4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:21:12 -0500 Subject: Fix invalid type due to typos lib/pleroma/web/o_auth/authorization.ex:61:unknown_type Unknown type: Authtorizatiton.t/0. --- lib/pleroma/web/o_auth/authorization.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/o_auth/authorization.ex b/lib/pleroma/web/o_auth/authorization.ex index 3340b4b12..22e5bfc53 100644 --- a/lib/pleroma/web/o_auth/authorization.ex +++ b/lib/pleroma/web/o_auth/authorization.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Web.OAuth.Authorization do put_change(changeset, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), lifespan)) end - @spec use_changeset(Authtorizatiton.t(), map()) :: Ecto.Changeset.t() + @spec use_changeset(Authorization.t(), map()) :: Ecto.Changeset.t() def use_changeset(%Authorization{} = auth, params) do auth |> cast(params, [:used]) -- cgit v1.2.3 From f050a75b92546968f9ec351f78a48e6b6ac75a6b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:22:40 -0500 Subject: Fix invalid types due to typos lib/pleroma/web/feed/feed_view.ex:135:unknown_type Unknown type: NativeDateTime.t/0. lib/pleroma/web/feed/feed_view.ex:148:unknown_type Unknown type: NativeDateTime.t/0. --- lib/pleroma/web/feed/feed_view.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index 034722eb2..e1ee33d62 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -132,7 +132,7 @@ defmodule Pleroma.Web.Feed.FeedView do |> safe_to_string() end - @spec to_rfc3339(String.t() | NativeDateTime.t()) :: String.t() + @spec to_rfc3339(String.t() | NaiveDateTime.t()) :: String.t() def to_rfc3339(date) when is_binary(date) do date |> Timex.parse!("{ISO:Extended}") @@ -145,7 +145,7 @@ defmodule Pleroma.Web.Feed.FeedView do |> Timex.format!("{RFC3339}") end - @spec to_rfc2822(String.t() | DateTime.t() | NativeDateTime.t()) :: String.t() + @spec to_rfc2822(String.t() | DateTime.t() | NaiveDateTime.t()) :: String.t() def to_rfc2822(datestr) when is_binary(datestr) do datestr |> Timex.parse!("{ISO:Extended}") -- cgit v1.2.3 From 551e90cd52a11dcb11840c6c4074db4a3c580703 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:28:54 -0500 Subject: Fix invalid type lib/pleroma/upload.ex:89:unknown_type Unknown type: Map.t/0. --- lib/pleroma/upload.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index bedd7889a..3c355e64d 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -86,7 +86,7 @@ defmodule Pleroma.Upload do end end - @spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()} + @spec store(source, options :: [option()]) :: {:ok, map()} | {:error, any()} @doc "Store a file. If using a `Plug.Upload{}` as the source, be sure to use `Majic.Plug` to ensure its content_type and filename is correct." def store(upload, opts \\ []) do opts = get_opts(opts) -- cgit v1.2.3 From 2061a1d917a15bfe12b767c665c05750597e6145 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:29:27 -0500 Subject: Fix invalid type lib/pleroma/uploaders/uploader.ex:43:unknown_type Unknown type: Map.t/0. --- lib/pleroma/uploaders/uploader.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex index 77f6f02dd..23caaff1a 100644 --- a/lib/pleroma/uploaders/uploader.ex +++ b/lib/pleroma/uploaders/uploader.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Uploaders.Uploader do @callback delete_file(file :: String.t()) :: :ok | {:error, String.t()} - @callback http_callback(Plug.Conn.t(), Map.t()) :: + @callback http_callback(Plug.Conn.t(), map()) :: {:ok, Plug.Conn.t()} | {:ok, Plug.Conn.t(), file_spec()} | {:error, Plug.Conn.t(), String.t()} -- cgit v1.2.3 From ec5ae83da6f3b630f174288cf3687bf12cee9787 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:31:07 -0500 Subject: Fix invalid types lib/pleroma/web/activity_pub/mrf/policy.ex:6:unknown_type Unknown type: Map.t/0. lib/pleroma/web/activity_pub/mrf/policy.ex:7:unknown_type Unknown type: Map.t/0. --- lib/pleroma/web/activity_pub/mrf/policy.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 0234de4d5..1f34883e7 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -3,8 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.Policy do - @callback filter(Map.t()) :: {:ok | :reject, Map.t()} - @callback describe() :: {:ok | :error, Map.t()} + @callback filter(map()) :: {:ok | :reject, map()} + @callback describe() :: {:ok | :error, map()} @callback config_description() :: %{ optional(:children) => [map()], key: atom(), -- cgit v1.2.3 From 4f0711610896cb76322d0f527cbea5ed9d0ef474 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:32:19 -0500 Subject: Fix invalid type lib/pleroma/web/activity_pub/publisher.ex:31:unknown_type Unknown type: Map.t/0. --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index fb7c6f005..2449a3a3e 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do @doc """ Enqueue publishing a single activity. """ - @spec enqueue_one(Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} + @spec enqueue_one(map(), Keyword.t()) :: {:ok, %Oban.Job{}} def enqueue_one(%{} = params, worker_args \\ []) do PublisherWorker.enqueue( "publish_one", -- cgit v1.2.3 From 83eece7764243d5902df96d2a8a6b99d7465ddba Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:33:37 -0500 Subject: Fix invalid type lib/pleroma/web/auth/authenticator.ex:8:unknown_type Unknown type: User.t/0. --- lib/pleroma/web/auth/authenticator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index a0bd154db..01bf1575c 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.Auth.Authenticator do @callback get_user(Plug.Conn.t()) :: {:ok, user :: struct()} | {:error, any()} @callback create_from_registration(Plug.Conn.t(), registration :: struct()) :: - {:ok, User.t()} | {:error, any()} + {:ok, Pleroma.User.t()} | {:error, any()} @callback get_registration(Plug.Conn.t()) :: {:ok, registration :: struct()} | {:error, any()} @callback handle_error(Plug.Conn.t(), any()) :: any() @callback auth_template() :: String.t() | nil -- cgit v1.2.3 From 38d01ff511ecd7223787772044ca6e3b20578c05 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:37:27 -0500 Subject: Fix invalid types --- lib/pleroma/reverse_proxy.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 880940d07..ec7732946 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -81,9 +81,9 @@ defmodule Pleroma.ReverseProxy do import Plug.Conn @type option() :: - {:max_read_duration, :timer.time() | :infinity} + {:max_read_duration, non_neg_integer() | :infinity} | {:max_body_length, non_neg_integer() | :infinity} - | {:failed_request_ttl, :timer.time() | :infinity} + | {:failed_request_ttl, non_neg_integer() | :infinity} | {:http, []} | {:req_headers, [{String.t(), String.t()}]} | {:resp_headers, [{String.t(), String.t()}]} -- cgit v1.2.3 From ea26add540b1da91ebc266b0c0532e1f8c54f4d7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:43:34 -0500 Subject: Fix incorrect type definition for maybe_direct_follow/2 --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 45288bb44..416a5b5c9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1048,7 +1048,7 @@ defmodule Pleroma.User do def needs_update?(_), do: true - @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()} + @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t(), User.t()} | {:error, String.t()} # "Locked" (self-locked) users demand explicit authorization of follow requests def maybe_direct_follow(%User{} = follower, %User{local: true, is_locked: true} = followed) do -- cgit v1.2.3 From 2fbb67add70e77ba7da388623064bb5a671ce349 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:48:12 -0500 Subject: Fix typo in typespec --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 416a5b5c9..ac049ec17 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1011,7 +1011,7 @@ defmodule Pleroma.User do def maybe_send_confirmation_email(_), do: {:ok, :noop} - @spec send_confirmation_email(Uset.t()) :: User.t() + @spec send_confirmation_email(User.t()) :: User.t() def send_confirmation_email(%User{} = user) do user |> Pleroma.Emails.UserEmail.account_confirmation_email() -- cgit v1.2.3 From 7f649a7a19ae5f236dc21aaa11bb9a33c72dabbf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:50:21 -0500 Subject: Dialyzer: remove function that will never match --- lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 97d75ecf2..df4ba819c 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -56,8 +56,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do nick_score + name_score + actor_type_score end - defp determine_if_followbot(_), do: 0.0 - defp bot_allowed?(%{"object" => target}, bot_actor) do %User{} = user = normalize_by_ap_id(target) -- cgit v1.2.3 From 88042109a3590db25dd9e39f353b1456cbd4e44b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:56:32 -0500 Subject: Dialyzer: fix pattern match coverage --- .../ecto_type/activity_pub/object_validators/bare_uri.ex | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex index 1038296e7..a1af8faa1 100644 --- a/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex @@ -8,10 +8,12 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.BareUri do def type, do: :string def cast(uri) when is_binary(uri) do - case URI.parse(uri) do - %URI{scheme: nil} -> :error - %URI{} -> {:ok, uri} - _ -> :error + parsed = URI.parse(uri) + + if is_nil(parsed.scheme) do + :error + else + {:ok, uri} end end -- cgit v1.2.3 From 65ac5137768d90ee9d2667e68181037e5d5642d3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:58:47 -0500 Subject: Dialyzer: fix pattern match coverage --- lib/pleroma/release_tasks.ex | 6 ------ 1 file changed, 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/release_tasks.ex b/lib/pleroma/release_tasks.ex index f9e8d1948..bcfcd1243 100644 --- a/lib/pleroma/release_tasks.ex +++ b/lib/pleroma/release_tasks.ex @@ -55,12 +55,6 @@ defmodule Pleroma.ReleaseTasks do {:error, term} when is_binary(term) -> IO.puts(:stderr, "The database for #{inspect(@repo)} couldn't be created: #{term}") - - {:error, term} -> - IO.puts( - :stderr, - "The database for #{inspect(@repo)} couldn't be created: #{inspect(term)}" - ) end end end -- cgit v1.2.3 From 029aaf3d744184737666b1e553ed92d7708f1fee Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 18:14:27 -0500 Subject: Use config to control max_restarts --- lib/pleroma/application.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 8fa6f3fae..a01a13b18 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -116,12 +116,7 @@ defmodule Pleroma.Application do # If we have a lot of caches, default max_restarts can cause test # resets to fail. # Go for the default 3 unless we're in test - max_restarts = - if @mix_env == :test do - 100 - else - 3 - end + max_restarts = Application.get_env(:pleroma, __MODULE__)[:max_restarts] opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts] result = Supervisor.start_link(children, opts) -- cgit v1.2.3 From c7eda0b24ade372e4e167ae560a2debb555a6e02 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 18:22:49 -0500 Subject: Use config to control loading of custom modules --- lib/pleroma/application.ex | 34 ++++++++++++++++------------------ lib/pleroma/user.ex | 3 ++- lib/pleroma/web/o_auth/token.ex | 3 ++- 3 files changed, 20 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index a01a13b18..de7d06974 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -106,7 +106,7 @@ defmodule Pleroma.Application do {Oban, Config.get(Oban)}, Pleroma.Web.Endpoint ] ++ - task_children(@mix_env) ++ + task_children() ++ dont_run_in_test(@mix_env) ++ shout_child(shout_enabled?()) ++ [Pleroma.Gopher.Server] @@ -154,7 +154,7 @@ defmodule Pleroma.Application do raise "Invalid custom modules" {:ok, modules, _warnings} -> - if @mix_env != :test do + if Application.get_env(:pleroma, __MODULE__)[:load_custom_modules] do Enum.each(modules, fn mod -> Logger.info("Custom module loaded: #{inspect(mod)}") end) @@ -237,29 +237,27 @@ defmodule Pleroma.Application do defp shout_child(_), do: [] - defp task_children(:test) do - [ + defp task_children() do + children = [ %{ id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, restart: :temporary } ] - end - defp task_children(_) do - [ - %{ - id: :web_push_init, - start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, - restart: :temporary - }, - %{ - id: :internal_fetch_init, - start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, - restart: :temporary - } - ] + if Application.get_env(:pleroma, __MODULE__)[:internal_fetch] do + children ++ + [ + %{ + id: :internal_fetch_init, + start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, + restart: :temporary + } + ] + else + children + end end # start hackney and gun pools in tests diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ac049ec17..89a95c435 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1048,7 +1048,8 @@ defmodule Pleroma.User do def needs_update?(_), do: true - @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t(), User.t()} | {:error, String.t()} + @spec maybe_direct_follow(User.t(), User.t()) :: + {:ok, User.t(), User.t()} | {:error, String.t()} # "Locked" (self-locked) users demand explicit authorization of follow requests def maybe_direct_follow(%User{} = follower, %User{local: true, is_locked: true} = followed) do diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 4369b3164..d4a7e8999 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -56,7 +56,8 @@ defmodule Pleroma.Web.OAuth.Token do |> Repo.find_resource() end - @spec exchange_token(App.t(), Authorization.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()} + @spec exchange_token(App.t(), Authorization.t()) :: + {:ok, Token.t()} | {:error, Ecto.Changeset.t()} def exchange_token(app, auth) do with {:ok, auth} <- Authorization.use_token(auth), true <- auth.app_id == app.id do -- cgit v1.2.3 From 4bb57d4f25bcdc90a63163ba175b6171c9ddbc33 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 18:47:25 -0500 Subject: Use config to control background migrators --- lib/pleroma/application.ex | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index de7d06974..2eda212b3 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -108,6 +108,7 @@ defmodule Pleroma.Application do ] ++ task_children() ++ dont_run_in_test(@mix_env) ++ + background_migrators() ++ shout_child(shout_enabled?()) ++ [Pleroma.Gopher.Server] @@ -218,14 +219,18 @@ defmodule Pleroma.Application do keys: :duplicate, partitions: System.schedulers_online() ]} - ] ++ background_migrators() + ] end defp background_migrators do - [ - Pleroma.Migrators.HashtagsTableMigrator, - Pleroma.Migrators.ContextObjectsDeletionMigrator - ] + if Application.get_env(:pleroma, __MODULE__)[:background_migrators] do + [ + Pleroma.Migrators.HashtagsTableMigrator, + Pleroma.Migrators.ContextObjectsDeletionMigrator + ] + else + [] + end end defp shout_child(true) do -- cgit v1.2.3 From 17877f612e6c655290c5dc8bdb82f4b34e8b5b9f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 18:51:20 -0500 Subject: Use config to control streamer registry --- lib/pleroma/application.ex | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2eda212b3..272529972 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -107,7 +107,7 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint ] ++ task_children() ++ - dont_run_in_test(@mix_env) ++ + streamer_registry() ++ background_migrators() ++ shout_child(shout_enabled?()) ++ [Pleroma.Gopher.Server] @@ -209,17 +209,19 @@ defmodule Pleroma.Application do defp shout_enabled?, do: Config.get([:shout, :enabled]) - defp dont_run_in_test(env) when env in [:test, :benchmark], do: [] - - defp dont_run_in_test(_) do - [ - {Registry, - [ - name: Pleroma.Web.Streamer.registry(), - keys: :duplicate, - partitions: System.schedulers_online() - ]} - ] + defp streamer_registry() do + if Application.get_env(:pleroma, __MODULE__)[:streamer_registry] do + [ + {Registry, + [ + name: Pleroma.Web.Streamer.registry(), + keys: :duplicate, + partitions: System.schedulers_online() + ]} + ] + else + [] + end end defp background_migrators do -- cgit v1.2.3 From 23301003717a5e154f54c14a5b8cb10ea2033e1a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 19:10:57 -0500 Subject: Use config to control starting all HTTP pools in test env --- lib/pleroma/application.ex | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 272529972..c0bfb898d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -14,7 +14,6 @@ defmodule Pleroma.Application do @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] - @mix_env Mix.env() def name, do: @name def version, do: @version @@ -98,7 +97,7 @@ defmodule Pleroma.Application do {Task.Supervisor, name: Pleroma.TaskSupervisor} ] ++ cachex_children() ++ - http_children(adapter, @mix_env) ++ + http_children(adapter) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, @@ -268,11 +267,19 @@ defmodule Pleroma.Application do end # start hackney and gun pools in tests - defp http_children(_, :test) do - http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil) + defp http_children(adapter) do + if Application.get_env(:pleroma, __MODULE__)[:test_http_pools] do + http_children_hackney() ++ http_children_gun() + else + cond do + match?(Tesla.Adapter.Hackney, adapter) -> http_children_hackney() + match?(Tesla.Adapter.Gun, adapter) -> http_children_gun() + true -> [] + end + end end - defp http_children(Tesla.Adapter.Hackney, _) do + defp http_children_hackney() do pools = [:federation, :media] pools = @@ -288,13 +295,11 @@ defmodule Pleroma.Application do end end - defp http_children(Tesla.Adapter.Gun, _) do + defp http_children_gun() do Pleroma.Gun.ConnectionPool.children() ++ [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] end - defp http_children(_, _), do: [] - @spec limiters_setup() :: :ok def limiters_setup do config = Config.get(ConcurrentLimiter, []) -- cgit v1.2.3 From dcd010280034fd35254906831402d96fd59f8098 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 19:39:13 -0500 Subject: Credo --- lib/pleroma/application.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c0bfb898d..de668052f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -208,7 +208,7 @@ defmodule Pleroma.Application do defp shout_enabled?, do: Config.get([:shout, :enabled]) - defp streamer_registry() do + defp streamer_registry do if Application.get_env(:pleroma, __MODULE__)[:streamer_registry] do [ {Registry, @@ -243,7 +243,7 @@ defmodule Pleroma.Application do defp shout_child(_), do: [] - defp task_children() do + defp task_children do children = [ %{ id: :web_push_init, @@ -279,7 +279,7 @@ defmodule Pleroma.Application do end end - defp http_children_hackney() do + defp http_children_hackney do pools = [:federation, :media] pools = @@ -295,7 +295,7 @@ defmodule Pleroma.Application do end end - defp http_children_gun() do + defp http_children_gun do Pleroma.Gun.ConnectionPool.children() ++ [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] end -- cgit v1.2.3 From def088ce520ffac3e200cbf63b049f1d918d2b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 21 Jan 2024 18:23:24 +0100 Subject: format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec/schemas/poll.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/schemas/poll.ex b/lib/pleroma/web/api_spec/schemas/poll.ex index eeb1a5490..20cf5b061 100644 --- a/lib/pleroma/web/api_spec/schemas/poll.ex +++ b/lib/pleroma/web/api_spec/schemas/poll.ex @@ -60,7 +60,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Poll do pleroma: %Schema{ type: :object, properties: %{ - non_anonymous: %Schema{type: :boolean, description: "Can voters be publicly identified?"} + non_anonymous: %Schema{ + type: :boolean, + description: "Can voters be publicly identified?" + } } } }, -- cgit v1.2.3 From 0ac010ba3fa41c9bd06565259de57f2a5b5bb8ad Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 10:01:29 -0500 Subject: Replace custom fifo implementation with Exile This is for streaming media to ffmpeg thumbnailer. The existing implementation relies on undocumented behavior. Erlang open_port/2 does not officially support passing a string of a file path for opening. The specs clearly state you are to provide one of the following for open_port/2: {spawn, Command :: string() | binary()} | {spawn_driver, Command :: string() | binary()} | {spawn_executable, FileName :: file:name_all()} | {fd, In :: integer() >= 0, Out :: integer() >= 0} Our method technically works but is strongly discouraged as it can block the scheduler and dialyzer throws errors as it recognizes we're breaking the contract and some of the functions we wrote may never return. This is indirectly covered by the Erlang FAQ section "9.12 Why can't I open devices (e.g. a serial port) like normal files?" https://www.erlang.org/faq/problems#idm1127 --- lib/pleroma/helpers/media_helper.ex | 99 +++++++------------------------------ 1 file changed, 19 insertions(+), 80 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index 07dfea55b..1a414b37f 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -43,89 +43,28 @@ defmodule Pleroma.Helpers.MediaHelper do def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), {:ok, env} <- HTTP.get(url, [], pool: :media), - {:ok, fifo_path} <- mkfifo(), - args = [ - "-y", - "-i", - fifo_path, - "-vframes", - "1", - "-f", - "mjpeg", - "-loglevel", - "error", - "-" - ] do - run_fifo(fifo_path, env, executable, args) + {:ok, pid} <- StringIO.open(env.body) do + body_stream = IO.binstream(pid, 1) + + Exile.stream!( + [ + executable, + "-i", + "pipe:0", + "-vframes", + "1", + "-f", + "mjpeg", + "pipe:1" + ], + input: body_stream, + ignore_epipe: true, + stderr: :disable + ) + |> Enum.into(<<>>) else nil -> {:error, {:ffmpeg, :command_not_found}} {:error, _} = error -> error end end - - defp run_fifo(fifo_path, env, executable, args) do - pid = - Port.open({:spawn_executable, executable}, [ - :use_stdio, - :stream, - :exit_status, - :binary, - args: args - ]) - - fifo = Port.open(to_charlist(fifo_path), [:eof, :binary, :stream, :out]) - fix = Pleroma.Helpers.QtFastStart.fix(env.body) - true = Port.command(fifo, fix) - :erlang.port_close(fifo) - loop_recv(pid) - after - File.rm(fifo_path) - end - - defp mkfifo do - path = Path.join(System.tmp_dir!(), "pleroma-media-preview-pipe-#{Ecto.UUID.generate()}") - - case System.cmd("mkfifo", [path]) do - {_, 0} -> - spawn(fifo_guard(path)) - {:ok, path} - - {_, err} -> - {:error, {:fifo_failed, err}} - end - end - - defp fifo_guard(path) do - pid = self() - - fn -> - ref = Process.monitor(pid) - - receive do - {:DOWN, ^ref, :process, ^pid, _} -> - File.rm(path) - end - end - end - - defp loop_recv(pid) do - loop_recv(pid, <<>>) - end - - defp loop_recv(pid, acc) do - receive do - {^pid, {:data, data}} -> - loop_recv(pid, acc <> data) - - {^pid, {:exit_status, 0}} -> - {:ok, acc} - - {^pid, {:exit_status, status}} -> - {:error, status} - after - 5000 -> - :erlang.port_close(pid) - {:error, :timeout} - end - end end -- cgit v1.2.3 From 653b14e1c798bbddc34821fa56f32fef9c227f01 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 14:40:14 -0500 Subject: Use config to control Uploader callback timeout --- lib/pleroma/uploaders/uploader.ex | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex index 23caaff1a..3396fe06a 100644 --- a/lib/pleroma/uploaders/uploader.ex +++ b/lib/pleroma/uploaders/uploader.ex @@ -5,8 +5,6 @@ defmodule Pleroma.Uploaders.Uploader do import Pleroma.Web.Gettext - @mix_env Mix.env() - @moduledoc """ Defines the contract to put and get an uploaded file to any backend. """ @@ -75,10 +73,5 @@ defmodule Pleroma.Uploaders.Uploader do end end - defp callback_timeout do - case @mix_env do - :test -> 1_000 - _ -> 30_000 - end - end + defp callback_timeout, do: Application.get_env(:pleroma, __MODULE__)[:timeout] end -- cgit v1.2.3 From 6df93e61c4bb4ccacee7092cd20efe9add9a8500 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 14:57:15 -0500 Subject: Use config to determine sending to the streamer registry instead of MIX_ENV compile time function definition --- lib/pleroma/web/streamer.ex | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 0c9f04f82..35c015e24 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -20,7 +20,6 @@ defmodule Pleroma.Web.Streamer do alias Pleroma.Web.Plugs.OAuthScopesPlug alias Pleroma.Web.StreamerView - @mix_env Mix.env() @registry Pleroma.Web.StreamerRegistry def registry, do: @registry @@ -396,25 +395,20 @@ defmodule Pleroma.Web.Streamer do end end - # In test environment, only return true if the registry is started. - # In benchmark environment, returns false. - # In any other environment, always returns true. - cond do - @mix_env == :test -> - def should_env_send? do - case Process.whereis(@registry) do - nil -> - false + # In dev/prod the streamer registry is expected to be started, so return true + # In test it is possible to have the registry started for a test so it will check + # In benchmark it will never find the process alive and return false + def should_env_send? do + if Application.get_env(:pleroma, Pleroma.Application)[:streamer_registry] do + true + else + case Process.whereis(@registry) do + nil -> + false - pid -> - Process.alive?(pid) - end + pid -> + Process.alive?(pid) end - - @mix_env == :benchmark -> - def should_env_send?, do: false - - true -> - def should_env_send?, do: true + end end end -- cgit v1.2.3 From eb4dd50f533380218bcde1a4107e2c0d38b4ec0d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 14:57:29 -0500 Subject: Use config to control inclusion of test emoji --- lib/pleroma/emoji/loader.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index eb6f6816b..b6e544323 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -15,8 +15,6 @@ defmodule Pleroma.Emoji.Loader do require Logger - @mix_env Mix.env() - @type pattern :: Regex.t() | module() | String.t() @type patterns :: pattern() | [pattern()] @type group_patterns :: keyword(patterns()) @@ -79,7 +77,7 @@ defmodule Pleroma.Emoji.Loader do # for testing emoji.txt entries we do not want exposed in normal operation test_emoji = - if @mix_env == :test do + if Application.get_env(:pleroma, __MODULE__)[:test_emoji] do load_from_file("test/config/emoji.txt", emoji_groups) else [] -- cgit v1.2.3 From 38ebefce9c25cf59e6c9e95969712bd13fa79df3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 21 Jan 2024 15:54:20 -0500 Subject: Announcement: fix dialyzer errors and add typespec for the changeset It was possible for this to raise (no_local_return) because the data key could be missing from the params --- lib/pleroma/announcement.ex | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/announcement.ex b/lib/pleroma/announcement.ex index d97c5e728..5a3c710e8 100644 --- a/lib/pleroma/announcement.ex +++ b/lib/pleroma/announcement.ex @@ -23,19 +23,21 @@ defmodule Pleroma.Announcement do timestamps(type: :utc_datetime) end - def change(struct, params \\ %{}) do - struct - |> cast(validate_params(struct, params), [:data, :starts_at, :ends_at, :rendered]) + @doc "Generates changeset for %Pleroma.Announcement{}" + @spec changeset(%__MODULE__{}, map()) :: %Ecto.Changeset{} + def changeset(announcement \\ %__MODULE__{}, params \\ %{data: %{}}) do + announcement + |> cast(validate_params(announcement, params), [:data, :starts_at, :ends_at, :rendered]) |> validate_required([:data]) end - defp validate_params(struct, params) do + defp validate_params(announcement, params) do base_data = %{ "content" => "", "all_day" => false } - |> Map.merge((struct && struct.data) || %{}) + |> Map.merge((announcement && announcement.data) || %{}) merged_data = Map.merge(base_data, params.data) @@ -61,13 +63,13 @@ defmodule Pleroma.Announcement do end def add(params) do - changeset = change(%__MODULE__{}, params) + changeset = changeset(%__MODULE__{}, params) Repo.insert(changeset) end def update(announcement, params) do - changeset = change(announcement, params) + changeset = changeset(announcement, params) Repo.update(changeset) end -- cgit v1.2.3 From 13618562138375a55a1286c15c82b011594a5fbd Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 17:11:22 -0500 Subject: Pleroma.User.Backup: fix some dialyzer errors lib/pleroma/user/backup.ex:207:call The function call will not succeed. :zip.create( string(), [:"\"actor.json\"", :"\"outbox.json\"", :"\"likes.json\"", :"\"bookmarks.json\""], [{:cwd, binary()}, ...] ) will never return since the success typing is: ( atom() | [atom() | [any()] | char()], [ atom() | [atom() | [any()] | char()] | {atom() | [atom() | [any()] | char()], binary()} | {atom() | [atom() | [any()] | char()], binary(), {:file_info, :undefined | non_neg_integer(), :device | :directory | :other | :regular | :symlink | :undefined, :none | :read | :read_write | :undefined | :write, :undefined | non_neg_integer() | {_, _}, :undefined | non_neg_integer() | {_, _}, :undefined | non_neg_integer() | {_, _}, :undefined | non_neg_integer(), :undefined | non_neg_integer(), :undefined | non_neg_integer(), :undefined | non_neg_integer(), :undefined | non_neg_integer(), :undefined | non_neg_integer(), :undefined | non_neg_integer()}} ], [ :cooked | :memory | :verbose | {:comment, string()} | {:compress, :all | [[any()]] | {:add, [any()]} | {:del, [any()]}} | {:cwd, string()} | {:uncompress, :all | [[any()]] | {:add, [any()]} | {:del, [any()]}} ] ) :: {:error, _} | {:ok, atom() | [atom() | [any()] | char()] | {atom() | [atom() | [any()] | char()], binary()}} and the contract is (name, fileList, options) :: retValue when name: :file.name(), fileList: [:FileSpec], fileSpec: :file.name() | {:file.name(), binary()} | {:file.name(), binary(), :file.file_info()}, options: [:Option], option: create_option(), retValue: {:ok, FileName :: filename()} | {:ok, {FileName :: filename(), binary()}} | {:error, Reason :: term()} --- lib/pleroma/user/backup.ex | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 74e0ec073..b7f00bbf7 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -22,6 +22,8 @@ defmodule Pleroma.User.Backup do alias Pleroma.Web.ActivityPub.UserView alias Pleroma.Workers.BackupWorker + @type t :: %__MODULE__{} + schema "backups" do field(:content_type, :string) field(:file_name, :string) @@ -195,6 +197,7 @@ defmodule Pleroma.User.Backup do end @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + @spec export(Pleroma.User.Backup.t(), pid()) :: {:ok, String.t()} | :error def export(%__MODULE__{} = backup, caller_pid) do backup = Repo.preload(backup, :user) dir = backup_tempdir(backup) @@ -204,9 +207,11 @@ defmodule Pleroma.User.Backup do :ok <- statuses(dir, backup.user, caller_pid), :ok <- likes(dir, backup.user, caller_pid), :ok <- bookmarks(dir, backup.user, caller_pid), - {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), + {:ok, zip_path} <- :zip.create(backup.file_name, @files, cwd: dir), {:ok, _} <- File.rm_rf(dir) do - {:ok, to_string(zip_path)} + {:ok, zip_path} + else + _ -> :error end end @@ -382,6 +387,8 @@ defmodule Pleroma.User.Backup.Processor do [:file_size, :processed, :state] ) |> Repo.update() + else + e -> {:error, e} end end end -- cgit v1.2.3 From 40feac086f3efd463f7aeb630adf55a0a1fa4365 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 17:13:45 -0500 Subject: Pleroma.User: fix dialyzer errors lib/pleroma/user.ex:1514:unknown_type Unknown type: Pleroma.UserRelationship.t/0. lib/pleroma/user.ex:2629:unknown_type Unknown type: Pleroma.UserRelationship.t/0. lib/pleroma/user.ex:2638:unknown_type Unknown type: Pleroma.UserRelationship.t/0. --- lib/pleroma/user_relationship.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/user_relationship.ex b/lib/pleroma/user_relationship.ex index fbecf3129..82fcc1cdd 100644 --- a/lib/pleroma/user_relationship.ex +++ b/lib/pleroma/user_relationship.ex @@ -14,6 +14,8 @@ defmodule Pleroma.UserRelationship do alias Pleroma.User alias Pleroma.UserRelationship + @type t :: %__MODULE__{} + schema "user_relationships" do belongs_to(:source, User, type: FlakeId.Ecto.CompatType) belongs_to(:target, User, type: FlakeId.Ecto.CompatType) -- cgit v1.2.3 From 10f3a2833fe6f6909b6540343955c05392763d17 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 17:17:13 -0500 Subject: Pleroma.User.Query: fix dialyzer error lib/pleroma/user/query.ex:74:unknown_type Unknown type: Query.t/0. --- lib/pleroma/user/query.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 874e8ec2b..cd9586452 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -71,7 +71,7 @@ defmodule Pleroma.User.Query do @equal_criteria [:email] @contains_criteria [:ap_id, :nickname] - @spec build(Query.t(), criteria()) :: Query.t() + @spec build(Ecto.Query.t(), criteria()) :: Ecto.Query.t() def build(query \\ base_query(), criteria) do prepare_query(query, criteria) end -- cgit v1.2.3 From 39da451b6d76bb84a7e070525a1946aa1c80bc51 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 17:20:21 -0500 Subject: Pleroma.Web.ActivityPub.Builder: fix dialyzer errors lib/pleroma/web/activity_pub/builder.ex:35:unknown_type Unknown type: Activity.t/0. lib/pleroma/web/activity_pub/builder.ex:40:unknown_type Unknown type: Activity.t/0. lib/pleroma/web/activity_pub/builder.ex:144:unknown_type Unknown type: Activity.t/0. ________________________________________________________________________________ lib/pleroma/web/activity_pub/builder.ex:204:unknown_type Unknown type: Pleroma.Web.CommonAPI.ActivityDraft.t/0. --- lib/pleroma/web/activity_pub/builder.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index eb0bb0e33..404975757 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do This module encodes our addressing policies and general shape of our objects. """ + alias Pleroma.Activity alias Pleroma.Emoji alias Pleroma.Object alias Pleroma.User -- cgit v1.2.3 From 36355d3ed971e56274f7b09b0c8ce7d3f5c0da14 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 17:22:36 -0500 Subject: Pleroma.Web.ActivityPub.Builder: fix dialyzer error lib/pleroma/web/activity_pub/builder.ex:205:unknown_type Unknown type: Pleroma.Web.CommonAPI.ActivityDraft.t/0. --- lib/pleroma/web/common_api/activity_draft.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 8910ad5b8..bc46a8a36 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -14,6 +14,8 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do import Pleroma.Web.Gettext import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + @type t :: %__MODULE__{} + defstruct valid?: true, errors: [], user: nil, -- cgit v1.2.3 From c74c5f479aff38c67c3c9d0b3d0822f06a3d2f9c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 17:24:01 -0500 Subject: Pleroma.Migrators.Support.BaseMigratorState: fix dialyzer error lib/pleroma/migrators/support/base_migrator_state.ex:10:unknown_type Unknown type: Pleroma.DataMigration.t/0. --- lib/pleroma/data_migration.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/data_migration.ex b/lib/pleroma/data_migration.ex index 8451678fc..be4bf6489 100644 --- a/lib/pleroma/data_migration.ex +++ b/lib/pleroma/data_migration.ex @@ -12,6 +12,8 @@ defmodule Pleroma.DataMigration do import Ecto.Changeset import Ecto.Query + @type t :: %__MODULE__{} + schema "data_migrations" do field(:name, :string) field(:state, State, default: :pending) -- cgit v1.2.3 From 65d49ac090bfe70cfea9b00bae79f9cba5c50c43 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 17:26:04 -0500 Subject: Pleroma.HTTP.AdapterHelper: fix dialyzer errors lib/pleroma/http/adapter_helper.ex:18:unknown_type Unknown type: Connection.host/0. lib/pleroma/http/adapter_helper.ex:19:unknown_type Unknown type: Connection.host/0. lib/pleroma/http/adapter_helper.ex:19:unknown_type Unknown type: Connection.proxy_type/0. --- lib/pleroma/http/adapter_helper.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index e9bb2023a..dcb27a29d 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -15,8 +15,8 @@ defmodule Pleroma.HTTP.AdapterHelper do require Logger @type proxy :: - {Connection.host(), pos_integer()} - | {Connection.proxy_type(), Connection.host(), pos_integer()} + {host(), pos_integer()} + | {proxy_type(), host(), pos_integer()} @callback options(keyword(), URI.t()) :: keyword() -- cgit v1.2.3 From 6ce7011a2e07d9e8e7d7fdc1a6fe340bac6e1404 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 17:48:55 -0500 Subject: Pleroma.Gun.ConnectionPool.WorkerSupervisor: fix dialyzer error lib/pleroma/gun/connection_pool/worker_supervisor.ex:24:guard_fail The guard clause: when _ :: true === nil can never succeed. --- lib/pleroma/gun/connection_pool/worker_supervisor.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index d26a70be3..b2be4ff87 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do def start_worker(opts, retry \\ false) do case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do {:error, :max_children} -> - if retry or free_pool() == :error do + if Enum.any?([retry, free_pool()], &match?(&1, :error)) do :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) {:error, :pool_full} else -- cgit v1.2.3 From a7fa6f18dc7919b3a441fcea2a35e1c313f89555 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 18:05:41 -0500 Subject: Pleroma.Migrators.Support.BaseMigrator: Fix dialyzer errors lib/pleroma/migrators/context_objects_deletion_migrator.ex:13:exact_eq The test :error | float() == 0 can never evaluate to 'true'. lib/pleroma/migrators/hashtags_table_migrator.ex:13:exact_eq The test :error | float() == 0 can never evaluate to 'true'. --- lib/pleroma/migrators/support/base_migrator.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/support/base_migrator.ex b/lib/pleroma/migrators/support/base_migrator.ex index ce88caac7..76a5d4590 100644 --- a/lib/pleroma/migrators/support/base_migrator.ex +++ b/lib/pleroma/migrators/support/base_migrator.ex @@ -188,10 +188,11 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do end defp fault_rate do - with failures_count when is_integer(failures_count) <- failures_count() do + with failures_count when is_integer(failures_count) <- failures_count(), + true <- failures_count > 0 do failures_count / Enum.max([get_stat(:affected_count, 0), 1]) else - _ -> :error + _ -> 0 end end -- cgit v1.2.3 From 5f71928f6b12584ba456cbeb8a92c38078a468ae Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 18:09:15 -0500 Subject: MRF.InlineQuotePolicy: fix dialyzer error lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex:60:callback_type_mismatch Type mismatch for @callback config_description/0 in Pleroma.Web.ActivityPub.MRF.Policy behaviour. Expected type: %{ :description => binary(), :key => atom(), :label => binary(), :related_policy => binary(), :children => [map()] } Actual type: %{ :children => [ %{ :description => <<_::808>>, :key => :template, :suggestions => [any(), ...], :type => :string }, ... ], :description => <<_::336>>, :key => :mrf_inline_quote, :label => <<_::184>>, :related_policy => <<_::360>>, :type => :group } --- lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index 171b22c5e..b7a01c27c 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -62,7 +62,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do key: :mrf_inline_quote, related_policy: "Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy", label: "MRF Inline Quote Policy", - type: :group, description: "Force quote url to appear in post content.", children: [ %{ -- cgit v1.2.3 From 0dd65246eac9c1c738cc4ea47798caec1797ad6d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 18:11:15 -0500 Subject: MRF.HashtagPolicy: fix dialyzer error lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex:87:exact_eq The test <<_::32>> == <<_::48>> can never evaluate to 'true'. --- lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index d13d980cc..fdb9a9dba 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -84,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do if hashtags != [] do with {:ok, message} <- check_reject(message, hashtags), {:ok, message} <- - (if "type" == "Create" do + (if type == "Create" do check_ftl_removal(message, hashtags) else {:ok, message} -- cgit v1.2.3 From 115b2ad63875b1dc92d194e7e17b6ccde3b3d395 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 18:27:33 -0500 Subject: MRF.KeywordPolicy: fix dialyzer error lib/pleroma/web/activity_pub/mrf/keyword_policy.ex:13:neg_guard_fail Guard test: not is_binary(_string :: binary()) can never succeed. --- lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 874fe9ab9..729da4e9c 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -10,15 +10,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do @moduledoc "Reject or Word-Replace messages with a keyword or regex" @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp string_matches?(string, _) when not is_binary(string) do - false - end defp string_matches?(string, pattern) when is_binary(pattern) do String.contains?(string, pattern) end - defp string_matches?(string, pattern) do + defp string_matches?(string, %Regex{} = pattern) do String.match?(string, pattern) end -- cgit v1.2.3 From 3a8594e92759aa10d3e6ad5b4a1892beb6d74cc9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jan 2024 18:35:07 -0500 Subject: MastodonAPI.Controller.StatusController: fix dialyzer error lib/pleroma/web/mastodon_api/controllers/status_controller.ex:333:pattern_match The pattern can never match the type. Pattern: {:ok, _activity} Type: {:error, _} --- lib/pleroma/web/common_api.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index dfc0a625d..61d9e1c7c 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -505,7 +505,7 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec unpin(String.t(), User.t()) :: {:ok, User.t()} | {:error, term()} + @spec unpin(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, term()} def unpin(id, user) do with %Activity{} = activity <- create_activity_by_id(id), {:ok, unpin_data, _} <- Builder.unpin(user, activity.object), -- cgit v1.2.3 From 0de1a7629c5dbf3f6fba69a41bbeff5947558899 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 25 Jan 2024 00:26:55 +0100 Subject: Maps: Add filter_empty_values/1 --- lib/pleroma/maps.ex | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/maps.ex b/lib/pleroma/maps.ex index 6d586e53e..5020a8ff8 100644 --- a/lib/pleroma/maps.ex +++ b/lib/pleroma/maps.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors +# Copyright © 2017-2024 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Maps do @@ -18,4 +18,17 @@ defmodule Pleroma.Maps do rescue _ -> data end + + def filter_empty_values(data) do + # TODO: Change to Map.filter in Elixir 1.13+ + data + |> Enum.filter(fn + {_k, nil} -> false + {_k, ""} -> false + {_k, []} -> false + {_k, %{} = v} -> Map.keys(v) != [] + {_k, _v} -> true + end) + |> Map.new() + end end -- cgit v1.2.3 From acef2a4a407058c6dfa4e18aaa9920ab1a82eb94 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 25 Jan 2024 00:27:15 +0100 Subject: CommonFixes: Use Maps.filter_empty_values on fix_object_defaults --- lib/pleroma/web/activity_pub/object_validators/common_fixes.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 4d9be0bdd..99c62cb4e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Maps alias Pleroma.Object alias Pleroma.Object.Containment alias Pleroma.User @@ -24,6 +25,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do end def fix_object_defaults(data) do + data = Maps.filter_empty_values(data) + context = Utils.maybe_create_context( data["context"] || data["conversation"] || data["inReplyTo"] || data["id"] -- cgit v1.2.3 From 799891d359363ab66fc0e351ad8140b689027fc4 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 26 Jan 2024 17:10:10 +0100 Subject: Transmogrifier: Cleanup obsolete handling of `"contentMap": null` --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ---- 1 file changed, 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a4cf395f2..80f64cf1e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -336,10 +336,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_tag(object), do: object - def fix_content_map(%{"contentMap" => nil} = object) do - Map.drop(object, ["contentMap"]) - end - # content map usually only has one language so this will do for now. def fix_content_map(%{"contentMap" => content_map} = object) do content_groups = Map.to_list(content_map) -- cgit v1.2.3 From 18d38486a568f8cda8ab0f6227959810b253e75a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 Jan 2024 15:57:50 -0500 Subject: InetCidr.parse/2 is deprecated --- lib/pleroma/web/plugs/remote_ip.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/remote_ip.ex b/lib/pleroma/web/plugs/remote_ip.ex index f207d9fef..9f733a96f 100644 --- a/lib/pleroma/web/plugs/remote_ip.ex +++ b/lib/pleroma/web/plugs/remote_ip.ex @@ -43,6 +43,6 @@ defmodule Pleroma.Web.Plugs.RemoteIp do InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" end - InetCidr.parse(proxy, true) + InetCidr.parse_cidr!(proxy, true) end end -- cgit v1.2.3 From 5b95abaeea886a41b26d4f3134bfe3faf60b1abe Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 26 Jan 2024 16:55:08 -0500 Subject: Credo.Check.Readability.PredicateFunctionNames This check was recently improved in Credo and it does make sense for readability. The offending functions in Pleroma have been renamed and a couple missing the ? suffix have been fixed as well. --- lib/pleroma/conversation.ex | 2 +- lib/pleroma/emoji.ex | 10 ++--- lib/pleroma/gopher/server.ex | 2 +- lib/pleroma/user.ex | 6 +-- lib/pleroma/web/activity_pub/activity_pub.ex | 10 ++--- lib/pleroma/web/activity_pub/builder.ex | 6 +-- .../web/activity_pub/mrf/no_empty_policy.ex | 16 +++---- .../activity_pub/mrf/quote_to_link_tag_policy.ex | 2 +- .../object_validators/announce_validator.ex | 2 +- .../activity_pub/object_validators/common_fixes.ex | 6 +-- .../object_validators/emoji_react_validator.ex | 8 ++-- lib/pleroma/web/activity_pub/pipeline.ex | 2 +- lib/pleroma/web/activity_pub/publisher.ex | 6 +-- lib/pleroma/web/activity_pub/relay.ex | 2 +- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- lib/pleroma/web/activity_pub/utils.ex | 4 +- lib/pleroma/web/activity_pub/visibility.ex | 50 +++++++++++----------- lib/pleroma/web/common_api.ex | 4 +- lib/pleroma/web/common_api/utils.ex | 2 +- lib/pleroma/web/embed_controller.ex | 2 +- lib/pleroma/web/gettext.ex | 4 +- lib/pleroma/web/mastodon_api/views/account_view.ex | 4 +- lib/pleroma/web/o_auth/token.ex | 4 +- lib/pleroma/web/o_status/o_status_controller.ex | 8 ++-- lib/pleroma/web/plugs/o_auth_plug.ex | 4 +- .../web/rich_media/parser/ttl/aws_signed_url.ex | 6 +-- lib/pleroma/web/static_fe/static_fe_controller.ex | 2 +- lib/pleroma/web/streamer.ex | 2 +- .../controllers/remote_follow_controller.ex | 4 +- 30 files changed, 92 insertions(+), 92 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/conversation.ex b/lib/pleroma/conversation.ex index 42028aa51..0be609a22 100644 --- a/lib/pleroma/conversation.ex +++ b/lib/pleroma/conversation.ex @@ -57,7 +57,7 @@ defmodule Pleroma.Conversation do 3. Bump all relevant participations to 'unread' """ def create_or_bump_for(activity, opts \\ []) do - with true <- Pleroma.Web.ActivityPub.Visibility.is_direct?(activity), + with true <- Pleroma.Web.ActivityPub.Visibility.direct?(activity), "Create" <- activity.data["type"], %Object{} = object <- Object.normalize(activity, fetch: false), true <- object.data["type"] in ["Note", "Question"], diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 48302f396..c2b71b4d2 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -138,23 +138,23 @@ defmodule Pleroma.Emoji do emojis = emojis ++ regional_indicators for emoji <- emojis do - def is_unicode_emoji?(unquote(emoji)), do: true + def unicode?(unquote(emoji)), do: true end - def is_unicode_emoji?(_), do: false + def unicode?(_), do: false @emoji_regex ~r/:[A-Za-z0-9_-]+(@.+)?:/ - def is_custom_emoji?(s) when is_binary(s), do: Regex.match?(@emoji_regex, s) + def custom?(s) when is_binary(s), do: Regex.match?(@emoji_regex, s) - def is_custom_emoji?(_), do: false + def custom?(_), do: false def maybe_strip_name(name) when is_binary(name), do: String.trim(name, ":") def maybe_strip_name(name), do: name def maybe_quote(name) when is_binary(name) do - if is_unicode_emoji?(name) do + if unicode?(name) do name else if String.starts_with?(name, ":") do diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index 0fde0adcf..54245c9fa 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -114,7 +114,7 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do def response("/notices/" <> id) do with %Activity{} = activity <- Activity.get_by_id(id), - true <- Visibility.is_public?(activity) do + true <- Visibility.public?(activity) do activities = ActivityPub.fetch_activities_for_context(activity.data["context"]) |> render_activities diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 89a95c435..3e407171a 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2404,9 +2404,9 @@ defmodule Pleroma.User do defp put_password_hash(changeset), do: changeset - def is_internal_user?(%User{nickname: nil}), do: true - def is_internal_user?(%User{local: true, nickname: "internal." <> _}), do: true - def is_internal_user?(_), do: false + def internal?(%User{nickname: nil}), do: true + def internal?(%User{local: true, nickname: "internal." <> _}), do: true + def internal?(_), do: false # A hack because user delete activities have a fake id for whatever reason # TODO: Get rid of this diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a12438f56..2017c696d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -74,22 +74,22 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp check_remote_limit(_), do: true def increase_note_count_if_public(actor, object) do - if is_public?(object), do: User.increase_note_count(actor), else: {:ok, actor} + if public?(object), do: User.increase_note_count(actor), else: {:ok, actor} end def decrease_note_count_if_public(actor, object) do - if is_public?(object), do: User.decrease_note_count(actor), else: {:ok, actor} + if public?(object), do: User.decrease_note_count(actor), else: {:ok, actor} end def update_last_status_at_if_public(actor, object) do - if is_public?(object), do: User.update_last_status_at(actor), else: {:ok, actor} + if public?(object), do: User.update_last_status_at(actor), else: {:ok, actor} end defp increase_replies_count_if_reply(%{ "object" => %{"inReplyTo" => reply_ap_id} = object, "type" => "Create" }) do - if is_public?(object) do + if public?(object) do Object.increase_replies_count(reply_ap_id) end end @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do "object" => %{"quoteUrl" => quote_ap_id} = object, "type" => "Create" }) do - if is_public?(object) do + if public?(object) do Object.increase_quotes_count(quote_ap_id) end end diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 404975757..2a1e56278 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -132,7 +132,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do def emoji_react(actor, object, emoji) do with {:ok, data, meta} <- object_action(actor, object) do data = - if Emoji.is_unicode_emoji?(emoji) do + if Emoji.unicode?(emoji) do unicode_emoji_react(object, data, emoji) else custom_emoji_react(object, data, emoji) @@ -348,7 +348,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do actor.ap_id == Relay.ap_id() -> [actor.follower_address] - public? and Visibility.is_local_public?(object) -> + public? and Visibility.local_public?(object) -> [actor.follower_address, object.data["actor"], Utils.as_local_public()] public? -> @@ -376,7 +376,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do # Address the actor of the object, and our actor's follower collection if the post is public. to = - if Visibility.is_public?(object) do + if Visibility.public?(object) do [actor.follower_address, object.data["actor"]] else [object.data["actor"]] diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index 855cda3b9..12bf4ddd2 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -10,9 +10,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do @impl true def filter(%{"actor" => actor} = object) do - with true <- is_local?(actor), - true <- is_eligible_type?(object), - true <- is_note?(object), + with true <- local?(actor), + true <- eligible_type?(object), + true <- note?(object), false <- has_attachment?(object), true <- only_mentions?(object) do {:reject, "[NoEmptyPolicy]"} @@ -24,7 +24,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do def filter(object), do: {:ok, object} - defp is_local?(actor) do + defp local?(actor) do if actor |> String.starts_with?("#{Endpoint.url()}") do true else @@ -59,11 +59,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do defp only_mentions?(_), do: false - defp is_note?(%{"object" => %{"type" => "Note"}}), do: true - defp is_note?(_), do: false + defp note?(%{"object" => %{"type" => "Note"}}), do: true + defp note?(_), do: false - defp is_eligible_type?(%{"type" => type}) when type in ["Create", "Update"], do: true - defp is_eligible_type?(_), do: false + defp eligible_type?(%{"type" => type}) when type in ["Create", "Update"], do: true + defp eligible_type?(_), do: false @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex index f1c573d1b..ac353f03f 100644 --- a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do tags = object["tag"] || [] if Enum.any?(tags, fn tag -> - CommonFixes.is_object_link_tag(tag) and tag["href"] == quote_url + CommonFixes.object_link_tag?(tag) and tag["href"] == quote_url end) do object else diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex index c2c7ba1a8..d0218583e 100644 --- a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex @@ -82,7 +82,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do object when is_binary(object) <- get_field(cng, :object), %User{} = actor <- User.get_cached_by_ap_id(actor), %Object{} = object <- Object.get_cached_by_ap_id(object), - false <- Visibility.is_public?(object) do + false <- Visibility.public?(object) do same_actor = object.data["actor"] == actor.ap_id recipients = get_field(cng, :to) ++ get_field(cng, :cc) local_public = Utils.as_local_public() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 4d9be0bdd..ab56ba468 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -99,7 +99,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do end def fix_quote_url(%{"tag" => [_ | _] = tags} = data) do - tag = Enum.find(tags, &is_object_link_tag/1) + tag = Enum.find(tags, &object_link_tag?/1) if not is_nil(tag) do data @@ -112,7 +112,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do def fix_quote_url(data), do: data # https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md - def is_object_link_tag(%{ + def object_link_tag?(%{ "type" => "Link", "mediaType" => media_type, "href" => href @@ -121,5 +121,5 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do true end - def is_object_link_tag(_), do: false + def object_link_tag?(_), do: false end diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex index a0b82b325..65ba047e6 100644 --- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex @@ -74,10 +74,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do new_emoji = Pleroma.Emoji.fully_qualify_emoji(emoji) cond do - Pleroma.Emoji.is_unicode_emoji?(emoji) -> + Pleroma.Emoji.unicode?(emoji) -> data - Pleroma.Emoji.is_unicode_emoji?(new_emoji) -> + Pleroma.Emoji.unicode?(new_emoji) -> data |> Map.put("content", new_emoji) true -> @@ -90,7 +90,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do defp validate_emoji(cng) do content = get_field(cng, :content) - if Emoji.is_unicode_emoji?(content) || Emoji.is_custom_emoji?(content) do + if Emoji.unicode?(content) || Emoji.custom?(content) do cng else cng @@ -101,7 +101,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do defp maybe_validate_tag_presence(cng) do content = get_field(cng, :content) - if Emoji.is_unicode_emoji?(content) do + if Emoji.unicode?(content) do cng else tag = get_field(cng, :tag) diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index ca8653ab1..40184bd97 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -62,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do with {:ok, local} <- Keyword.fetch(meta, :local) do do_not_federate = meta[:do_not_federate] || !config().get([:instance, :federating]) - if !do_not_federate and local and not Visibility.is_local_public?(activity) do + if !do_not_federate and local and not Visibility.local_public?(activity) do activity = if object = Keyword.get(meta, :object_data) do %{activity | data: Map.put(activity.data, "object", object)} diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 2449a3a3e..c27612697 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -66,7 +66,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do @doc """ Determine if an activity can be represented by running it through Transmogrifier. """ - def is_representable?(%Activity{} = activity) do + def representable?(%Activity{} = activity) do with {:ok, _data} <- Transmogrifier.prepare_outgoing(activity.data) do true else @@ -246,7 +246,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity) when is_list(bcc) and bcc != [] do - public = is_public?(activity) + public = public?(activity) {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) [priority_recipients, recipients] = recipients(actor, activity) @@ -291,7 +291,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do # Publishes an activity to all relevant peers. def publish(%User{} = actor, %Activity{} = activity) do - public = is_public?(activity) + public = public?(activity) if public && Config.get([:instance, :allow_relay]) do Logger.debug(fn -> "Relaying #{activity.data["id"]} out" end) diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index 2010351d1..91a647f29 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do @spec publish(any()) :: {:ok, Activity.t()} | {:error, any()} def publish(%Activity{data: %{"type" => "Create"}} = activity) do with %User{} = user <- get_actor(), - true <- Visibility.is_public?(activity) do + true <- Visibility.public?(activity) do CommonAPI.repeat(activity.id, user) else error -> format_error(error) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 59b19180d..eec04e6a5 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -258,7 +258,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Utils.add_announce_to_object(object, announced_object) - if !User.is_internal_user?(user) do + if !User.internal?(user) do Notification.create_notifications(object) ap_streamer().stream_out(object) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a4cf395f2..69854b084 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -783,7 +783,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Object.normalize(fetch: false) data = - if Visibility.is_private?(object) && object.data["actor"] == ap_id do + if Visibility.private?(object) && object.data["actor"] == ap_id do data |> Map.put("object", object |> Map.get(:data) |> prepare_object) else data |> maybe_fix_object_url diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index e2fc2640d..37a96e7d7 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -167,7 +167,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do with true <- Config.get!([:instance, :federating]), true <- type != "Block" || outgoing_blocks, - false <- Visibility.is_local_public?(activity) do + false <- Visibility.local_public?(activity) do Pleroma.Web.Federator.publish(activity) end @@ -277,7 +277,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do object_actor = User.get_cached_by_ap_id(object_actor_id) to = - if Visibility.is_public?(object) do + if Visibility.public?(object) do [actor.follower_address, object.data["actor"]] else [object.data["actor"]] diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex index 7c57f88f9..97fc7fa1b 100644 --- a/lib/pleroma/web/activity_pub/visibility.ex +++ b/lib/pleroma/web/activity_pub/visibility.ex @@ -11,28 +11,28 @@ defmodule Pleroma.Web.ActivityPub.Visibility do require Pleroma.Constants - @spec is_public?(Object.t() | Activity.t() | map()) :: boolean() - def is_public?(%Object{data: %{"type" => "Tombstone"}}), do: false - def is_public?(%Object{data: data}), do: is_public?(data) - def is_public?(%Activity{data: %{"type" => "Move"}}), do: true - def is_public?(%Activity{data: data}), do: is_public?(data) - def is_public?(%{"directMessage" => true}), do: false - - def is_public?(data) do + @spec public?(Object.t() | Activity.t() | map()) :: boolean() + def public?(%Object{data: %{"type" => "Tombstone"}}), do: false + def public?(%Object{data: data}), do: public?(data) + def public?(%Activity{data: %{"type" => "Move"}}), do: true + def public?(%Activity{data: data}), do: public?(data) + def public?(%{"directMessage" => true}), do: false + + def public?(data) do Utils.label_in_message?(Pleroma.Constants.as_public(), data) or Utils.label_in_message?(Utils.as_local_public(), data) end - def is_local_public?(%Object{data: data}), do: is_local_public?(data) - def is_local_public?(%Activity{data: data}), do: is_local_public?(data) + def local_public?(%Object{data: data}), do: local_public?(data) + def local_public?(%Activity{data: data}), do: local_public?(data) - def is_local_public?(data) do + def local_public?(data) do Utils.label_in_message?(Utils.as_local_public(), data) and not Utils.label_in_message?(Pleroma.Constants.as_public(), data) end - def is_private?(activity) do - with false <- is_public?(activity), + def private?(activity) do + with false <- public?(activity), %User{follower_address: follower_address} <- User.get_cached_by_ap_id(activity.data["actor"]) do follower_address in activity.data["to"] @@ -41,20 +41,20 @@ defmodule Pleroma.Web.ActivityPub.Visibility do end end - def is_announceable?(activity, user, public \\ true) do - is_public?(activity) || - (!public && is_private?(activity) && activity.data["actor"] == user.ap_id) + def announceable?(activity, user, public \\ true) do + public?(activity) || + (!public && private?(activity) && activity.data["actor"] == user.ap_id) end - def is_direct?(%Activity{data: %{"directMessage" => true}}), do: true - def is_direct?(%Object{data: %{"directMessage" => true}}), do: true + def direct?(%Activity{data: %{"directMessage" => true}}), do: true + def direct?(%Object{data: %{"directMessage" => true}}), do: true - def is_direct?(activity) do - !is_public?(activity) && !is_private?(activity) + def direct?(activity) do + !public?(activity) && !private?(activity) end - def is_list?(%{data: %{"listMessage" => _}}), do: true - def is_list?(_), do: false + def list?(%{data: %{"listMessage" => _}}), do: true + def list?(_), do: false @spec visible_for_user?(Object.t() | Activity.t() | nil, User.t() | nil) :: boolean() def visible_for_user?(%Object{data: %{"type" => "Tombstone"}}, _), do: false @@ -77,7 +77,7 @@ defmodule Pleroma.Web.ActivityPub.Visibility do when module in [Activity, Object] do if restrict_unauthenticated_access?(message), do: false, - else: is_public?(message) and not is_local_public?(message) + else: public?(message) and not local_public?(message) end def visible_for_user?(%{__struct__: module} = message, user) @@ -86,8 +86,8 @@ defmodule Pleroma.Web.ActivityPub.Visibility do y = [message.data["actor"]] ++ message.data["to"] ++ (message.data["cc"] || []) user_is_local = user.local - federatable = not is_local_public?(message) - (is_public?(message) || Enum.any?(x, &(&1 in y))) and (user_is_local || federatable) + federatable = not local_public?(message) + (public?(message) || Enum.any?(x, &(&1 in y))) and (user_is_local || federatable) end def entire_thread_visible_for_user?(%Activity{} = activity, %User{} = user) do diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 61d9e1c7c..27e82ecc8 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -372,7 +372,7 @@ defmodule Pleroma.Web.CommonAPI do do: visibility in ~w(public unlisted) def public_announce?(object, _) do - Visibility.is_public?(object) + Visibility.public?(object) end def get_visibility(_, _, %Participation{}), do: {"direct", "direct"} @@ -500,7 +500,7 @@ defmodule Pleroma.Web.CommonAPI do end defp activity_is_public(activity) do - with false <- Visibility.is_public?(activity) do + with false <- Visibility.public?(activity) do {:error, :visibility_error} end end diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index dcda3e0e8..52c08f00f 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -109,7 +109,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do def get_to_and_cc(%{visibility: "direct"} = draft) do # If the OP is a DM already, add the implicit actor. - if draft.in_reply_to && Visibility.is_direct?(draft.in_reply_to) do + if draft.in_reply_to && Visibility.direct?(draft.in_reply_to) do {Enum.uniq([draft.in_reply_to.data["actor"] | draft.mentions]), []} else {draft.mentions, []} diff --git a/lib/pleroma/web/embed_controller.ex b/lib/pleroma/web/embed_controller.ex index 8b9f0a051..ab0df9c5a 100644 --- a/lib/pleroma/web/embed_controller.ex +++ b/lib/pleroma/web/embed_controller.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.EmbedController do def show(conn, %{"id" => id}) do with %Activity{local: true} = activity <- Activity.get_by_id_with_object(id), - true <- Visibility.is_public?(activity.object) do + true <- Visibility.public?(activity.object) do {:ok, author} = User.get_or_fetch(activity.object.data["actor"]) conn diff --git a/lib/pleroma/web/gettext.ex b/lib/pleroma/web/gettext.ex index 5ef49d841..1fa3f9768 100644 --- a/lib/pleroma/web/gettext.ex +++ b/lib/pleroma/web/gettext.ex @@ -85,12 +85,12 @@ defmodule Pleroma.Web.Gettext do Process.get({Pleroma.Web.Gettext, :locales}, []) end - def is_locale_list(locales) do + def locale_list?(locales) do Enum.all?(locales, &is_binary/1) end def put_locales(locales) do - if is_locale_list(locales) do + if locale_list?(locales) do Process.put({Pleroma.Web.Gettext, :locales}, Enum.uniq(locales)) Gettext.put_locale(Enum.at(locales, 0, Gettext.get_locale())) :ok diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index df8fdc8b8..267c3e3ed 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -214,7 +214,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do do: user.follower_count, else: 0 - bot = is_bot?(user) + bot = bot?(user) emojis = Enum.map(user.emoji, fn {shortcode, raw_url} -> @@ -471,7 +471,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp image_url(%{"url" => [%{"href" => href} | _]}), do: href defp image_url(_), do: nil - defp is_bot?(user) do + defp bot?(user) do # Because older and/or Mastodon clients may not recognize a Group actor properly, # and currently the group actor can only boost things, we should let these clients # think groups are bots. diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index d4a7e8999..a5ad2e909 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -138,9 +138,9 @@ defmodule Pleroma.Web.OAuth.Token do |> Repo.all() end - def is_expired?(%__MODULE__{valid_until: valid_until}) do + def expired?(%__MODULE__{valid_until: valid_until}) do NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) > 0 end - def is_expired?(_), do: false + def expired?(_), do: false end diff --git a/lib/pleroma/web/o_status/o_status_controller.ex b/lib/pleroma/web/o_status/o_status_controller.ex index ea4994bd0..4f2cf02c3 100644 --- a/lib/pleroma/web/o_status/o_status_controller.ex +++ b/lib/pleroma/web/o_status/o_status_controller.ex @@ -37,7 +37,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do with id <- Endpoint.url() <> conn.request_path, {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id_with_object(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)} do + {_, true} <- {:public?, Visibility.public?(activity)} do redirect(conn, to: "/notice/#{activity.id}") else reason when reason in [{:public?, false}, {:activity, nil}] -> @@ -56,7 +56,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do def activity(conn, _params) do with id <- Endpoint.url() <> conn.request_path, {_, %Activity{} = activity} <- {:activity, Activity.normalize(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)} do + {_, true} <- {:public?, Visibility.public?(activity)} do redirect(conn, to: "/notice/#{activity.id}") else reason when reason in [{:public?, false}, {:activity, nil}] -> @@ -69,7 +69,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do def notice(%{assigns: %{format: format}} = conn, %{"id" => id}) do with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, - {_, true} <- {:public?, Visibility.is_public?(activity)}, + {_, true} <- {:public?, Visibility.public?(activity)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do cond do format in ["json", "activity+json"] -> @@ -106,7 +106,7 @@ defmodule Pleroma.Web.OStatus.OStatusController do # Returns an HTML embedded