From e154ebbf7933123e91d5b5c6f5070e78eb3e383b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 15 Aug 2021 21:53:04 +0300 Subject: Initial meilisearch implementation, doesn't delete posts yet --- lib/mix/tasks/pleroma/search/meilisearch.ex | 38 ++++++++++++++ lib/pleroma/activity.ex | 1 + lib/pleroma/activity/search.ex | 4 +- lib/pleroma/application.ex | 6 ++- lib/pleroma/search/meilisearch.ex | 60 ++++++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 6 +++ .../mastodon_api/controllers/search_controller.ex | 5 +- 7 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 lib/mix/tasks/pleroma/search/meilisearch.ex create mode 100644 lib/pleroma/search/meilisearch.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..2af8e5853 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Meilisearch do + import Mix.Pleroma + + import Ecto.Query + + def run(["index"]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + Pleroma.Repo.chunk_stream( + from(Pleroma.Object, + limit: 200, + where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") + ), + 100, + :batches + ) + |> Stream.map(fn objects -> + Enum.map(objects, fn object -> + data = object.data + %{id: object.id, source: data["source"], ap: data["id"]} + end) + end) + |> Stream.each(fn activities -> + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!(activities) + ) + end) + |> Stream.run() + end +end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index ebfd4ed45..9563136f9 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -367,6 +367,7 @@ defmodule Pleroma.Activity do end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + def add_to_index(_activity), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 0b9b24aa4..3dce9d355 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -136,7 +136,7 @@ defmodule Pleroma.Activity.Search do ) end - defp maybe_restrict_local(q, user) do + def maybe_restrict_local(q, user) do limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) case {limit, user} do @@ -149,7 +149,7 @@ defmodule Pleroma.Activity.Search do defp restrict_local(q), do: where(q, local: true) - defp maybe_fetch(activities, user, search_query) do + def maybe_fetch(activities, user, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 1c1db8c10..62d1b8b39 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -321,7 +321,11 @@ defmodule Pleroma.Application do def limiters_setup do config = Config.get(ConcurrentLimiter, []) - [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy] + [ + Pleroma.Web.RichMedia.Helpers, + Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, + Pleroma.Search + ] |> Enum.each(fn module -> mod_config = Keyword.get(config, module, []) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..92e0d3429 --- /dev/null +++ b/lib/pleroma/search/meilisearch.ex @@ -0,0 +1,60 @@ +defmodule Pleroma.Search.Meilisearch do + require Logger + + alias Pleroma.Activity + + import Pleroma.Activity.Search + import Ecto.Query + + def search(user, query, options \\ []) do + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/search", + Jason.encode!(%{q: query, offset: offset, limit: limit}) + ) + + hits = Jason.decode!(result.body)["hits"] |> Enum.map(& &1["ap"]) + + try do + hits + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object() + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> maybe_fetch(user, query) + |> order_by([activity], desc: activity.id) + |> Pleroma.Repo.all() + rescue + _ -> maybe_fetch([], user, query) + end + end + + def add_to_index(activity) do + object = activity.object + + if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" do + data = object.data + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!([%{id: object.id, source: data["source"], ap: data["id"]}]) + ) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + Logger.error("Failed to add activity #{activity.id} to index: #{result.body}") + end + end + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a5d7036d9..034c3b185 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -140,6 +140,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + {:ok, activity} else %Activity{} = activity -> diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 5e6e04734..99c33eba6 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do use Pleroma.Web, :controller - alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ControllerHelper @@ -100,7 +99,9 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end) + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + statuses = with_fallback(fn -> search_module.search(options[:for_user], query, options) end) StatusView.render("index.json", activities: statuses, -- cgit v1.2.3 From 0318e9a59945d7a5625111157867f0f9ebaffd91 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 10:18:01 +0300 Subject: Add logging to milisiearch index and make it use desc(id) --- lib/mix/tasks/pleroma/search/meilisearch.ex | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2af8e5853..1fece96e5 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -3,8 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - import Mix.Pleroma + require Logger + import Mix.Pleroma import Ecto.Query def run(["index"]) do @@ -12,12 +13,25 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/settings/ranking-rules", + Jason.encode!([ + "desc(id)", + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ]) + ) + Pleroma.Repo.chunk_stream( from(Pleroma.Object, - limit: 200, where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") ), - 100, + 200, :batches ) |> Stream.map(fn objects -> @@ -26,12 +40,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do %{id: object.id, source: data["source"], ap: data["id"]} end) end) - |> Stream.each(fn activities -> + |> Stream.each(fn objects -> {:ok, _} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!(activities) + Jason.encode!(objects) ) + + IO.puts("Indexed #{Enum.count(objects)} entries") end) |> Stream.run() end -- cgit v1.2.3 From 365024abec905e427babb5403f0fccbde65f4bcd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 22:24:31 +0300 Subject: Ensure only indexing public posts and implement clearing and delete --- lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++++- lib/pleroma/activity.ex | 1 + lib/pleroma/search/meilisearch.ex | 17 ++++++++++++++++- lib/pleroma/web/common_api.ex | 7 +++++++ 4 files changed, 38 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 1fece96e5..0b86fdece 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do require Logger + require Pleroma.Constants import Mix.Pleroma import Ecto.Query @@ -29,7 +30,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.Repo.chunk_stream( from(Pleroma.Object, - where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") + # Only index public posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + fragment("LENGTH(data->>'source') > 0") and + fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) ), 200, :batches @@ -51,4 +56,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end) |> Stream.run() end + + def run(["clear"]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + end end diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 9563136f9..2c168fd41 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -368,6 +368,7 @@ defmodule Pleroma.Activity do defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 92e0d3429..dbe6b2d67 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -1,5 +1,6 @@ defmodule Pleroma.Search.Meilisearch do require Logger + require Pleroma.Constants alias Pleroma.Activity @@ -41,7 +42,8 @@ defmodule Pleroma.Search.Meilisearch do def add_to_index(activity) do object = activity.object - if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" do + if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" and + Pleroma.Constants.as_public() in object.data["to"] do data = object.data endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) @@ -57,4 +59,17 @@ defmodule Pleroma.Search.Meilisearch do end end end + + def remove_from_index(object) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = + Pleroma.HTTP.request( + :delete, + "#{endpoint}/indexes/objects/documents/#{object.id}", + "", + [], + [] + ) + end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 89f5dd606..54a8aa213 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -147,6 +147,13 @@ defmodule Pleroma.Web.CommonAPI do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + {:ok, delete} else {:find_activity, _} -> -- cgit v1.2.3 From ea6a6a128712e81c4f298b2bb2cedfadf2295cff Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 22:30:56 +0300 Subject: Make the indexing batch differently and more, show number indexed --- lib/mix/tasks/pleroma/search/meilisearch.ex | 63 +++++++++++++++++------------ 1 file changed, 38 insertions(+), 25 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 0b86fdece..2a6438528 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,33 +28,46 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) - Pleroma.Repo.chunk_stream( - from(Pleroma.Object, - # Only index public posts which are notes and have some text - where: - fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'source') > 0") and - fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) - ), - 200, - :batches - ) - |> Stream.map(fn objects -> - Enum.map(objects, fn object -> - data = object.data - %{id: object.id, source: data["source"], ap: data["id"]} - end) - end) - |> Stream.each(fn objects -> - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!(objects) + chunk_size = 100_000 + + Pleroma.Repo.transaction( + fn -> + Pleroma.Repo.stream( + from(Pleroma.Object, + # Only index public posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + fragment("LENGTH(data->>'source') > 0") and + fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), + order_by: fragment("data->'published' DESC") + ), + timeout: :infinity ) + |> Stream.chunk_every(chunk_size) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) - IO.puts("Indexed #{Enum.count(objects)} entries") - end) - |> Stream.run() + IO.puts("Indexed #{new_acc} entries") + + {[objects], new_acc} + end) + |> Stream.map(fn objects -> + Enum.map(objects, fn object -> + data = object.data + %{id: object.id, source: data["source"], ap: data["id"]} + end) + end) + |> Stream.each(fn objects -> + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!(objects) + ) + end) + |> Stream.run() + end, + timeout: :infinity + ) end def run(["clear"]) do -- cgit v1.2.3 From 38996f551a4ec014e9f4cb4a691d31beecab43ba Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:06:32 +0300 Subject: Make meilisearch sort on publish date converted to unix time --- lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2a6438528..2dd9c0a62 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -18,7 +18,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.HTTP.post( "#{endpoint}/indexes/objects/settings/ranking-rules", Jason.encode!([ - "desc(id)", + "desc(published)", "typo", "words", "proximity", @@ -54,7 +54,15 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do |> Stream.map(fn objects -> Enum.map(objects, fn object -> data = object.data - %{id: object.id, source: data["source"], ap: data["id"]} + + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + source: data["source"], + ap: data["id"], + published: published |> DateTime.to_unix() + } end) end) |> Stream.each(fn objects -> -- cgit v1.2.3 From 9beaebd97e1746df010aecfcc01d9e2e9a4c60ac Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:30:14 +0300 Subject: Tweak search ordering to hopefully return newer results --- lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++--- lib/pleroma/search/meilisearch.ex | 13 +++++++++++-- 2 files changed, 23 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2dd9c0a62..dcecbd7cf 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -39,7 +39,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), - order_by: fragment("data->'published' DESC") + order_by: [desc: fragment("data->'published'")] ), timeout: :infinity ) @@ -66,11 +66,15 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end) end) |> Stream.each(fn objects -> - {:ok, _} = + {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", Jason.encode!(objects) ) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + IO.puts("Failed to index: #{result}") + end end) |> Stream.run() end, @@ -83,6 +87,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + {:ok, result} = + Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + IO.puts("Failed to clear: #{result}") + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index dbe6b2d67..9fdb0a07f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Search.Meilisearch do |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) |> maybe_fetch(user, query) - |> order_by([activity], desc: activity.id) + |> order_by([object: obj], desc: obj.data["published"]) |> Pleroma.Repo.all() rescue _ -> maybe_fetch([], user, query) @@ -48,10 +48,19 @@ defmodule Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!([%{id: object.id, source: data["source"], ap: data["id"]}]) + Jason.encode!([ + %{ + id: object.id, + source: data["source"], + ap: data["id"], + published: published |> DateTime.to_unix() + } + ]) ) if not Map.has_key?(Jason.decode!(result.body), "updateId") do -- cgit v1.2.3 From 00c48a33acf0bd59fa7e7b58a67b049e4f4adc31 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:57:53 +0300 Subject: Use content instead of source and scrub it --- lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++-------- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index dcecbd7cf..5270de255 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -37,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do # Only index public posts which are notes and have some text where: fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'source') > 0") and + fragment("LENGTH(data->>'content') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), order_by: [desc: fragment("data->'published'")] ), @@ -56,10 +56,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do data = object.data {:ok, published, _} = DateTime.from_iso8601(data["published"]) + {:ok, content} = FastSanitize.strip_tags(data["content"]) %{ id: object.id, - source: data["source"], + content: content, ap: data["id"], published: published |> DateTime.to_unix() } @@ -87,11 +88,6 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = - Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) - - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - IO.puts("Failed to clear: #{result}") - end + {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], []) end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 9fdb0a07f..87fdeaf5e 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Search.Meilisearch do Jason.encode!([ %{ id: object.id, - source: data["source"], + content: data["content"] |> Pleroma.HTML.filter_tags(), ap: data["id"], published: published |> DateTime.to_unix() } -- cgit v1.2.3 From e35d87ea54f70a39206f6103ef0e7334e2a428cc Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 01:37:43 +0300 Subject: Make the chunk size smaller --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 5270de255..44af25f3e 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) - chunk_size = 100_000 + chunk_size = 10_000 Pleroma.Repo.transaction( fn -> -- cgit v1.2.3 From 2b2e409ad72862967cabf06344874ae9bff9860f Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 16:37:52 +0300 Subject: Also index incoming federated posts --- lib/pleroma/search/search.ex | 18 ++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 7 ++----- lib/pleroma/web/activity_pub/side_effects.ex | 7 +++++++ lib/pleroma/web/common_api.ex | 8 ++------ 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/search/search.ex (limited to 'lib') diff --git a/lib/pleroma/search/search.ex b/lib/pleroma/search/search.ex new file mode 100644 index 000000000..e363abf19 --- /dev/null +++ b/lib/pleroma/search/search.ex @@ -0,0 +1,18 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 034c3b185..7178cf9eb 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -140,11 +140,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + # Add local posts to search index + Pleroma.Search.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 5eefd2824..15e006b18 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -197,6 +197,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Increase replies count # - Set up ActivityExpiration # - Set up notifications + # - Index incoming posts for search (if needed) @impl true def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), @@ -226,6 +227,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + meta = meta |> add_notifications(notifications) @@ -286,6 +289,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Reduce the user note count # - Reduce the reply count # - Stream out the activity + # - Removes posts from search index (if needed) @impl true def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do deleted_object = @@ -325,6 +329,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) + + Pleroma.Search.remove_from_index(object) + {:ok, object, meta} else {:error, result} diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 54a8aa213..ba6c07975 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -147,12 +147,8 @@ defmodule Pleroma.Web.CommonAPI do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + # Remove from search index for local posts + Pleroma.Search.remove_from_index(object) {:ok, delete} else -- cgit v1.2.3 From 9f16ca80e0fe60b8b0e3e8ddb9b06ca0bec31002 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 18:47:41 +0300 Subject: Mark only content as searchable for meilisearch --- lib/mix/tasks/pleroma/search/meilisearch.ex | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 44af25f3e..ebd3cc81f 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,6 +28,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/settings/searchable-attributes", + Jason.encode!([ + "content" + ]) + ) + chunk_size = 10_000 Pleroma.Repo.transaction( @@ -55,8 +63,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Enum.map(objects, fn object -> data = object.data + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + {:ok, published, _} = DateTime.from_iso8601(data["published"]) - {:ok, content} = FastSanitize.strip_tags(data["content"]) + {:ok, content} = FastSanitize.strip_tags(content_str) %{ id: object.id, -- cgit v1.2.3 From 3dedadf192a3acd0c1dfc2b11eba5a247ae7f61c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 19:38:03 +0300 Subject: Adjust content indexing to skip more unneeded stuff --- lib/mix/tasks/pleroma/search/meilisearch.ex | 47 +++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index ebd3cc81f..3704e0bdc 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -52,13 +52,6 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do timeout: :infinity ) |> Stream.chunk_every(chunk_size) - |> Stream.transform(0, fn objects, acc -> - new_acc = acc + Enum.count(objects) - - IO.puts("Indexed #{new_acc} entries") - - {[objects], new_acc} - end) |> Stream.map(fn objects -> Enum.map(objects, fn object -> data = object.data @@ -70,15 +63,34 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end {:ok, published, _} = DateTime.from_iso8601(data["published"]) - {:ok, content} = FastSanitize.strip_tags(content_str) - - %{ - id: object.id, - content: content, - ap: data["id"], - published: published |> DateTime.to_unix() - } + + content = + with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + trimmed <- String.trim(scrubbed) do + trimmed + end + + # Only index if there is anything in the string. If there is a single symbol, + # it's probably a dot from mastodon posts with just the picture + if String.length(content) > 1 do + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + else + nil + end end) + |> Enum.filter(fn o -> not is_nil(o) end) + end) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) + + IO.puts("Indexed #{new_acc} entries") + + {[objects], new_acc} end) |> Stream.each(fn objects -> {:ok, result} = @@ -102,6 +114,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], []) + {:ok, _} = + Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], + timeout: :infinity + ) end end -- cgit v1.2.3 From 35e9192cedcbc56fb07c9933e2988bf900256b53 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 22:53:18 +0300 Subject: Rework task indexing to share code with the main module The code in the main module now scrubs new posts too --- lib/mix/tasks/pleroma/search/meilisearch.ex | 35 ++-------------------- lib/pleroma/search/meilisearch.ex | 46 ++++++++++++++++++++--------- 2 files changed, 34 insertions(+), 47 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 3704e0bdc..b5a394e34 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -51,40 +51,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ), timeout: :infinity ) + |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) + |> Stream.filter(fn o -> not is_nil(o) end) |> Stream.chunk_every(chunk_size) - |> Stream.map(fn objects -> - Enum.map(objects, fn object -> - data = object.data - - content_str = - case data["content"] do - [nil | rest] -> to_string(rest) - str -> str - end - - {:ok, published, _} = DateTime.from_iso8601(data["published"]) - - content = - with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), - trimmed <- String.trim(scrubbed) do - trimmed - end - - # Only index if there is anything in the string. If there is a single symbol, - # it's probably a dot from mastodon posts with just the picture - if String.length(content) > 1 do - %{ - id: object.id, - content: content, - ap: data["id"], - published: published |> DateTime.to_unix() - } - else - nil - end - end) - |> Enum.filter(fn o -> not is_nil(o) end) - end) |> Stream.transform(0, fn objects, acc -> new_acc = acc + Enum.count(objects) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 87fdeaf5e..10468e36c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -39,28 +39,46 @@ defmodule Pleroma.Search.Meilisearch do end end - def add_to_index(activity) do - object = activity.object - - if activity.data["type"] == "Create" and not is_nil(object) and object.data["type"] == "Note" and + def object_to_search_data(object) do + if not is_nil(object) and object.data["type"] == "Note" and Pleroma.Constants.as_public() in object.data["to"] do data = object.data - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + + content = + with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + trimmed <- String.trim(scrubbed) do + trimmed + end + + if String.length(content) > 1 do + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + end + end + end - {:ok, published, _} = DateTime.from_iso8601(data["published"]) + def add_to_index(activity) do + maybe_search_data = object_to_search_data(activity) + + if activity.data["type"] == "Create" and maybe_search_data do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!([ - %{ - id: object.id, - content: data["content"] |> Pleroma.HTML.filter_tags(), - ap: data["id"], - published: published |> DateTime.to_unix() - } - ]) + Jason.encode!([maybe_search_data]) ) if not Map.has_key?(Jason.decode!(result.body), "updateId") do -- cgit v1.2.3 From 410c8cb765bbec1014cb2bbdbcc44d3a25f834e1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 23:47:43 +0300 Subject: Make indexing logs rewrite themselves --- lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index b5a394e34..2485a441d 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -57,7 +57,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do |> Stream.transform(0, fn objects, acc -> new_acc = acc + Enum.count(objects) - IO.puts("Indexed #{new_acc} entries") + # Reset to the beginning of the line and rewrite it + IO.write("\r") + IO.write("Indexed #{new_acc} entries") {[objects], new_acc} end) @@ -76,6 +78,8 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end, timeout: :infinity ) + + IO.write("\n") end def run(["clear"]) do -- cgit v1.2.3 From 2c7d973af7797ae860829c1764ade521a17e7263 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 19:35:21 +0300 Subject: Implement meilisearch auth --- lib/mix/tasks/pleroma/search/meilisearch.ex | 75 +++++++++++++++++------------ lib/pleroma/search/meilisearch.ex | 69 ++++++++++++++++---------- 2 files changed, 88 insertions(+), 56 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2485a441d..230be5aa1 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,32 +9,30 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Mix.Pleroma import Ecto.Query + import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1] + def run(["index"]) do start_pleroma() - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/settings/ranking-rules", - Jason.encode!([ - "desc(published)", - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ]) - ) + meili_post!( + "/indexes/objects/settings/ranking-rules", + [ + "desc(published)", + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ] + ) - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/settings/searchable-attributes", - Jason.encode!([ - "content" - ]) - ) + meili_post!( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) chunk_size = 10_000 @@ -64,14 +62,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> - {:ok, result} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!(objects) + result = + meili_post!( + "/indexes/objects/documents", + objects ) - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - IO.puts("Failed to index: #{result}") + if not Map.has_key?(result, "updateId") do + IO.puts("Failed to index: #{inspect(result)}") end end) |> Stream.run() @@ -85,11 +83,26 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["clear"]) do start_pleroma() + meili_delete!("/indexes/objects/documents") + end + + def run(["show-private-key", master_key]) do + start_pleroma() + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = - Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], - timeout: :infinity + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, "/keys"), + [{"X-Meili-API-Key", master_key}] ) + + decoded = Jason.decode!(result.body) + + if decoded["private"] do + IO.puts(decoded["private"]) + else + IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 10468e36c..8745d539d 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,20 +7,50 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Activity.Search import Ecto.Query - def search(user, query, options \\ []) do - limit = Enum.min([Keyword.get(options, :limit), 40]) - offset = Keyword.get(options, :offset, 0) - author = Keyword.get(options, :author) + defp meili_headers() do + private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) + if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + end + + def meili_post!(path, params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) {:ok, result} = Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/search", - Jason.encode!(%{q: query, offset: offset, limit: limit}) + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers() + ) + + Jason.decode!(result.body) + end + + def meili_delete!(path) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + timeout: :infinity + ) + end + + def search(user, query, options \\ []) do + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + result = + meili_post!( + "/indexes/objects/search", + %{q: query, offset: offset, limit: limit} ) - hits = Jason.decode!(result.body)["hits"] |> Enum.map(& &1["ap"]) + hits = result["hits"] |> Enum.map(& &1["ap"]) try do hits @@ -73,30 +103,19 @@ defmodule Pleroma.Search.Meilisearch do maybe_search_data = object_to_search_data(activity) if activity.data["type"] == "Create" and maybe_search_data do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, result} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!([maybe_search_data]) + result = + meili_post!( + "/indexes/objects/documents", + [maybe_search_data] ) - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - Logger.error("Failed to add activity #{activity.id} to index: #{result.body}") + if not Map.has_key?(result, "updateId") do + Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") end end end def remove_from_index(object) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, _} = - Pleroma.HTTP.request( - :delete, - "#{endpoint}/indexes/objects/documents/#{object.id}", - "", - [], - [] - ) + meili_delete!("/indexes/objects/documents/#{object.id}") end end -- cgit v1.2.3 From a67f9da5cc46b4e184aa1afe3dd1bd1df31de15b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:02:34 +0300 Subject: Add a message with a count of posts to index --- lib/mix/tasks/pleroma/search/meilisearch.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 230be5aa1..557b06182 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -38,7 +38,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.Repo.transaction( fn -> - Pleroma.Repo.stream( + query = from(Pleroma.Object, # Only index public posts which are notes and have some text where: @@ -46,7 +46,13 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fragment("LENGTH(data->>'content') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), order_by: [desc: fragment("data->'published'")] - ), + ) + + count = query |> Pleroma.Repo.aggregate(:count, :data) + IO.puts("Entries to index: #{count}") + + Pleroma.Repo.stream( + query, timeout: :infinity ) |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) -- cgit v1.2.3 From 09a1ae1b6eca4efbb935aa1c0da950009d110fb2 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:21:46 +0300 Subject: Add the meilisearch.stats command --- lib/mix/tasks/pleroma/search/meilisearch.ex | 10 +++++++++- lib/pleroma/search/meilisearch.ex | 12 ++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 557b06182..f2d9fe312 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Mix.Pleroma import Ecto.Query - import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1] + import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] def run(["index"]) do start_pleroma() @@ -111,4 +111,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") end end + + def run(["stats"]) do + start_pleroma() + + result = meili_get!("/indexes/objects/stats") + IO.puts("Number of entries: #{result["numberOfDocuments"]}") + IO.puts("Indexing? #{result["isIndexing"]}") + end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 8745d539d..1ad17bf9f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -13,6 +13,18 @@ defmodule Pleroma.Search.Meilisearch do if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end + def meili_get!(path) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, path), + meili_headers() + ) + + Jason.decode!(result.body) + end + def meili_post!(path, params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) -- cgit v1.2.3 From d9ef7e075880ba39dd4ca8e21566c680070faa42 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 21:15:15 +0300 Subject: Fix activity being passed to objec_to_search_data --- lib/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 1ad17bf9f..212bdd473 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -112,7 +112,7 @@ defmodule Pleroma.Search.Meilisearch do end def add_to_index(activity) do - maybe_search_data = object_to_search_data(activity) + maybe_search_data = object_to_search_data(activity.object) if activity.data["type"] == "Create" and maybe_search_data do result = -- cgit v1.2.3 From 40280cc273ad7f2b355846e2f41b9873a8d5ff2c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 28 Aug 2021 15:59:13 +0300 Subject: Reorder ranking rules for (maybe) better results --- lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index f2d9fe312..cdf9ab0bd 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -18,12 +18,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do "/indexes/objects/settings/ranking-rules", [ "desc(published)", - "typo", "words", + "exactness", "proximity", - "attribute", "wordsPosition", - "exactness" + "typo", + "attribute" ] ) -- cgit v1.2.3 From 6beef2d1179ab9a377e87872b7fbe2997bbbbebd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 8 Oct 2021 12:24:37 +0300 Subject: Move add_to_index / remove_from_index to Pleroma.Actitivy.Search --- lib/pleroma/activity.ex | 2 -- lib/pleroma/activity/search.ex | 3 +++ 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 2c168fd41..ebfd4ed45 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -367,8 +367,6 @@ defmodule Pleroma.Activity do end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search - def add_to_index(_activity), do: nil - def remove_from_index(_object), do: nil def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index 3dce9d355..47ab5208c 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -45,6 +45,9 @@ defmodule Pleroma.Activity.Search do end end + def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end -- cgit v1.2.3 From 95cb2bb694e3f8857895b21331b02b9277d65d9b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 12 Oct 2021 19:17:37 +0300 Subject: Don't try removing from index again in common_api It's already removed in the side effects of the pipeline --- lib/pleroma/web/common_api.ex | 3 --- 1 file changed, 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index ba6c07975..89f5dd606 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -147,9 +147,6 @@ defmodule Pleroma.Web.CommonAPI do true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, delete_data, _} <- Builder.delete(user, object.data["id"]), {:ok, delete, _} <- Pipeline.common_pipeline(delete_data, local: true) do - # Remove from search index for local posts - Pleroma.Search.remove_from_index(object) - {:ok, delete} else {:find_activity, _} -> -- cgit v1.2.3 From cf558208c202d5188954e26077d35bcc1ae02fce Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 12 Oct 2021 19:34:57 +0300 Subject: Use proper deleted object for removing from index --- lib/pleroma/web/activity_pub/side_effects.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 15e006b18..4762b5ac6 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(object) + Pleroma.Search.remove_from_index(deleted_object) {:ok, object, meta} else -- cgit v1.2.3 From e4b7a3f51f270f468c15cc4ce850c847633c030b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 00:38:00 +0300 Subject: Modify some meilisearch variables --- lib/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 212bdd473..b8248e40c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -47,7 +47,7 @@ defmodule Pleroma.Search.Meilisearch do Path.join(endpoint, path), "", meili_headers(), - timeout: :infinity + [] ) end -- cgit v1.2.3 From 0b4fd0d342e3ced073e82355b380cbfee5478c60 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 13:58:24 +0300 Subject: Set content-type to application/json --- lib/pleroma/search/meilisearch.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index b8248e40c..d94ab8b64 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -10,7 +10,8 @@ defmodule Pleroma.Search.Meilisearch do defp meili_headers() do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) - if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + [{"Content-Type", "application/json"}] ++ + if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end def meili_get!(path) do -- cgit v1.2.3 From 4445421297f4a4375ce9df4857a66ad08e984507 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 29 Oct 2021 21:04:59 +0300 Subject: Only add local posts to index in activity_pub Remote ones are already added in another place --- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7178cf9eb..cdc70aacf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) # Add local posts to search index - Pleroma.Search.add_to_index(activity) + if local, do: Pleroma.Search.add_to_index(activity) {:ok, activity} else -- cgit v1.2.3 From e928e307f34542b0a0af8b615c986aeac478b637 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 10 Nov 2021 21:25:12 +0300 Subject: Add a reindex option Signed-off-by: Ekaterina Vaartis --- lib/mix/tasks/pleroma/search/meilisearch.ex | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index cdf9ab0bd..2a3c3a8b9 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -11,9 +11,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] - def run(["index"]) do + def run(["index" | args]) do start_pleroma() + is_reindex = "--reindex" in args + meili_post!( "/indexes/objects/settings/ranking-rules", [ @@ -68,6 +70,19 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> + objects = + objects + |> Enum.filter(fn o -> + if is_reindex do + result = meili_get!("/indexes/objects/documents/#{o.id}") + + # Filter out the already indexed documents. This is true when the document does not exist + result["errorCode"] == "document_not_found" + else + true + end + end) + result = meili_post!( "/indexes/objects/documents", -- cgit v1.2.3 From 9c1a9307079c8d007ae7cbf3e089d2bc5ea6b733 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 13 Nov 2021 15:07:51 +0300 Subject: Support reindexing meilisearch >=0.24.0 It has has a different error code key --- lib/mix/tasks/pleroma/search/meilisearch.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2a3c3a8b9..3b134ad3f 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -76,8 +76,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do if is_reindex do result = meili_get!("/indexes/objects/documents/#{o.id}") + # With >= 0.24.0 the name for "errorCode" is just "code" + error_code_key = + if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"), + do: "code", + else: "errorCode" + # Filter out the already indexed documents. This is true when the document does not exist - result["errorCode"] == "document_not_found" + result[error_code_key] == "document_not_found" else true end -- cgit v1.2.3 From 7009ef5672ad20f92374d218cd614a38cd70515e Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 14 Nov 2021 20:24:05 +0300 Subject: Move the search.ex file so credo doesn't complain --- lib/pleroma/search.ex | 18 ++++++++++++++++++ lib/pleroma/search/search.ex | 18 ------------------ 2 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 lib/pleroma/search.ex delete mode 100644 lib/pleroma/search/search.ex (limited to 'lib') diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex new file mode 100644 index 000000000..e363abf19 --- /dev/null +++ b/lib/pleroma/search.ex @@ -0,0 +1,18 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end +end diff --git a/lib/pleroma/search/search.ex b/lib/pleroma/search/search.ex deleted file mode 100644 index e363abf19..000000000 --- a/lib/pleroma/search/search.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule Pleroma.Search do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) - end - - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) - end -end -- cgit v1.2.3 From 39e596a5b51c0c86b6d6bd5f23177a1e6a64cf0b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 14 Nov 2021 21:42:18 +0300 Subject: Style fixes --- lib/mix/tasks/pleroma/search/meilisearch.ex | 3 ++- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 3b134ad3f..62ace7e39 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -82,7 +82,8 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do do: "code", else: "errorCode" - # Filter out the already indexed documents. This is true when the document does not exist + # Filter out the already indexed documents. + # This is true when the document does not exist result[error_code_key] == "document_not_found" else true diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index d94ab8b64..41f99ad9f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Activity.Search import Ecto.Query - defp meili_headers() do + defp meili_headers do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) [{"Content-Type", "application/json"}] ++ -- cgit v1.2.3 From 0fae71f88d142f64ec18a49ff4292db816dacdc8 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 16 Nov 2021 21:54:26 +0300 Subject: Rename search.ex to database_search.ex and add search/2 --- lib/pleroma/search.ex | 18 ---------------- lib/pleroma/search/database_search.ex | 24 ++++++++++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/side_effects.ex | 4 ++-- .../mastodon_api/controllers/search_controller.ex | 4 +--- 5 files changed, 28 insertions(+), 24 deletions(-) delete mode 100644 lib/pleroma/search.ex create mode 100644 lib/pleroma/search/database_search.ex (limited to 'lib') diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex deleted file mode 100644 index e363abf19..000000000 --- a/lib/pleroma/search.ex +++ /dev/null @@ -1,18 +0,0 @@ -defmodule Pleroma.Search do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) - end - - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) - end -end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex new file mode 100644 index 000000000..be0e19be0 --- /dev/null +++ b/lib/pleroma/search/database_search.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Search.DatabaseSearch do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end +end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index cdc70aacf..7e3444676 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) # Add local posts to search index - if local, do: Pleroma.Search.add_to_index(activity) + if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 4762b5ac6..fa57eab69 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -227,7 +227,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object)) meta = meta @@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(deleted_object) + Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object) {:ok, object, meta} else diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 99c33eba6..10f1aa532 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -99,9 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - - statuses = with_fallback(fn -> search_module.search(options[:for_user], query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end) StatusView.render("index.json", activities: statuses, -- cgit v1.2.3 From a6946048fbe049aa223d094d36eb767739ab5ff2 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 17 Nov 2021 22:29:49 +0300 Subject: Rename Activity.Search to Search.DatabaseSearch --- lib/pleroma/activity.ex | 2 +- lib/pleroma/activity/search.ex | 165 --------------------- lib/pleroma/search.ex | 24 +++ lib/pleroma/search/database_search.ex | 157 ++++++++++++++++++-- lib/pleroma/search/meilisearch.ex | 2 +- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- lib/pleroma/web/activity_pub/side_effects.ex | 4 +- .../mastodon_api/controllers/search_controller.ex | 2 +- 8 files changed, 173 insertions(+), 185 deletions(-) delete mode 100644 lib/pleroma/activity/search.ex create mode 100644 lib/pleroma/search.ex (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index ebfd4ed45..389c80691 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -366,7 +366,7 @@ defmodule Pleroma.Activity do from(activity in query, where: activity.actor not in subquery(deactivated_users_query)) end - defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex deleted file mode 100644 index 47ab5208c..000000000 --- a/lib/pleroma/activity/search.ex +++ /dev/null @@ -1,165 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Activity.Search do - alias Pleroma.Activity - alias Pleroma.Object.Fetcher - alias Pleroma.Pagination - alias Pleroma.User - alias Pleroma.Web.ActivityPub.Visibility - - require Pleroma.Constants - - import Ecto.Query - - def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin - limit = Enum.min([Keyword.get(options, :limit), 40]) - offset = Keyword.get(options, :offset, 0) - author = Keyword.get(options, :author) - - search_function = - if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do - :websearch - else - :plain - end - - try do - Activity - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> restrict_public(user) - |> query_with(index_type, search_query, search_function) - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> Pagination.fetch_paginated( - %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, - :offset - ) - |> maybe_fetch(user, search_query) - rescue - _ -> maybe_fetch([], user, search_query) - end - end - - def add_to_index(_activity), do: nil - def remove_from_index(_object), do: nil - - def maybe_restrict_author(query, %User{} = author) do - Activity.Queries.by_author(query, author) - end - - def maybe_restrict_author(query, _), do: query - - def maybe_restrict_blocked(query, %User{} = user) do - Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) - end - - def maybe_restrict_blocked(query, _), do: query - - defp restrict_public(q, user) when not is_nil(user) do - intended_recipients = [ - Pleroma.Constants.as_public(), - Pleroma.Web.ActivityPub.Utils.as_local_public() - ] - - from([a, o] in q, - where: fragment("?->>'type' = 'Create'", a.data), - where: fragment("? && ?", ^intended_recipients, a.recipients) - ) - end - - defp restrict_public(q, _user) do - from([a, o] in q, - where: fragment("?->>'type' = 'Create'", a.data), - where: ^Pleroma.Constants.as_public() in a.recipients - ) - end - - defp query_with(q, :gin, search_query, :plain) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :gin, search_query, :websearch) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :rum, search_query, :plain) do - from([a, o] in q, - where: - fragment( - "? @@ plainto_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - defp query_with(q, :rum, search_query, :websearch) do - from([a, o] in q, - where: - fragment( - "? @@ websearch_to_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - def maybe_restrict_local(q, user) do - limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) - - case {limit, user} do - {:all, _} -> restrict_local(q) - {:unauthenticated, %User{}} -> q - {:unauthenticated, _} -> restrict_local(q) - {false, _} -> q - end - end - - defp restrict_local(q), do: where(q, local: true) - - def maybe_fetch(activities, user, search_query) do - with true <- Regex.match?(~r/https?:/, search_query), - {:ok, object} <- Fetcher.fetch_object_from_id(search_query), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), - true <- Visibility.visible_for_user?(activity, user) do - [activity | activities] - else - _ -> activities - end - end -end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex new file mode 100644 index 000000000..ae0b28c54 --- /dev/null +++ b/lib/pleroma/search.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Search do + def add_to_index(activity) do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.add_to_index(activity) end) + end) + end + + def remove_from_index(object) do + # Also delete from search index + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + ConcurrentLimiter.limit(Pleroma.Search, fn -> + Task.start(fn -> search_module.remove_from_index(object) end) + end) + end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end +end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index be0e19be0..5a8b8ca67 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -1,24 +1,153 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Search.DatabaseSearch do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + alias Pleroma.Activity + alias Pleroma.Object.Fetcher + alias Pleroma.Pagination + alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility + + require Pleroma.Constants + + import Ecto.Query + + def search(user, search_query, options \\ []) do + index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + search_function = + if :persistent_term.get({Pleroma.Repo, :postgres_version}) >= 11 do + :websearch + else + :plain + end + + try do + Activity + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> restrict_public() + |> query_with(index_type, search_query, search_function) + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> Pagination.fetch_paginated( + %{"offset" => offset, "limit" => limit, "skip_order" => index_type == :rum}, + :offset + ) + |> maybe_fetch(user, search_query) + rescue + _ -> maybe_fetch([], user, search_query) + end + end + + def add_to_index(_activity), do: nil + def remove_from_index(_object), do: nil + + def maybe_restrict_author(query, %User{} = author) do + Activity.Queries.by_author(query, author) + end + + def maybe_restrict_author(query, _), do: query + + def maybe_restrict_blocked(query, %User{} = user) do + Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) + end + + def maybe_restrict_blocked(query, _), do: query + + def restrict_public(q) do + from([a, o] in q, + where: fragment("?->>'type' = 'Create'", a.data), + where: ^Pleroma.Constants.as_public() in a.recipients + ) + end + + defp query_with(q, :gin, search_query, :plain) do + %{rows: [[tsc]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select current_setting('default_text_search_config')::regconfig::oid;" + ) + + from([a, o] in q, + where: + fragment( + "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", + ^tsc, + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :gin, search_query, :websearch) do + %{rows: [[tsc]]} = + Ecto.Adapters.SQL.query!( + Pleroma.Repo, + "select current_setting('default_text_search_config')::regconfig::oid;" + ) + + from([a, o] in q, + where: + fragment( + "to_tsvector(?::oid::regconfig, ?->>'content') @@ websearch_to_tsquery(?)", + ^tsc, + o.data, + ^search_query + ) + ) + end + + defp query_with(q, :rum, search_query, :plain) do + from([a, o] in q, + where: + fragment( + "? @@ plainto_tsquery(?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + defp query_with(q, :rum, search_query, :websearch) do + from([a, o] in q, + where: + fragment( + "? @@ websearch_to_tsquery(?)", + o.fts_content, + ^search_query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) end - def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + def maybe_restrict_local(q, user) do + limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + case {limit, user} do + {:all, _} -> restrict_local(q) + {:unauthenticated, %User{}} -> q + {:unauthenticated, _} -> restrict_local(q) + {false, _} -> q + end end - def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + defp restrict_local(q), do: where(q, local: true) - search_module.search(options[:for_user], query, options) + def maybe_fetch(activities, user, search_query) do + with true <- Regex.match?(~r/https?:/, search_query), + {:ok, object} <- Fetcher.fetch_object_from_id(search_query), + %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), + true <- Visibility.visible_for_user?(activity, user) do + [activity | activities] + else + _ -> activities + end end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 41f99ad9f..fa9e27b03 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Search.Meilisearch do alias Pleroma.Activity - import Pleroma.Activity.Search + import Pleroma.Search.DatabaseSearch import Ecto.Query defp meili_headers do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7e3444676..cdc70aacf 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) # Add local posts to search index - if local, do: Pleroma.Search.DatabaseSearch.add_to_index(activity) + if local, do: Pleroma.Search.add_to_index(activity) {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index fa57eab69..4762b5ac6 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -227,7 +227,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) - Pleroma.Search.DatabaseSearch.add_to_index(Map.put(activity, :object, object)) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) meta = meta @@ -330,7 +330,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.DatabaseSearch.remove_from_index(deleted_object) + Pleroma.Search.remove_from_index(deleted_object) {:ok, object, meta} else diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 10f1aa532..e4acba226 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -99,7 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Pleroma.Search.DatabaseSearch.search(query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end) StatusView.render("index.json", activities: statuses, -- cgit v1.2.3 From a12f63bc81481e3f852934e8cc1269e16a57cf0a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 22 Nov 2021 21:39:54 +0300 Subject: Implement suggestions from the Meilisearch MR - Index unlisted posts - Move version check outside of the streaming and only do it once - Use a PUT request instead of checking manually if there is need to insert - Add error handling, sort of --- lib/mix/tasks/pleroma/search/meilisearch.ex | 86 ++++++++++++---------------- lib/pleroma/search/meilisearch.ex | 87 +++++++++++++++++++---------- 2 files changed, 95 insertions(+), 78 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 62ace7e39..6730a99a9 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -3,38 +3,40 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - require Logger require Pleroma.Constants import Mix.Pleroma import Ecto.Query - import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] + import Pleroma.Search.Meilisearch, + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] - def run(["index" | args]) do + def run(["index"]) do start_pleroma() - is_reindex = "--reindex" in args - - meili_post!( - "/indexes/objects/settings/ranking-rules", - [ - "desc(published)", - "words", - "exactness", - "proximity", - "wordsPosition", - "typo", - "attribute" - ] - ) + {:ok, _} = + meili_post( + "/indexes/objects/settings/ranking-rules", + [ + "desc(published)", + "words", + "exactness", + "proximity", + "wordsPosition", + "typo", + "attribute" + ] + ) - meili_post!( - "/indexes/objects/settings/searchable-attributes", - [ - "content" - ] - ) + {:ok, _} = + meili_post( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) + + IO.puts("Created indices. Starting to insert posts.") chunk_size = 10_000 @@ -42,11 +44,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fn -> query = from(Pleroma.Object, - # Only index public posts which are notes and have some text + # Only index public and unlisted posts which are notes and have some text where: fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'content') > 0") and - fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), + (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or + fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())), order_by: [desc: fragment("data->'published'")] ) @@ -70,34 +72,18 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> - objects = - objects - |> Enum.filter(fn o -> - if is_reindex do - result = meili_get!("/indexes/objects/documents/#{o.id}") - - # With >= 0.24.0 the name for "errorCode" is just "code" - error_code_key = - if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"), - do: "code", - else: "errorCode" - - # Filter out the already indexed documents. - # This is true when the document does not exist - result[error_code_key] == "document_not_found" - else - true - end - end) - result = - meili_post!( + meili_put( "/indexes/objects/documents", objects ) - if not Map.has_key?(result, "updateId") do - IO.puts("Failed to index: #{inspect(result)}") + with {:ok, res} <- result do + if not Map.has_key?(res, "updateId") do + IO.puts("\nFailed to index: #{inspect(result)}") + end + else + e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}") end end) |> Stream.run() @@ -137,7 +123,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["stats"]) do start_pleroma() - result = meili_get!("/indexes/objects/stats") + {:ok, result} = meili_get("/indexes/objects/stats") IO.puts("Number of entries: #{result["numberOfDocuments"]}") IO.puts("Indexing? #{result["isIndexing"]}") end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index fa9e27b03..21b44de86 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -14,29 +14,50 @@ defmodule Pleroma.Search.Meilisearch do if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] end - def meili_get!(path) do + def meili_get(path) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = + result = Pleroma.HTTP.get( Path.join(endpoint, path), meili_headers() ) - Jason.decode!(result.body) + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end end - def meili_post!(path, params) do + def meili_post(path, params) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = + result = Pleroma.HTTP.post( Path.join(endpoint, path), Jason.encode!(params), meili_headers() ) - Jason.decode!(result.body) + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_put(path, params) do + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.request( + :put, + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers(), + [] + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end end def meili_delete!(path) do @@ -57,34 +78,40 @@ defmodule Pleroma.Search.Meilisearch do offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) - result = - meili_post!( + res = + meili_post( "/indexes/objects/search", %{q: query, offset: offset, limit: limit} ) - hits = result["hits"] |> Enum.map(& &1["ap"]) - - try do - hits - |> Activity.create_by_object_ap_id() - |> Activity.with_preloaded_object() - |> Activity.with_preloaded_object() - |> Activity.restrict_deactivated_users() - |> maybe_restrict_local(user) - |> maybe_restrict_author(author) - |> maybe_restrict_blocked(user) - |> maybe_fetch(user, query) - |> order_by([object: obj], desc: obj.data["published"]) - |> Pleroma.Repo.all() - rescue - _ -> maybe_fetch([], user, query) + with {:ok, result} <- res do + hits = result["hits"] |> Enum.map(& &1["ap"]) + + try do + hits + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object() + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> maybe_fetch(user, query) + |> order_by([object: obj], desc: obj.data["published"]) + |> Pleroma.Repo.all() + rescue + _ -> maybe_fetch([], user, query) + end end end def object_to_search_data(object) do + # Only index public or unlisted Notes if not is_nil(object) and object.data["type"] == "Note" and - Pleroma.Constants.as_public() in object.data["to"] do + not is_nil(object.data["content"]) and + (Pleroma.Constants.as_public() in object.data["to"] or + Pleroma.Constants.as_public() in object.data["cc"]) and + String.length(object.data["content"]) > 1 do data = object.data content_str = @@ -117,13 +144,17 @@ defmodule Pleroma.Search.Meilisearch do if activity.data["type"] == "Create" and maybe_search_data do result = - meili_post!( + meili_put( "/indexes/objects/documents", [maybe_search_data] ) - if not Map.has_key?(result, "updateId") do - Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") + with {:ok, res} <- result, + true <- Map.has_key?(res, "updateId") do + # Do nothing + else + _ -> + Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") end end end -- cgit v1.2.3 From 3179ed0921197a8a8f32a519c7d41dc09011024d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 18:48:52 +0300 Subject: Make chunk size configurable --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 6730a99a9..021552f7b 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -38,7 +38,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do IO.puts("Created indices. Starting to insert posts.") - chunk_size = 10_000 + chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size]) Pleroma.Repo.transaction( fn -> -- cgit v1.2.3 From 571533ae2618478f26db312e52265e143356debd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 19:05:59 +0300 Subject: Don't support meilisearch < 0.24.0, since it breaks things --- lib/mix/tasks/pleroma/search/meilisearch.ex | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 021552f7b..5098668ad 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -14,17 +14,29 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["index"]) do start_pleroma() + meili_version = + ( + {:ok, result} = meili_get("/version") + + result["pkgVersion"] + ) + + # The ranking rule syntax was changed but nothing about that is mentioned in the changelog + if not Version.match?(meili_version, ">= 0.24.0") do + raise "Meilisearch <0.24.0 not supported" + end + {:ok, _} = meili_post( "/indexes/objects/settings/ranking-rules", [ - "desc(published)", + "published:desc", "words", "exactness", "proximity", - "wordsPosition", "typo", - "attribute" + "attribute", + "sort" ] ) -- cgit v1.2.3 From 6f2f457751ea09507045e6dd5d5869a14befd3d1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 22:38:50 +0300 Subject: Add a search backend behaviour --- lib/pleroma/search/database_search.ex | 5 +++++ lib/pleroma/search/meilisearch.ex | 4 ++++ lib/pleroma/search/search_backend.ex | 17 +++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 lib/pleroma/search/search_backend.ex (limited to 'lib') diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 5a8b8ca67..3735a5fab 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -13,6 +13,8 @@ defmodule Pleroma.Search.DatabaseSearch do import Ecto.Query + @behaviour Pleroma.Search.SearchBackend + def search(user, search_query, options \\ []) do index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) @@ -45,7 +47,10 @@ defmodule Pleroma.Search.DatabaseSearch do end end + @impl true def add_to_index(_activity), do: nil + + @impl true def remove_from_index(_object), do: nil def maybe_restrict_author(query, %User{} = author) do diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 21b44de86..33bbf8392 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -7,6 +7,8 @@ defmodule Pleroma.Search.Meilisearch do import Pleroma.Search.DatabaseSearch import Ecto.Query + @behaviour Pleroma.Search.SearchBackend + defp meili_headers do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) @@ -139,6 +141,7 @@ defmodule Pleroma.Search.Meilisearch do end end + @impl true def add_to_index(activity) do maybe_search_data = object_to_search_data(activity.object) @@ -159,6 +162,7 @@ defmodule Pleroma.Search.Meilisearch do end end + @impl true def remove_from_index(object) do meili_delete!("/indexes/objects/documents/#{object.id}") end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex new file mode 100644 index 000000000..ed6bfd329 --- /dev/null +++ b/lib/pleroma/search/search_backend.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.Search.SearchBackend do + @doc """ + Add the object associated with the activity to the search index. + + The whole activity is passed, to allow filtering on things such as scope. + """ + @callback add_to_index(activity :: Pleroma.Activity.t()) :: nil + + @doc """ + Remove the object from the index. + + Just the object, as opposed to the whole activity, is passed, since the object + is what contains the actual content and there is no need for fitlering when removing + from index. + """ + @callback remove_from_index(object :: Pleroma.Object.t()) :: nil +end -- cgit v1.2.3 From 2bc21c6f1884bae3226f760ed1da39dd9c5f2958 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 15:23:11 +0300 Subject: Use oban for search indexing --- lib/pleroma/search.ex | 15 ++++----------- lib/pleroma/workers/search_indexing_worker.ex | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+), 11 deletions(-) create mode 100644 lib/pleroma/workers/search_indexing_worker.ex (limited to 'lib') diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index ae0b28c54..af858fc46 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -1,19 +1,12 @@ defmodule Pleroma.Search do - def add_to_index(activity) do - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + alias Pleroma.Workers.SearchIndexingWorker - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.add_to_index(activity) end) - end) + def add_to_index(activity) do + SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id}) end def remove_from_index(object) do - # Also delete from search index - search_module = Pleroma.Config.get([Pleroma.Search, :module]) - - ConcurrentLimiter.limit(Pleroma.Search, fn -> - Task.start(fn -> search_module.remove_from_index(object) end) - end) + SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id}) end def search(query, options) do diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex new file mode 100644 index 000000000..43b7bad1e --- /dev/null +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -0,0 +1,21 @@ +defmodule Pleroma.Workers.SearchIndexingWorker do + use Pleroma.Workers.WorkerHelper, queue: "search_indexing" + + @impl Oban.Worker + + def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do + activity = Pleroma.Activity.get_by_id_with_object(activity_id) + + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + search_module.add_to_index(activity) + end + + def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do + object = Pleroma.Object.get_by_id(object_id) + + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + + search_module.remove_from_index(object) + end +end -- cgit v1.2.3 From d89dc5518b5c0eb232e7ac85ddd538f89c32606d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 16:31:32 +0300 Subject: Fix meilisearch tests and jobs for oban --- lib/pleroma/workers/search_indexing_worker.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 43b7bad1e..70a8d42d0 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -9,6 +9,8 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.add_to_index(activity) + + :ok end def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do @@ -17,5 +19,7 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.remove_from_index(object) + + :ok end end -- cgit v1.2.3 From 3387935e8354e32171fe6e28a8f96f49154acbb3 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 22 Jan 2022 16:52:06 +0300 Subject: Don't try removing deleted users and such from index as posts --- lib/pleroma/search.ex | 8 ++++---- lib/pleroma/web/activity_pub/side_effects.ex | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index af858fc46..3b266e59b 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -1,12 +1,12 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker - def add_to_index(activity) do - SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity.id}) + def add_to_index(%Pleroma.Activity{id: activity_id}) do + SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id}) end - def remove_from_index(object) do - SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object.id}) + def remove_from_index(%Pleroma.Object{id: object_id}) do + SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id}) end def search(query, options) do diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 4762b5ac6..644e62630 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -330,7 +330,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if result == :ok do Notification.create_notifications(object) - Pleroma.Search.remove_from_index(deleted_object) + # Only remove from index when deleting actual objects, not users or anything else + with %Pleroma.Object{} <- deleted_object do + Pleroma.Search.remove_from_index(deleted_object) + end {:ok, object, meta} else -- cgit v1.2.3 From 1e23f527e3e22108b402552a0766e488048ed3f4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 22 Mar 2022 20:29:17 +0300 Subject: Change the meilisearch key auth to conform to 0.25.0 --- lib/mix/tasks/pleroma/search/meilisearch.ex | 14 ++++++++------ lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 5098668ad..db56876fa 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ) # The ranking rule syntax was changed but nothing about that is mentioned in the changelog - if not Version.match?(meili_version, ">= 0.24.0") do + if not Version.match?(meili_version, ">= 0.25.0") do raise "Meilisearch <0.24.0 not supported" end @@ -112,7 +112,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do meili_delete!("/indexes/objects/documents") end - def run(["show-private-key", master_key]) do + def run(["show-keys", master_key]) do start_pleroma() endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) @@ -120,15 +120,17 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {:ok, result} = Pleroma.HTTP.get( Path.join(endpoint, "/keys"), - [{"X-Meili-API-Key", master_key}] + [{"Authorization", "Bearer #{master_key}"}] ) decoded = Jason.decode!(result.body) - if decoded["private"] do - IO.puts(decoded["private"]) + if decoded["results"] do + Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} -> + IO.puts("#{desc}: #{key}") + end) else - IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") + IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}") end end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 33bbf8392..0f9182ffc 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -13,7 +13,7 @@ defmodule Pleroma.Search.Meilisearch do private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) [{"Content-Type", "application/json"}] ++ - if is_nil(private_key), do: [], else: [{"X-Meili-API-Key", private_key}] + if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}] end def meili_get(path) do -- cgit v1.2.3 From 84608be87e2c5961a4deb9030307c978bf1168e5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 22 Mar 2022 20:45:49 +0300 Subject: Change updateId to uid because apparently that's the new name --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- lib/pleroma/search/meilisearch.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index db56876fa..d4a83c3cd 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -91,7 +91,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ) with {:ok, res} <- result do - if not Map.has_key?(res, "updateId") do + if not Map.has_key?(res, "uid") do IO.puts("\nFailed to index: #{inspect(result)}") end else diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 0f9182ffc..3db65f261 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -153,7 +153,7 @@ defmodule Pleroma.Search.Meilisearch do ) with {:ok, res} <- result, - true <- Map.has_key?(res, "updateId") do + true <- Map.has_key?(res, "uid") do # Do nothing else _ -> -- cgit v1.2.3 From e20f74c71b078d706bc93632773f9b590d2fb018 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Fri, 26 Aug 2022 23:39:58 +0300 Subject: Remove duplicate function call --- lib/pleroma/search/meilisearch.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 3db65f261..53f8a2544 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -93,7 +93,6 @@ defmodule Pleroma.Search.Meilisearch do hits |> Activity.create_by_object_ap_id() |> Activity.with_preloaded_object() - |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() |> maybe_restrict_local(user) |> maybe_restrict_author(author) -- cgit v1.2.3 From 119b2b847b76c7300bd71699d9f2e5676bdb0bb4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 00:09:37 +0300 Subject: Instead of checking string length, explicitly check for "" and "." --- lib/pleroma/search/meilisearch.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 53f8a2544..2e13b8407 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -112,7 +112,7 @@ defmodule Pleroma.Search.Meilisearch do not is_nil(object.data["content"]) and (Pleroma.Constants.as_public() in object.data["to"] or Pleroma.Constants.as_public() in object.data["cc"]) and - String.length(object.data["content"]) > 1 do + object.data["content"] not in ["", "."] do data = object.data content_str = @@ -127,7 +127,8 @@ defmodule Pleroma.Search.Meilisearch do trimmed end - if String.length(content) > 1 do + # Make sure we have a non-empty string + if content != "" do {:ok, published, _} = DateTime.from_iso8601(data["published"]) %{ -- cgit v1.2.3 From 102ebb42bdba1673da39a8fa8ed1662bc8565aa4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 00:19:08 +0300 Subject: Make search a callback --- lib/pleroma/search/database_search.ex | 1 + lib/pleroma/search/meilisearch.ex | 1 + lib/pleroma/search/search_backend.ex | 11 +++++++++-- 3 files changed, 11 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 3735a5fab..9a340abf1 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Search.DatabaseSearch do @behaviour Pleroma.Search.SearchBackend + @impl true def search(user, search_query, options \\ []) do index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2e13b8407..4e88169d2 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -75,6 +75,7 @@ defmodule Pleroma.Search.Meilisearch do ) end + @impl true def search(user, query, options \\ []) do limit = Enum.min([Keyword.get(options, :limit), 40]) offset = Keyword.get(options, :offset, 0) diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index ed6bfd329..a42e2f5f6 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -1,10 +1,17 @@ defmodule Pleroma.Search.SearchBackend do + @doc """ + Search statuses with a query, restricting to only those the user should have access to. + """ + @callback search(user :: Pleroma.User.t(), query :: String.t(), options :: [any()]) :: [ + Pleroma.Activity.t() + ] + @doc """ Add the object associated with the activity to the search index. The whole activity is passed, to allow filtering on things such as scope. """ - @callback add_to_index(activity :: Pleroma.Activity.t()) :: nil + @callback add_to_index(activity :: Pleroma.Activity.t()) :: :ok | {:error, any()} @doc """ Remove the object from the index. @@ -13,5 +20,5 @@ defmodule Pleroma.Search.SearchBackend do is what contains the actual content and there is no need for fitlering when removing from index. """ - @callback remove_from_index(object :: Pleroma.Object.t()) :: nil + @callback remove_from_index(object :: Pleroma.Object.t()) :: {:ok, any()} | {:error, any()} end -- cgit v1.2.3 From 5ac67632384bfb284ac51f2a450d41cf3913378a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 00:31:36 +0300 Subject: Make add_to_index and remove_from_index report errors --- lib/mix/tasks/pleroma/search/meilisearch.ex | 4 ++-- lib/pleroma/search/meilisearch.ex | 27 ++++++++++++++++----------- lib/pleroma/workers/search_indexing_worker.ex | 4 ---- 3 files changed, 18 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index d4a83c3cd..72a558228 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Ecto.Query import Pleroma.Search.Meilisearch, - only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1] def run(["index"]) do start_pleroma() @@ -109,7 +109,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["clear"]) do start_pleroma() - meili_delete!("/indexes/objects/documents") + meili_delete("/indexes/objects/documents") end def run(["show-keys", master_key]) do diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 4e88169d2..24789b00c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -62,17 +62,16 @@ defmodule Pleroma.Search.Meilisearch do end end - def meili_delete!(path) do + def meili_delete(path) do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = - Pleroma.HTTP.request( - :delete, - Path.join(endpoint, path), - "", - meili_headers(), - [] - ) + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + [] + ) end @impl true @@ -155,16 +154,22 @@ defmodule Pleroma.Search.Meilisearch do with {:ok, res} <- result, true <- Map.has_key?(res, "uid") do - # Do nothing + # Added successfully + :ok else _ -> + # There was an error, report it Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") + {:error, result} end + else + # The post isn't something we can search, that's ok + :ok end end @impl true def remove_from_index(object) do - meili_delete!("/indexes/objects/documents/#{object.id}") + meili_delete("/indexes/objects/documents/#{object.id}") end end diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 70a8d42d0..43b7bad1e 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.add_to_index(activity) - - :ok end def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do @@ -19,7 +17,5 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.remove_from_index(object) - - :ok end end -- cgit v1.2.3 From 6256822afd368e5f6b410d47c5ff9b584e50a461 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 01:11:50 +0300 Subject: Check for updateId, not uid --- lib/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 24789b00c..0b90971b1 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -153,7 +153,7 @@ defmodule Pleroma.Search.Meilisearch do ) with {:ok, res} <- result, - true <- Map.has_key?(res, "uid") do + true <- Map.has_key?(res, "updateId") do # Added successfully :ok else -- cgit v1.2.3 From 5a39866388c411f2bcee9848352f8c420513f34f Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 01:43:59 +0300 Subject: Specifically strip mentions for search indexing --- lib/mix/tasks/pleroma/search/meilisearch.ex | 1 + lib/pleroma/search/meilisearch.ex | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 72a558228..8379a0c25 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -13,6 +13,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["index"]) do start_pleroma() + Pleroma.HTML.compile_scrubbers() meili_version = ( diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 0b90971b1..7af7f460a 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -122,7 +122,8 @@ defmodule Pleroma.Search.Meilisearch do end content = - with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + with {:ok, scrubbed} <- + FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing), trimmed <- String.trim(scrubbed) do trimmed end -- cgit v1.2.3 From e3110cb34e350bc9353d082328778daae47ba9e8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 13:36:21 -0400 Subject: Fix deprecated calls to get_flash/2 --- lib/pleroma/web.ex | 2 +- lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex | 8 ++++---- lib/pleroma/web/templates/o_auth/mfa/totp.html.eex | 8 ++++---- lib/pleroma/web/templates/o_auth/o_auth/register.html.eex | 8 ++++---- lib/pleroma/web/templates/o_auth/o_auth/show.html.eex | 8 ++++---- 5 files changed, 17 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index aee41b0fe..7a8b176cd 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -136,7 +136,7 @@ defmodule Pleroma.Web do namespace: Pleroma.Web # Import convenience functions from controllers - import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1] + import Phoenix.Controller, only: [get_csrf_token: 0, view_module: 1] import Pleroma.Web.ErrorHelpers import Pleroma.Web.Gettext diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index e45d13bdf..e3639aae7 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - +<%= if Phoenix.Flash.get(@flash, :info) do %> + <% end %> -<%= if get_flash(@conn, :error) do %> - +<%= if Phoenix.Flash.get(@flash, :error) do %> + <% end %>

<%= Gettext.dpgettext("static_pages", "mfa recover page title", "Two-factor recovery") %>

diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index 50e6c04b6..f995b8805 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - +<%= if Phoenix.Flash.get(@flash, :info) do %> + <% end %> -<%= if get_flash(@conn, :error) do %> - +<%= if Phoenix.Flash.get(@flash, :error) do %> + <% end %>

<%= Gettext.dpgettext("static_pages", "mfa auth page title", "Two-factor authentication") %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index 1f661efb2..e7f65266f 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - +<%= if Phoenix.Flash.get(@flash, :info) do %> + <% end %> -<%= if get_flash(@conn, :error) do %> - +<%= if Phoenix.Flash.get(@flash, :error) do %> + <% end %>

<%= Gettext.dpgettext("static_pages", "oauth register page title", "Registration Details") %>

diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index b3654f3eb..5b38f7142 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - +<%= if Phoenix.Flash.get(@flash, :info) do %> + <% end %> -<%= if get_flash(@conn, :error) do %> - +<%= if Phoenix.Flash.get(@flash, :error) do %> + <% end %> <%= form_for @conn, Routes.o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> -- cgit v1.2.3 From ffee478ed0298f2cf29d4d51ed28105119552496 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 15:30:58 -0400 Subject: Move websocket config for Shoutbox to the Endpoint This is the modern way of configuring it --- lib/pleroma/web/endpoint.ex | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index d8d40cceb..9e0340cd2 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,7 +9,20 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config - socket("/socket", Pleroma.Web.UserSocket) + socket("/socket", Pleroma.Web.UserSocket, + websocket: [ + path: "/websocket", + serializer: [ + {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"}, + {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"} + ], + timeout: 60_000, + transport_log: false, + compress: false + ], + longpoll: false + ) + socket("/live", Phoenix.LiveView.Socket) plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) -- cgit v1.2.3 From 62322f71e2f2e607ee66a91a99e35eecbf89914d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 16:22:40 -0400 Subject: Clean up Plug.Parsers.MULTIPART deprecation warnings There is no need to the length setting to :multipart. The length setting is global for all of the parsers. --- lib/pleroma/web/endpoint.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 9e0340cd2..201a92653 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -116,7 +116,7 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Parsers, parsers: [ :urlencoded, - {:multipart, length: {Config, :get, [[:instance, :upload_limit]]}}, + :multipart, :json ], pass: ["*/*"], -- cgit v1.2.3 From 2b8bbb288c26a95970e2d9e988f5eb303e052644 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 22:11:17 +0000 Subject: Phoenix.Socket.Transport.force_ssl/4 no longer exists --- lib/phoenix/transports/web_socket/raw.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/phoenix/transports/web_socket/raw.ex b/lib/phoenix/transports/web_socket/raw.ex index 8cf9c32a2..cf4fda79f 100644 --- a/lib/phoenix/transports/web_socket/raw.ex +++ b/lib/phoenix/transports/web_socket/raw.ex @@ -26,7 +26,6 @@ defmodule Phoenix.Transports.WebSocket.Raw do conn |> fetch_query_params |> Transport.transport_log(opts[:transport_log]) - |> Transport.force_ssl(handler, endpoint, opts) |> Transport.check_origin(handler, endpoint, opts) case conn do -- cgit v1.2.3 From f0e5f0e8375bb55e13b0f054fcf09871d2967d24 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 May 2023 22:14:55 +0000 Subject: Fix compile warning warning: doing a prefix match with globs is deprecated, invalid segment "pleroma*path" --- lib/pleroma/web/router.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index c1a690e28..6b9e158a3 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -996,8 +996,8 @@ defmodule Pleroma.Web.Router do scope "/", Pleroma.Web.Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) - match(:*, "/api/pleroma*path", LegacyPleromaApiRerouterPlug, []) - get("/api*path", RedirectController, :api_not_implemented) + match(:*, "/api/pleroma/*path", LegacyPleromaApiRerouterPlug, []) + get("/api/*path", RedirectController, :api_not_implemented) get("/*path", RedirectController, :redirector_with_preload) options("/*path", RedirectController, :empty) -- cgit v1.2.3 From 63ef1dcedca7fb315f611388e477d3847d1acaa1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 3 Jun 2023 14:17:49 -0400 Subject: Phoenix.Router.routes/1 is the public function we are meant to be using here --- lib/pleroma/web/router.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..a6e180c4c 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -1003,9 +1003,8 @@ defmodule Pleroma.Web.Router do options("/*path", RedirectController, :empty) end - # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+ def get_api_routes do - __MODULE__.__routes__() + Phoenix.Router.routes(__MODULE__) |> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end) |> Enum.map(fn r -> r.path -- cgit v1.2.3 From 9effa24f308917f70276c41f91fb204b7684d942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 10 Aug 2023 22:52:38 +0200 Subject: Implement api/v2/instance route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/api_spec/operations/instance_operation.ex | 172 +++++++++++++++++++++ .../controllers/instance_controller.ex | 5 + .../web/mastodon_api/views/instance_view.ex | 120 +++++++++++--- lib/pleroma/web/router.ex | 3 + lib/pleroma/web/web_finger.ex | 2 +- 5 files changed, 279 insertions(+), 23 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index a07be7e40..8e395bde8 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -23,6 +23,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + def show2_operation do + %Operation{ + tags: ["Instance misc"], + summary: "Retrieve instance information", + description: "Information about the server", + operationId: "InstanceController.show2", + responses: %{ + 200 => Operation.response("Instance", "application/json", instance2()) + } + } + end + def peers_operation do %Operation{ tags: ["Instance misc"], @@ -165,6 +177,166 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + defp instance2 do + %Schema{ + type: :object, + properties: %{ + domain: %Schema{type: :string, description: "The domain name of the instance"}, + title: %Schema{type: :string, description: "The title of the website"}, + version: %Schema{ + type: :string, + description: "The version of Pleroma installed on the instance" + }, + source_url: %Schema{ + type: :string, + description: "The version of Pleroma installed on the instance" + }, + description: %Schema{ + type: :string, + description: "Admin-defined description of the Pleroma site" + }, + usage: %Schema{ + type: :object, + description: "Instance usage statistics", + properties: %{ + users: %Schema{ + type: :object, + description: "User count statistics", + properties: %{ + active_month: %Schema{ + type: :integer, + description: "Monthly active users" + } + } + } + } + }, + email: %Schema{ + type: :string, + description: "An email that may be contacted for any inquiries", + format: :email + }, + urls: %Schema{ + type: :object, + description: "URLs of interest for clients apps", + properties: %{} + }, + stats: %Schema{ + type: :object, + description: "Statistics about how much information the instance contains", + properties: %{ + user_count: %Schema{ + type: :integer, + description: "Users registered on this instance" + }, + status_count: %Schema{ + type: :integer, + description: "Statuses authored by users on instance" + }, + domain_count: %Schema{ + type: :integer, + description: "Domains federated with this instance" + } + } + }, + thumbnail: %Schema{ + type: :object, + properties: %{ + url: %Schema{ + type: :string, + description: "Banner image for the website", + nullable: true + } + } + }, + languages: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: "Primary langauges of the website and its staff" + }, + registrations: %Schema{ + type: :object, + description: "Registrations-related configuration", + properties: %{ + enabled: %Schema{ + type: :boolean, + description: "Whether registrations are enabled" + }, + approval_required: %Schema{ + type: :boolean, + description: "Whether users need to be manually approved by admin" + } + } + }, + configuration: %Schema{ + type: :object, + description: "Instance configuration", + properties: %{ + urls: %Schema{ + type: :object, + properties: %{ + streaming: %Schema{ + type: :string, + description: "Websockets address for push streaming" + } + } + }, + statuses: %Schema{ + type: :object, + description: "A map with poll limits for local statuses", + properties: %{ + max_characters: %Schema{ + type: :integer, + description: "Posts character limit (CW/Subject included in the counter)" + }, + max_media_attachments: %Schema{ + type: :integer, + description: "Media attachment limit" + } + } + }, + media_attachments: %Schema{ + type: :object, + description: "A map with poll limits for media attachments", + properties: %{ + image_size_limit: %Schema{ + type: :integer, + description: "File size limit of uploaded images" + }, + video_size_limit: %Schema{ + type: :integer, + description: "File size limit of uploaded videos" + } + } + }, + polls: %Schema{ + type: :object, + description: "A map with poll limits for local polls", + properties: %{ + max_options: %Schema{ + type: :integer, + description: "Maximum number of options." + }, + max_characters_per_option: %Schema{ + type: :integer, + description: "Maximum number of characters per option." + }, + min_expiration: %Schema{ + type: :integer, + description: "Minimum expiration time (in seconds)." + }, + max_expiration: %Schema{ + type: :integer, + description: "Maximum expiration time (in seconds)." + } + } + } + } + } + } + } + end + defp array_of_domains do %Schema{ type: :array, diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 6410e872c..3757a850a 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -16,6 +16,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do render(conn, "show.json") end + @doc "GET /api/v2/instance" + def show2(conn, _params) do + render(conn, "show2.json") + end + @doc "GET /api/v1/instance/peers" def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index efd2a0af6..c987cafd6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do instance = Config.get(:instance) %{ - uri: Pleroma.Web.Endpoint.url(), + uri: Pleroma.Web.WebFinger.domain(), title: Keyword.get(instance, :name), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), @@ -30,6 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do languages: Keyword.get(instance, :languages, ["en"]), registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), + configuration: configuration(), # Extra (not present in Mastodon): max_toot_chars: Keyword.get(instance, :limit), max_media_attachments: Keyword.get(instance, :max_media_attachments), @@ -41,19 +42,38 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), - pleroma: %{ - metadata: %{ - account_activation_required: Keyword.get(instance, :account_activation_required), - features: features(), - federation: federation(), - fields_limits: fields_limits(), - post_formats: Config.get([:instance, :allowed_post_formats]), - birthday_required: Config.get([:instance, :birthday_required]), - birthday_min_age: Config.get([:instance, :birthday_min_age]) - }, - stats: %{mau: Pleroma.User.active_user_count()}, - vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) - } + pleroma: pleroma_configuration(instance) + } + end + + def render("show2.json", _) do + instance = Config.get(:instance) + + %{ + domain: Pleroma.Web.WebFinger.domain(), + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", + source_url: Pleroma.Application.repository(), + description: Keyword.get(instance, :short_description), + usage: %{users: %{active_month: Pleroma.User.active_user_count()}}, + thumbnail: %{ + url: + URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) + |> to_string + }, + languages: Keyword.get(instance, :languages, ["en"]), + configuration: configuration2(), + registrations: %{ + enabled: Keyword.get(instance, :registrations_open), + approval_required: Keyword.get(instance, :account_approval_required), + message: nil + }, + contact: %{ + email: Keyword.get(instance, :email), + account: nil + }, + # Extra (not present in Mastodon): + pleroma: pleroma_configuration2(instance) } end @@ -68,6 +88,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "shareable_emoji_packs", "multifetch", "pleroma:api/v1/notifications:include_types_filter", + "quote_posting", "editing", if Config.get([:activitypub, :blockers_visible]) do "blockers_visible" @@ -78,13 +99,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do if Config.get([:gopher, :enabled]) do "gopher" end, - # backwards compat - if Config.get([:shout, :enabled]) do - "chat" - end, - if Config.get([:shout, :enabled]) do - "shout" - end, if Config.get([:instance, :allow_relay]) do "relay" end, @@ -94,6 +108,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "pleroma_emoji_reactions", "pleroma_custom_emoji_reactions", "pleroma_chat_messages", + "email_list", if Config.get([:instance, :show_reactions]) do "exposable_reactions" end, @@ -132,7 +147,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do |> Map.put(:enabled, Config.get([:instance, :federating])) end - def fields_limits do + defp fields_limits do %{ max_fields: Config.get([:instance, :max_account_fields]), max_remote_fields: Config.get([:instance, :max_remote_account_fields]), @@ -140,4 +155,65 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do value_length: Config.get([:instance, :account_field_value_length]) } end + + defp configuration do + %{ + statuses: %{ + max_characters: Config.get([:instance, :limit]), + max_media_attachments: Config.get([:instance, :max_media_attachments]) + }, + media_attachments: %{ + image_size_limit: Config.get([:instance, :upload_limit]), + video_size_limit: Config.get([:instance, :upload_limit]) + }, + polls: %{ + max_options: Config.get([:instance, :poll_limits, :max_options]), + max_characters_per_option: Config.get([:instance, :poll_limits, :max_option_chars]), + min_expiration: Config.get([:instance, :poll_limits, :min_expiration]), + max_expiration: Config.get([:instance, :poll_limits, :max_expiration]) + } + } + end + + defp configuration2 do + configuration() + |> Map.merge(%{ + urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()} + }) + end + + defp pleroma_configuration(instance) do + %{ + metadata: %{ + account_activation_required: Keyword.get(instance, :account_activation_required), + features: features(), + federation: federation(), + fields_limits: fields_limits(), + post_formats: Config.get([:instance, :allowed_post_formats]), + birthday_required: Config.get([:instance, :birthday_required]), + birthday_min_age: Config.get([:instance, :birthday_min_age]) + }, + stats: %{mau: Pleroma.User.active_user_count()}, + vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } + end + + defp pleroma_configuration2(instance) do + configuration = pleroma_configuration(instance) + + configuration + |> Map.merge(%{ + metadata: + configuration.metadata + |> Map.merge(%{ + avatar_upload_limit: Keyword.get(instance, :avatar_upload_limit), + background_upload_limit: Keyword.get(instance, :background_upload_limit), + banner_upload_limit: Keyword.get(instance, :banner_upload_limit), + background_image: + Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), + description_limit: Keyword.get(instance, :description_limit), + shout_limit: Config.get([:shout, :limit]) + }) + }) + end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..5362100cc 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -774,11 +774,14 @@ defmodule Pleroma.Web.Router do scope "/api/v2", Pleroma.Web.MastodonAPI do pipe_through(:api) + get("/search", SearchController, :search2) post("/media", MediaController, :create2) get("/suggestions", SuggestionController, :index2) + + get("/instance", InstanceController, :show2) end scope "/api", Pleroma.Web do diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index f95dc2458..b28fad8d1 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -96,7 +96,7 @@ defmodule Pleroma.Web.WebFinger do |> XmlBuilder.to_doc() end - defp domain do + def domain do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end -- cgit v1.2.3 From 79e46ce73f782a83654986adc9fd0b256be6a2e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Fri, 11 Aug 2023 13:57:22 +0200 Subject: InstanceView: Add common_information function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- .../web/mastodon_api/views/instance_view.ex | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index c987cafd6..e7a2feb7f 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -13,12 +13,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def render("show.json", _) do instance = Config.get(:instance) - %{ + common_information(instance) + |> Map.merge(%{ uri: Pleroma.Web.WebFinger.domain(), - title: Keyword.get(instance, :name), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", email: Keyword.get(instance, :email), urls: %{ streaming_api: Pleroma.Web.Endpoint.websocket_url() @@ -27,7 +26,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do thumbnail: URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) |> to_string, - languages: Keyword.get(instance, :languages, ["en"]), registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), configuration: configuration(), @@ -43,16 +41,15 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), pleroma: pleroma_configuration(instance) - } + }) end def render("show2.json", _) do instance = Config.get(:instance) - %{ + common_information(instance) + |> Map.merge(%{ domain: Pleroma.Web.WebFinger.domain(), - title: Keyword.get(instance, :name), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", source_url: Pleroma.Application.repository(), description: Keyword.get(instance, :short_description), usage: %{users: %{active_month: Pleroma.User.active_user_count()}}, @@ -61,7 +58,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) |> to_string }, - languages: Keyword.get(instance, :languages, ["en"]), configuration: configuration2(), registrations: %{ enabled: Keyword.get(instance, :registrations_open), @@ -74,6 +70,14 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }, # Extra (not present in Mastodon): pleroma: pleroma_configuration2(instance) + }) + end + + defp common_information(instance) do + %{ + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", + languages: Keyword.get(instance, :languages, ["en"]) } end -- cgit v1.2.3 From 28ef5ebd3c7c2fc083424dbf4434392fcd2a7aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 7 Sep 2023 15:00:24 +0200 Subject: Update InstanceView.features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index e7a2feb7f..c1cecf6e6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -92,7 +92,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "shareable_emoji_packs", "multifetch", "pleroma:api/v1/notifications:include_types_filter", - "quote_posting", "editing", if Config.get([:activitypub, :blockers_visible]) do "blockers_visible" @@ -103,6 +102,13 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do if Config.get([:gopher, :enabled]) do "gopher" end, + # backwards compat + if Config.get([:shout, :enabled]) do + "chat" + end, + if Config.get([:shout, :enabled]) do + "shout" + end, if Config.get([:instance, :allow_relay]) do "relay" end, @@ -112,7 +118,6 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "pleroma_emoji_reactions", "pleroma_custom_emoji_reactions", "pleroma_chat_messages", - "email_list", if Config.get([:instance, :show_reactions]) do "exposable_reactions" end, -- cgit v1.2.3 From 2f6fc6a7ab0e757abfe9ec535842a12b887d2fe6 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 24 Sep 2023 22:52:41 +0200 Subject: TwitterAPI: Return proper error when healthcheck is disabled --- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index d5a24ae6c..ca8a98960 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -345,13 +345,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def healthcheck(conn, _params) do - with true <- Config.get([:instance, :healthcheck]), + with {:cfg, true} <- {:cfg, Config.get([:instance, :healthcheck])}, %{healthy: true} = info <- Healthcheck.system_info() do json(conn, info) else %{healthy: false} = info -> service_unavailable(conn, info) + {:cfg, false} -> + service_unavailable(conn, %{"error" => "Healthcheck disabled"}) + _ -> service_unavailable(conn, %{}) end -- cgit v1.2.3 From 5ff3783d018476bad954881e39f714fac0630436 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Tue, 22 Nov 2022 23:34:23 +0100 Subject: Use correct domain for fqn and InstanceView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 +- lib/pleroma/web/web_finger.ex | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ce125d608..a0d4aca66 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2252,7 +2252,7 @@ defmodule Pleroma.User do if String.contains?(user.nickname, "@") do user.nickname else - %{host: host} = URI.parse(user.ap_id) + host = Pleroma.Web.WebFinger.domain() user.nickname <> "@" <> host end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 1b01d7371..aef02a418 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do instance = Config.get(:instance) %{ - uri: Pleroma.Web.Endpoint.url(), + uri: Pleroma.Web.WebFinger.domain(), title: Keyword.get(instance, :name), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index f95dc2458..b28fad8d1 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -96,7 +96,7 @@ defmodule Pleroma.Web.WebFinger do |> XmlBuilder.to_doc() end - defp domain do + def domain do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end -- cgit v1.2.3 From c6cedbb8106a16527e48ac8ae03907e1d66c5a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 28 Oct 2023 00:07:18 +0200 Subject: InstanceV2: skip auth MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/controllers/instance_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 3757a850a..3e664903a 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do plug(Pleroma.Web.ApiSpec.CastAndValidate) - plug(:skip_auth when action in [:show, :peers]) + plug(:skip_auth when action in [:show, :show2, :peers]) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.InstanceOperation -- cgit v1.2.3 From c62696c8e7a28390880a68392bbd14929b66a56d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 23 Oct 2023 16:31:29 +0200 Subject: Support /authorize-interaction route used by Mastodon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/router.ex | 2 ++ .../web/twitter_api/controllers/remote_follow_controller.ex | 7 +++++++ 2 files changed, 9 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..93a28c9ad 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -465,6 +465,8 @@ defmodule Pleroma.Web.Router do get("/main/ostatus", UtilController, :show_subscribe_form) get("/ostatus_subscribe", RemoteFollowController, :follow) post("/ostatus_subscribe", RemoteFollowController, :do_follow) + + get("/authorize_interaction", RemoteFollowController, :authorize_interaction) end scope "/api/pleroma", Pleroma.Web.TwitterAPI do diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index 6229d5d05..178ad2b43 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -121,6 +121,13 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do render(conn, "followed.html", %{error: "Insufficient permissions: follow | write:follows."}) end + # GET /authorize_interaction + # + def authorize_interaction(conn, %{"uri" => uri}) do + conn + |> redirect(to: Routes.remote_follow_path(conn, :follow, %{acct: uri})) + end + defp handle_follow_error(conn, {:mfa_token, followee, _} = _) do render(conn, "follow_login.html", %{error: "Wrong username or password", followee: followee}) end -- cgit v1.2.3 From bf426c53b4e1c025d7857adf485976421175cdf6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 Nov 2023 15:11:14 -0500 Subject: Fix digest email processing, consolidate Oban queues The email related jobs can all share a single Oban queue --- lib/pleroma/workers/cron/digest_emails_worker.ex | 2 +- lib/pleroma/workers/cron/new_users_digest_worker.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 1540c1605..0292bbb3b 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do The worker to send digest emails. """ - use Oban.Worker, queue: "digest_emails" + use Oban.Worker, queue: "mailer" alias Pleroma.Config alias Pleroma.Emails diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 267fe2837..1c3e445aa 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "new_users_digest" + use Pleroma.Workers.WorkerHelper, queue: "mailer" @impl Oban.Worker def perform(_job) do -- cgit v1.2.3 From e23672d82f0a6e61bf4ada65cfca6cfbc647710d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 8 Nov 2023 12:21:34 -0500 Subject: Ensure benchee doesn't run unless we are executing benchmarks --- lib/mix/tasks/pleroma/benchmark.ex | 113 ------------------------------------- 1 file changed, 113 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/benchmark.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex deleted file mode 100644 index f32492169..000000000 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ /dev/null @@ -1,113 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Benchmark do - import Mix.Pleroma - use Mix.Task - - def run(["search"]) do - start_pleroma() - - Benchee.run(%{ - "search" => fn -> - Pleroma.Activity.search(nil, "cofe") - end - }) - end - - def run(["tag"]) do - start_pleroma() - - Benchee.run(%{ - "tag" => fn -> - %{"type" => "Create", "tag" => "cofe"} - |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() - end - }) - end - - def run(["render_timeline", nickname | _] = args) do - start_pleroma() - user = Pleroma.User.get_by_nickname(nickname) - - activities = - %{} - |> Map.put("type", ["Create", "Announce"]) - |> Map.put("blocking_user", user) - |> Map.put("muting_user", user) - |> Map.put("user", user) - |> Map.put("limit", 4096) - |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() - |> Enum.reverse() - - inputs = %{ - "1 activity" => Enum.take_random(activities, 1), - "10 activities" => Enum.take_random(activities, 10), - "20 activities" => Enum.take_random(activities, 20), - "40 activities" => Enum.take_random(activities, 40), - "80 activities" => Enum.take_random(activities, 80) - } - - inputs = - if Enum.at(args, 2) == "extended" do - Map.merge(inputs, %{ - "200 activities" => Enum.take_random(activities, 200), - "500 activities" => Enum.take_random(activities, 500), - "2000 activities" => Enum.take_random(activities, 2000), - "4096 activities" => Enum.take_random(activities, 4096) - }) - else - inputs - end - - Benchee.run( - %{ - "Standart rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity - }) - end - }, - inputs: inputs - ) - end - - def run(["adapters"]) do - start_pleroma() - - :ok = - Pleroma.Gun.Conn.open( - "https://httpbin.org/stream-bytes/1500", - :gun_connections - ) - - Process.sleep(1_500) - - Benchee.run( - %{ - "Without conn and without pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - pool: :no_pool, - receive_conn: false - ) - end, - "Without conn and with pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false) - end, - "With reused conn and without pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool) - end, - "With reused conn and with pool" => fn -> - {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") - end - }, - parallel: 10 - ) - end -end -- cgit v1.2.3 From c1402af2934219b6ab5dc40a7d87a8c916554647 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 14:49:15 +0400 Subject: B Getting: Add default implementation, delegate, prepare test support. --- lib/pleroma/config/getting.ex | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/config/getting.ex b/lib/pleroma/config/getting.ex index f9b66bba6..0de4782ea 100644 --- a/lib/pleroma/config/getting.ex +++ b/lib/pleroma/config/getting.ex @@ -5,4 +5,11 @@ defmodule Pleroma.Config.Getting do @callback get(any()) :: any() @callback get(any(), any()) :: any() + + def get(key), do: get(key, nil) + def get(key, default), do: impl().get(key, default) + + def impl() do + Application.get_env(:pleroma, :config_impl, Pleroma.Config) + end end -- cgit v1.2.3 From d3f8950588b444dfdf46d87d5631720cc14a907c Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 14:49:50 +0400 Subject: B MeiliSearch, SearchIndexingWorker: Use Config.Getting, make tests async. --- lib/pleroma/search/meilisearch.ex | 32 ++++++++++++++++----------- lib/pleroma/workers/search_indexing_worker.ex | 6 +++-- 2 files changed, 23 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 7af7f460a..eed9fca1c 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -3,6 +3,7 @@ defmodule Pleroma.Search.Meilisearch do require Pleroma.Constants alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config import Pleroma.Search.DatabaseSearch import Ecto.Query @@ -10,14 +11,14 @@ defmodule Pleroma.Search.Meilisearch do @behaviour Pleroma.Search.SearchBackend defp meili_headers do - private_key = Pleroma.Config.get([Pleroma.Search.Meilisearch, :private_key]) + private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) [{"Content-Type", "application/json"}] ++ if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}] end def meili_get(path) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) result = Pleroma.HTTP.get( @@ -31,7 +32,7 @@ defmodule Pleroma.Search.Meilisearch do end def meili_post(path, params) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) result = Pleroma.HTTP.post( @@ -46,7 +47,7 @@ defmodule Pleroma.Search.Meilisearch do end def meili_put(path, params) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) result = Pleroma.HTTP.request( @@ -63,15 +64,20 @@ defmodule Pleroma.Search.Meilisearch do end def meili_delete(path) do - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - Pleroma.HTTP.request( - :delete, - Path.join(endpoint, path), - "", - meili_headers(), - [] - ) + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + with {:ok, _} <- + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + [] + ) do + :ok + else + _ -> :error + end end @impl true diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 43b7bad1e..8476a2be5 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -3,10 +3,12 @@ defmodule Pleroma.Workers.SearchIndexingWorker do @impl Oban.Worker + alias Pleroma.Config.Getting, as: Config + def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do activity = Pleroma.Activity.get_by_id_with_object(activity_id) - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + search_module = Config.get([Pleroma.Search, :module]) search_module.add_to_index(activity) end @@ -14,7 +16,7 @@ defmodule Pleroma.Workers.SearchIndexingWorker do def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do object = Pleroma.Object.get_by_id(object_id) - search_module = Pleroma.Config.get([Pleroma.Search, :module]) + search_module = Config.get([Pleroma.Search, :module]) search_module.remove_from_index(object) end -- cgit v1.2.3 From a1a25029da74949a79c73d400b6f2bc0bf1dc01a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 16:19:54 +0400 Subject: B DatabaseSearch: Fix local-only search. --- lib/pleroma/search/database_search.ex | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 9a340abf1..f4c405773 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Search.DatabaseSearch do alias Pleroma.Pagination alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Config require Pleroma.Constants @@ -17,7 +18,7 @@ defmodule Pleroma.Search.DatabaseSearch do @impl true def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + index_type = if Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) @@ -33,7 +34,7 @@ defmodule Pleroma.Search.DatabaseSearch do Activity |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() - |> restrict_public() + |> restrict_public(user) |> query_with(index_type, search_query, search_function) |> maybe_restrict_local(user) |> maybe_restrict_author(author) @@ -49,10 +50,10 @@ defmodule Pleroma.Search.DatabaseSearch do end @impl true - def add_to_index(_activity), do: nil + def add_to_index(_activity), do: :ok @impl true - def remove_from_index(_object), do: nil + def remove_from_index(_object), do: :ok def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) @@ -66,7 +67,19 @@ defmodule Pleroma.Search.DatabaseSearch do def maybe_restrict_blocked(query, _), do: query - def restrict_public(q) do + defp restrict_public(q, user) when not is_nil(user) do + intended_recipients = [ + Pleroma.Constants.as_public(), + Pleroma.Web.ActivityPub.Utils.as_local_public() + ] + + from([a, o] in q, + where: fragment("?->>'type' = 'Create'", a.data), + where: fragment("? && ?", ^intended_recipients, a.recipients) + ) + end + + defp restrict_public(q, _user) do from([a, o] in q, where: fragment("?->>'type' = 'Create'", a.data), where: ^Pleroma.Constants.as_public() in a.recipients @@ -134,7 +147,7 @@ defmodule Pleroma.Search.DatabaseSearch do end def maybe_restrict_local(q, user) do - limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) + limit = Config.get([:instance, :limit_to_local_content], :unauthenticated) case {limit, user} do {:all, _} -> restrict_local(q) -- cgit v1.2.3 From 59018d73c366d9297efe83d290c717d1a3e4756a Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 16:43:50 +0400 Subject: B Meilisearch: Update to current API responses. --- lib/pleroma/search/meilisearch.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index eed9fca1c..2bff663e8 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -76,7 +76,7 @@ defmodule Pleroma.Search.Meilisearch do ) do :ok else - _ -> :error + _ -> {:error, "Could not remove from index"} end end @@ -159,8 +159,7 @@ defmodule Pleroma.Search.Meilisearch do [maybe_search_data] ) - with {:ok, res} <- result, - true <- Map.has_key?(res, "updateId") do + with {:ok, %{"status" => "enqueued"}} <- result do # Added successfully :ok else -- cgit v1.2.3 From 3d62c71edf8782c5ceae5a0ea3ba5ec08dc5b948 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 12 Nov 2023 17:13:27 +0400 Subject: Credo fixes. --- lib/pleroma/config/getting.ex | 2 +- lib/pleroma/search/database_search.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/getting.ex b/lib/pleroma/config/getting.ex index 0de4782ea..ec93fd02a 100644 --- a/lib/pleroma/config/getting.ex +++ b/lib/pleroma/config/getting.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Config.Getting do def get(key), do: get(key, nil) def get(key, default), do: impl().get(key, default) - def impl() do + def impl do Application.get_env(:pleroma, :config_impl, Pleroma.Config) end end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index f4c405773..c6311e0c7 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -4,11 +4,11 @@ defmodule Pleroma.Search.DatabaseSearch do alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object.Fetcher alias Pleroma.Pagination alias Pleroma.User alias Pleroma.Web.ActivityPub.Visibility - alias Pleroma.Config require Pleroma.Constants -- cgit v1.2.3 From 9a063deacc75d3545dcd7a7eb33263ecbf0361ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 12 Nov 2023 13:38:08 +0000 Subject: Count and display post quotes --- lib/pleroma/object.ex | 46 +++++++++++++++ lib/pleroma/web/activity_pub/activity_pub.ex | 21 +++++++ .../object_validators/common_fields.ex | 1 + lib/pleroma/web/activity_pub/side_effects.ex | 8 +++ .../operations/pleroma_status_operation.ex | 45 +++++++++++++++ lib/pleroma/web/api_spec/schemas/status.ex | 7 ++- lib/pleroma/web/mastodon_api/views/status_view.ex | 3 +- .../pleroma_api/controllers/status_controller.ex | 66 ++++++++++++++++++++++ lib/pleroma/web/router.ex | 2 + 9 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex create mode 100644 lib/pleroma/web/pleroma_api/controllers/status_controller.ex (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index aa137d250..fa5baf1a4 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -328,6 +328,52 @@ defmodule Pleroma.Object do end end + def increase_quotes_count(ap_id) do + Object + |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id))) + |> update([o], + set: [ + data: + fragment( + """ + safe_jsonb_set(?, '{quotesCount}', + (coalesce((?->>'quotesCount')::int, 0) + 1)::varchar::jsonb, true) + """, + o.data, + o.data + ) + ] + ) + |> Repo.update_all([]) + |> case do + {1, [object]} -> set_cache(object) + _ -> {:error, "Not found"} + end + end + + def decrease_quotes_count(ap_id) do + Object + |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id))) + |> update([o], + set: [ + data: + fragment( + """ + safe_jsonb_set(?, '{quotesCount}', + (greatest(0, (?->>'quotesCount')::int - 1))::varchar::jsonb, true) + """, + o.data, + o.data + ) + ] + ) + |> Repo.update_all([]) + |> case do + {1, [object]} -> set_cache(object) + _ -> {:error, "Not found"} + end + end + def increase_vote_count(ap_id, name, actor) do with %Object{} = object <- Object.normalize(ap_id, fetch: false), "Question" <- object.data["type"] do diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3979d418e..a81d33fb6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -96,6 +96,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop + defp increase_quotes_count_if_quote(%{ + "object" => %{"quoteUrl" => quote_ap_id} = object, + "type" => "Create" + }) do + if is_public?(object) do + Object.increase_quotes_count(quote_ap_id) + end + end + + defp increase_quotes_count_if_quote(_create_data), do: :noop + @object_types ~w[ChatMessage Question Answer Audio Video Image Event Article Note Page] @impl true def persist(%{"type" => type} = object, meta) when type in @object_types do @@ -299,6 +310,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do with {:ok, activity} <- insert(create_data, local, fake), {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), + _ <- increase_quotes_count_if_quote(create_data), {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, {:ok, _actor} <- increase_note_count_if_public(actor, activity), {:ok, _actor} <- update_last_status_at_if_public(actor, activity), @@ -1237,6 +1249,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_unauthenticated(query, _), do: query + defp restrict_quote_url(query, %{quote_url: quote_url}) do + from([_activity, object] in query, + where: fragment("(?)->'quoteUrl' = ?", object.data, ^quote_url) + ) + end + + defp restrict_quote_url(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1399,6 +1419,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_instance(opts) |> restrict_announce_object_actor(opts) |> restrict_filtered(opts) + |> restrict_quote_url(opts) |> maybe_restrict_deactivated_users(opts) |> exclude_poll_votes(opts) |> exclude_chat_messages(opts) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 835ed97b7..1a5d02601 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -57,6 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) + field(:quotes_count, :integer, default: 0) field(:inReplyTo, ObjectValidators.ObjectID) field(:quoteUrl, ObjectValidators.ObjectID) field(:url, ObjectValidators.BareUri) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 098c177c7..6a7ac2806 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -209,6 +209,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.increase_replies_count(in_reply_to) end + if quote_url = object.data["quoteUrl"] do + Object.increase_quotes_count(quote_url) + end + reply_depth = (meta[:depth] || 0) + 1 # FIXME: Force inReplyTo to replies @@ -305,6 +309,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.decrease_replies_count(in_reply_to) end + if quote_url = deleted_object.data["quoteUrl"] do + Object.decrease_quotes_count(quote_url) + end + MessageReference.delete_for_object(deleted_object) ap_streamer().stream_out(object) diff --git a/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex new file mode 100644 index 000000000..6e69c5269 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaStatusOperation do + alias OpenApiSpex.Operation + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.StatusOperation + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def quotes_operation do + %Operation{ + tags: ["Retrieve status information"], + summary: "Quoted by", + description: "View quotes for a given status", + operationId: "PleromaAPI.StatusController.quotes", + parameters: [id_param() | pagination_params()], + security: [%{"oAuth" => ["read:statuses"]}], + responses: %{ + 200 => + Operation.response( + "Array of Status", + "application/json", + StatusOperation.array_of_statuses() + ), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def id_param do + Operation.parameter(:id, :path, FlakeID, "Status ID", + example: "9umDrYheeY451cQnEe", + required: true + ) + end +end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index 07f03134a..a4052803b 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -213,6 +213,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do type: :boolean, description: "`true` if the quoted post is visible to the user" }, + quotes_count: %Schema{ + type: :integer, + description: "How many statuses quoted this status" + }, local: %Schema{ type: :boolean, description: "`true` if the post was made on the local instance" @@ -367,7 +371,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do "in_reply_to_account_acct" => nil, "local" => true, "spoiler_text" => %{"text/plain" => ""}, - "thread_muted" => false + "thread_muted" => false, + "quotes_count" => 0 }, "poll" => nil, "reblog" => nil, diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d070262cc..e3b5760fa 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -447,7 +447,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do thread_muted: thread_muted?, emoji_reactions: emoji_reactions, parent_visible: visible_for_user?(reply_to, opts[:for]), - pinned_at: pinned_at + pinned_at: pinned_at, + quotes_count: object.data["quotesCount"] || 0 } } end diff --git a/lib/pleroma/web/pleroma_api/controllers/status_controller.ex b/lib/pleroma/web/pleroma_api/controllers/status_controller.ex new file mode 100644 index 000000000..482662fdd --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/status_controller.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.StatusController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + + require Ecto.Query + require Pleroma.Constants + + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + + plug( + OAuthScopesPlug, + %{scopes: ["read:statuses"], fallback: :proceed_unauthenticated} when action == :quotes + ) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaStatusOperation + + @doc "GET /api/v1/pleroma/statuses/:id/quotes" + def quotes(%{assigns: %{user: user}} = conn, %{id: id} = params) do + with %Activity{object: object} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + params = + params + |> Map.put(:type, "Create") + |> Map.put(:blocking_user, user) + |> Map.put(:quote_url, object.data["id"]) + + recipients = + if user do + [Pleroma.Constants.as_public()] ++ [user.ap_id | User.following(user)] + else + [Pleroma.Constants.as_public()] + end + + activities = + recipients + |> ActivityPub.fetch_activities(params) + |> Enum.reverse() + + conn + |> add_link_headers(activities) + |> put_view(StatusView) + |> render("index.json", + activities: activities, + for: user, + as: :activity + ) + else + nil -> {:error, :not_found} + false -> {:error, :not_found} + end + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..d40af499e 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -578,6 +578,8 @@ defmodule Pleroma.Web.Router do pipe_through(:api) get("/accounts/:id/favourites", AccountController, :favourites) get("/accounts/:id/endorsements", AccountController, :endorsements) + + get("/statuses/:id/quotes", StatusController, :quotes) end scope [] do -- cgit v1.2.3 From 66cb3294ed942d461cabc32881e2a10ebfd182af Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 2 Nov 2022 22:49:55 -0400 Subject: Switch to PromEx for prometheus metrics Recommending use of the separate HTTP server for exposing the metrics and securing it externally on your firewall or reverse proxy. It will listen on port 4021 by default. --- lib/pleroma/application.ex | 25 +------------------ lib/pleroma/config/transfer_task.ex | 3 +-- lib/pleroma/prom_ex.ex | 49 +++++++++++++++++++++++++++++++++++++ lib/pleroma/repo.ex | 2 -- lib/pleroma/web/endpoint.ex | 41 ------------------------------- 5 files changed, 51 insertions(+), 69 deletions(-) create mode 100644 lib/pleroma/prom_ex.ex (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7bbc132f1..52cd6e9a9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -54,7 +54,6 @@ defmodule Pleroma.Application do Config.DeprecationWarnings.warn() Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.ApplicationRequirements.verify!() - setup_instrumenters() load_custom_modules() Pleroma.Docs.JSON.compile() limiters_setup() @@ -91,6 +90,7 @@ defmodule Pleroma.Application do # Define workers and child supervisors to be supervised children = [ + Pleroma.PromEx, Pleroma.Repo, Config.TransferTask, Pleroma.Emoji, @@ -170,29 +170,6 @@ defmodule Pleroma.Application do end end - defp setup_instrumenters do - require Prometheus.Registry - - if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do - :ok = - :telemetry.attach( - "prometheus-ecto", - [:pleroma, :repo, :query], - &Pleroma.Repo.Instrumenter.handle_event/4, - %{} - ) - - Pleroma.Repo.Instrumenter.setup() - end - - Pleroma.Web.Endpoint.MetricsExporter.setup() - Pleroma.Web.Endpoint.PipelineInstrumenter.setup() - - # Note: disabled until prometheus-phx is integrated into prometheus-phoenix: - # Pleroma.Web.Endpoint.Instrumenter.setup() - PrometheusPhx.setup() - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 44a984019..6fd05b0e0 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -55,8 +55,7 @@ defmodule Pleroma.Config.TransferTask do started_applications = Application.started_applications() - # TODO: some problem with prometheus after restart! - reject = [nil, :prometheus, :postgrex] + reject = [nil, :postgrex] reject = if restart_pleroma? do diff --git a/lib/pleroma/prom_ex.ex b/lib/pleroma/prom_ex.ex new file mode 100644 index 000000000..6608708b7 --- /dev/null +++ b/lib/pleroma/prom_ex.ex @@ -0,0 +1,49 @@ +defmodule Pleroma.PromEx do + use PromEx, otp_app: :pleroma + + alias PromEx.Plugins + + @impl true + def plugins do + [ + # PromEx built in plugins + Plugins.Application, + Plugins.Beam, + {Plugins.Phoenix, router: Pleroma.Web.Router, endpoint: Pleroma.Web.Endpoint}, + Plugins.Ecto, + Plugins.Oban + # Plugins.PhoenixLiveView, + # Plugins.Absinthe, + # Plugins.Broadway, + + # Add your own PromEx metrics plugins + # Pleroma.Users.PromExPlugin + ] + end + + @impl true + def dashboard_assigns do + [ + datasource_id: Pleroma.Config.get([Pleroma.PromEx, :datasource]), + default_selected_interval: "30s" + ] + end + + @impl true + def dashboards do + [ + # PromEx built in Grafana dashboards + {:prom_ex, "application.json"}, + {:prom_ex, "beam.json"}, + {:prom_ex, "phoenix.json"}, + {:prom_ex, "ecto.json"}, + {:prom_ex, "oban.json"} + # {:prom_ex, "phoenix_live_view.json"}, + # {:prom_ex, "absinthe.json"}, + # {:prom_ex, "broadway.json"}, + + # Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"} + # {:pleroma, "/grafana_dashboards/user_metrics.json"} + ] + end +end diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 515b0c1ff..a50a59b3b 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -11,8 +11,6 @@ defmodule Pleroma.Repo do import Ecto.Query require Logger - defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter) - @doc """ Dynamically loads the repository url from the DATABASE_URL environment variable. diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 65dd72c49..307fa069e 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -151,47 +151,6 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Web.Plugs.RemoteIp) - defmodule Instrumenter do - use Prometheus.PhoenixInstrumenter - end - - defmodule PipelineInstrumenter do - use Prometheus.PlugPipelineInstrumenter - end - - defmodule MetricsExporter do - use Prometheus.PlugExporter - end - - defmodule MetricsExporterCaller do - @behaviour Plug - - def init(opts), do: opts - - def call(conn, opts) do - prometheus_config = Application.get_env(:prometheus, MetricsExporter, []) - ip_whitelist = List.wrap(prometheus_config[:ip_whitelist]) - - cond do - !prometheus_config[:enabled] -> - conn - - ip_whitelist != [] and - !Enum.find(ip_whitelist, fn ip -> - Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip} - end) -> - conn - - true -> - MetricsExporter.call(conn, opts) - end - end - end - - plug(PipelineInstrumenter) - - plug(MetricsExporterCaller) - plug(Pleroma.Web.Router) @doc """ -- cgit v1.2.3 From 13baba90f6deb27b0d6301a705db753cc32bb141 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 9 Nov 2022 13:45:57 -0500 Subject: Replace ImageMagick with Vips for Media Preview Proxy --- lib/pleroma/helpers/media_helper.ex | 57 +++++++++---------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index 24c845fcd..07dfea55b 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -8,11 +8,12 @@ defmodule Pleroma.Helpers.MediaHelper do """ alias Pleroma.HTTP + alias Vix.Vips.Operation require Logger def missing_dependencies do - Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> + Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> if Pleroma.Utils.command_available?(executable) do acc else @@ -22,54 +23,22 @@ defmodule Pleroma.Helpers.MediaHelper do end def image_resize(url, options) do - with executable when is_binary(executable) <- System.find_executable("convert"), - {:ok, args} <- prepare_image_resize_args(options), - {:ok, env} <- HTTP.get(url, [], pool: :media), - {:ok, fifo_path} <- mkfifo() do - args = List.flatten([fifo_path, args]) - run_fifo(fifo_path, env, executable, args) + with {:ok, env} <- HTTP.get(url, [], pool: :media), + {:ok, resized} <- + Operation.thumbnail_buffer(env.body, options.max_width, + height: options.max_height, + size: :VIPS_SIZE_DOWN + ) do + if options[:format] == "png" do + Operation.pngsave_buffer(resized, Q: options[:quality]) + else + Operation.jpegsave_buffer(resized, Q: options[:quality], interlace: true) + end else - nil -> {:error, {:convert, :command_not_found}} {:error, _} = error -> error end end - defp prepare_image_resize_args( - %{max_width: max_width, max_height: max_height, format: "png"} = options - ) do - quality = options[:quality] || 85 - resize = Enum.join([max_width, "x", max_height, ">"]) - - args = [ - "-resize", - resize, - "-quality", - to_string(quality), - "png:-" - ] - - {:ok, args} - end - - defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do - quality = options[:quality] || 85 - resize = Enum.join([max_width, "x", max_height, ">"]) - - args = [ - "-interlace", - "Plane", - "-resize", - resize, - "-quality", - to_string(quality), - "jpg:-" - ] - - {:ok, args} - end - - defp prepare_image_resize_args(_), do: {:error, :missing_options} - # Note: video thumbnail is intentionally not resized (always has original dimensions) def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), -- cgit v1.2.3 From 481b6ac0d5c610e840fc4b88a357efa28f510ab8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 10 Nov 2022 11:07:49 -0500 Subject: Add Pleroma.Upload.Filter.HeifToJpeg based on vips --- lib/pleroma/upload/filter/heif_to_jpeg.ex | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 lib/pleroma/upload/filter/heif_to_jpeg.ex (limited to 'lib') diff --git a/lib/pleroma/upload/filter/heif_to_jpeg.ex b/lib/pleroma/upload/filter/heif_to_jpeg.ex new file mode 100644 index 000000000..a2095ba01 --- /dev/null +++ b/lib/pleroma/upload/filter/heif_to_jpeg.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.HeifToJpeg do + @behaviour Pleroma.Upload.Filter + alias Pleroma.Upload + alias Vix.Vips.Operation + + @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} + @type conversions :: conversion() | [conversion()] + + @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} + def filter(%Pleroma.Upload{content_type: "image/avif"} = upload), do: apply_filter(upload) + def filter(%Pleroma.Upload{content_type: "image/heic"} = upload), do: apply_filter(upload) + def filter(%Pleroma.Upload{content_type: "image/heif"} = upload), do: apply_filter(upload) + + def filter(_), do: {:ok, :noop} + + defp apply_filter(%Pleroma.Upload{name: name, path: path, tempfile: tempfile} = upload) do + ext = String.split(path, ".") |> List.last() + + try do + name = name |> String.replace_suffix(ext, "jpg") + path = path |> String.replace_suffix(ext, "jpg") + {:ok, {vixdata, _vixflags}} = Operation.heifload(tempfile) + {:ok, jpegdata} = Operation.jpegsave_buffer(vixdata) + :ok = File.write(tempfile, jpegdata) + + {:ok, :filtered, %Upload{upload | name: name, path: path, content_type: "image/jpeg"}} + rescue + e in ErlangError -> + {:error, "#{__MODULE__}: #{inspect(e)}"} + end + end +end -- cgit v1.2.3 From a4b6e5613fe02fac1a0ec80b8fd6940d2b22f85d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 Nov 2023 22:03:20 +0000 Subject: Revert "Add Pleroma.Upload.Filter.HeifToJpeg based on vips" This reverts commit 31d4448ee61b4afac6aa23f8c0287d13aed411a1. This functionality is not reliably working with vips/vix due to codec patent junk --- lib/pleroma/upload/filter/heif_to_jpeg.ex | 36 ------------------------------- 1 file changed, 36 deletions(-) delete mode 100644 lib/pleroma/upload/filter/heif_to_jpeg.ex (limited to 'lib') diff --git a/lib/pleroma/upload/filter/heif_to_jpeg.ex b/lib/pleroma/upload/filter/heif_to_jpeg.ex deleted file mode 100644 index a2095ba01..000000000 --- a/lib/pleroma/upload/filter/heif_to_jpeg.ex +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Upload.Filter.HeifToJpeg do - @behaviour Pleroma.Upload.Filter - alias Pleroma.Upload - alias Vix.Vips.Operation - - @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} - @type conversions :: conversion() | [conversion()] - - @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} - def filter(%Pleroma.Upload{content_type: "image/avif"} = upload), do: apply_filter(upload) - def filter(%Pleroma.Upload{content_type: "image/heic"} = upload), do: apply_filter(upload) - def filter(%Pleroma.Upload{content_type: "image/heif"} = upload), do: apply_filter(upload) - - def filter(_), do: {:ok, :noop} - - defp apply_filter(%Pleroma.Upload{name: name, path: path, tempfile: tempfile} = upload) do - ext = String.split(path, ".") |> List.last() - - try do - name = name |> String.replace_suffix(ext, "jpg") - path = path |> String.replace_suffix(ext, "jpg") - {:ok, {vixdata, _vixflags}} = Operation.heifload(tempfile) - {:ok, jpegdata} = Operation.jpegsave_buffer(vixdata) - :ok = File.write(tempfile, jpegdata) - - {:ok, :filtered, %Upload{upload | name: name, path: path, content_type: "image/jpeg"}} - rescue - e in ErlangError -> - {:error, "#{__MODULE__}: #{inspect(e)}"} - end - end -end -- cgit v1.2.3 From 66f5ae0c5a0ab57dc6bf3f52bbf976128259800f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 8 Aug 2023 19:08:59 +0200 Subject: router: Make /federation_status publicly available --- lib/pleroma/web/router.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9abad65b0..eb8576b02 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -224,6 +224,12 @@ defmodule Pleroma.Web.Router do post("/remote_interaction", UtilController, :remote_interaction) end + scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do + pipe_through(:pleroma_api) + + get("/federation_status", InstancesController, :show) + end + scope "/api/v1/pleroma", Pleroma.Web do pipe_through(:pleroma_api) post("/uploader_callback/:upload_path", UploaderController, :callback) @@ -604,7 +610,6 @@ defmodule Pleroma.Web.Router do scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do pipe_through(:api) get("/accounts/:id/scrobbles", ScrobbleController, :index) - get("/federation_status", InstancesController, :show) end scope "/api/v2/pleroma", Pleroma.Web.PleromaAPI do -- cgit v1.2.3 From a5aa8ea79603e22541de04c26293dc87bd2f2ed8 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Sun, 29 Oct 2023 18:58:57 +0200 Subject: Add support for configuring a favicon and embed PWA manifest in server-generated-meta --- lib/pleroma/web/fallback/redirect_controller.ex | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 1a86f7a53..005a5da8b 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -18,9 +18,22 @@ defmodule Pleroma.Web.Fallback.RedirectController do end def redirector(conn, _params, code \\ 200) do + {:ok, index_content} = File.read(index_file_path()) + + title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" + + response = + index_content + |> String.replace( + "", + title <> favicon <> manifest + ) + conn |> put_resp_content_type("text/html") - |> send_file(code, index_file_path()) + |> send_resp(code, response) end def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} = params) do @@ -38,10 +51,15 @@ defmodule Pleroma.Web.Fallback.RedirectController do tags = build_tags(conn, params) preloads = preload_data(conn, params) title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" response = index_content - |> String.replace("", tags <> preloads <> title) + |> String.replace( + "", + tags <> preloads <> title <> favicon <> manifest + ) conn |> put_resp_content_type("text/html") @@ -56,10 +74,12 @@ defmodule Pleroma.Web.Fallback.RedirectController do {:ok, index_content} = File.read(index_file_path()) preloads = preload_data(conn, params) title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" response = index_content - |> String.replace("", preloads <> title) + |> String.replace("", preloads <> title <> favicon <> manifest) conn |> put_resp_content_type("text/html") -- cgit v1.2.3 From 5d3e145dc4090dc5709a649ee5d646b637da252e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 14 Nov 2023 11:01:02 +0100 Subject: RedirectController: Unify server-generated-meta insertion code --- lib/pleroma/web/fallback/redirect_controller.ex | 33 +++++++++++-------------- 1 file changed, 14 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 005a5da8b..4a0885fab 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -17,19 +17,24 @@ defmodule Pleroma.Web.Fallback.RedirectController do |> json(%{error: "Not implemented"}) end - def redirector(conn, _params, code \\ 200) do - {:ok, index_content} = File.read(index_file_path()) - + def add_generated_metadata(page_content, extra \\ "") do title = "#{Pleroma.Config.get([:instance, :name])}" favicon = "" manifest = "" + page_content + |> String.replace( + "", + title <> favicon <> manifest <> extra + ) + end + + def redirector(conn, _params, code \\ 200) do + {:ok, index_content} = File.read(index_file_path()) + response = index_content - |> String.replace( - "", - title <> favicon <> manifest - ) + |> add_generated_metadata() conn |> put_resp_content_type("text/html") @@ -47,19 +52,12 @@ defmodule Pleroma.Web.Fallback.RedirectController do def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = build_tags(conn, params) preloads = preload_data(conn, params) - title = "#{Pleroma.Config.get([:instance, :name])}" - favicon = "" - manifest = "" response = index_content - |> String.replace( - "", - tags <> preloads <> title <> favicon <> manifest - ) + |> add_generated_metadata(tags <> preloads) conn |> put_resp_content_type("text/html") @@ -73,13 +71,10 @@ defmodule Pleroma.Web.Fallback.RedirectController do def redirector_with_preload(conn, params) do {:ok, index_content} = File.read(index_file_path()) preloads = preload_data(conn, params) - title = "#{Pleroma.Config.get([:instance, :name])}" - favicon = "" - manifest = "" response = index_content - |> String.replace("", preloads <> title <> favicon <> manifest) + |> add_generated_metadata(preloads) conn |> put_resp_content_type("text/html") -- cgit v1.2.3 From 8ac7cc98c1ba241ce104cc2ee15723f087c367d2 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 16 Nov 2023 15:48:37 +0100 Subject: MastoAPI AccountView: Change last_status_at to be a date Changed in Mastodon 3.1.0 with: https://github.com/tootsuite/mastodon/pull/12966 --- lib/pleroma/web/mastodon_api/views/account_view.ex | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index cc3e3582f..34478cec8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors +# Copyright © 2017-2023 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.AccountView do @@ -249,6 +249,13 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do nil end + last_status_at = + if not is_nil(user.last_status_at) do + user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601() + else + nil + end + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -277,7 +284,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do actor_type: user.actor_type } }, - last_status_at: user.last_status_at, + last_status_at: last_status_at, # Pleroma extensions # Note: it's insecure to output :email but fully-qualified nickname may serve as safe stub -- cgit v1.2.3 From 0c6a54b37cb208e577fc9e40782bb8b820730428 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 Nov 2023 17:04:47 -0500 Subject: Upload.Filter.AnalyzeMetadata: Blurhash with a Rust NIF, and use Vix to retrieve image metadata --- lib/pleroma/upload/filter/analyze_metadata.ex | 36 +++++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 9a76a998b..15dec5564 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -8,22 +8,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do """ require Logger + alias Vix.Vips.Image + alias Vix.Vips.Operation + @behaviour Pleroma.Upload.Filter @spec filter(Pleroma.Upload.t()) :: {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do try do - image = - file - |> Mogrify.open() - |> Mogrify.verbose() + {:ok, image} = Image.new_from_file(file) + {width, height} = {Image.width(image), Image.height(image)} upload = upload - |> Map.put(:width, image.width) - |> Map.put(:height, image.height) - |> Map.put(:blurhash, get_blurhash(file)) + |> Map.put(:width, width) + |> Map.put(:height, height) + |> Map.put(:blurhash, get_blurhash(image)) {:ok, :filtered, upload} rescue @@ -53,7 +54,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do def filter(_), do: {:ok, :noop} defp get_blurhash(file) do - with {:ok, blurhash} <- :eblurhash.magick(file) do + with {:ok, blurhash} <- vips_blurhash(file) do blurhash else _ -> nil @@ -80,4 +81,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {:error, _} = error -> error end end + + defp vips_blurhash(image = %Vix.Vips.Image{}) do + {:ok, resized_image} = Operation.thumbnail_image(image, 20) + {height, width} = {Image.height(resized_image), Image.width(resized_image)} + max = max(height, width) + {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} + + {:ok, rgba} = + if Image.has_alpha?(resized_image) do + Image.to_list(resized_image) + else + Operation.bandjoin_const!(resized_image, [255]) + |> Image.to_list() + end + + rgba = List.flatten(rgba) + + Blurhash.encode(x, y, width, height, rgba) + end end -- cgit v1.2.3 From 88cc7e6a0431385b5fce292179c7eb05e64e3a24 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 17 Nov 2023 11:06:31 -0500 Subject: Resize images to 100 pixels before hashing --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 15dec5564..99d1ec5bc 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end defp vips_blurhash(image = %Vix.Vips.Image{}) do - {:ok, resized_image} = Operation.thumbnail_image(image, 20) + {:ok, resized_image} = Operation.thumbnail_image(image, 100) {height, width} = {Image.height(resized_image), Image.width(resized_image)} max = max(height, width) {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} -- cgit v1.2.3 From 510a7b64f1354d4313ab565d557f422b7e059432 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Wed, 15 Nov 2023 00:43:58 -0700 Subject: Add optional URL value for scrobbles --- lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex | 6 +++++- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/pleroma_api/views/scrobble_view.ex | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index ca40da930..68c586b73 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -59,6 +59,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, length: %Schema{type: :integer, description: "The length of the media playing"}, + url: %Schema{type: :string, description: "A URL referencing the media playing"}, visibility: %Schema{ allOf: [VisibilityScope], default: "public", @@ -69,7 +70,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "title" => "Some Title", "artist" => "Some Artist", "album" => "Some Album", - "length" => 180_000 + "length" => 180_000, + "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title" } } end @@ -83,6 +85,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do title: %Schema{type: :string, description: "The title of the media playing"}, album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, + url: %Schema{type: :string, description: "A URL referencing the media playing"}, length: %Schema{ type: :integer, description: "The length of the media playing", @@ -97,6 +100,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, + "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at" => "2019-09-28T12:40:45.000Z" } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index ca1329284..00cbacbb4 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp listen_object(draft) do object = draft.params - |> Map.take([:album, :artist, :title, :length]) + |> Map.take([:album, :artist, :title, :length, :url]) |> Map.new(fn {key, value} -> {to_string(key), value} end) |> Map.put("type", "Audio") |> Map.put("to", draft.to) diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index a5985fb2a..7a983f8b5 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -27,6 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do title: object.data["title"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(), album: object.data["album"] |> HTML.strip_tags(), + url: object.data["url"], length: object.data["length"] } end -- cgit v1.2.3 From 299c548b124377e51f6c089bc0df31b2989be3ef Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 23 Nov 2023 16:15:53 -0500 Subject: Prevent a blurhash failure from breaking all metadata collection --- lib/pleroma/upload/filter/analyze_metadata.ex | 34 ++++++++++++++------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 99d1ec5bc..92b80b1b1 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -83,21 +83,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end defp vips_blurhash(image = %Vix.Vips.Image{}) do - {:ok, resized_image} = Operation.thumbnail_image(image, 100) - {height, width} = {Image.height(resized_image), Image.width(resized_image)} - max = max(height, width) - {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} - - {:ok, rgba} = - if Image.has_alpha?(resized_image) do - Image.to_list(resized_image) - else - Operation.bandjoin_const!(resized_image, [255]) - |> Image.to_list() - end - - rgba = List.flatten(rgba) - - Blurhash.encode(x, y, width, height, rgba) + with {:ok, resized_image} <- Operation.thumbnail_image(image, 100), + {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, + max <- max(height, width), + {x, y} <- {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} do + {:ok, rgba} = + if Image.has_alpha?(resized_image) do + Image.to_list(resized_image) + else + Operation.bandjoin_const!(resized_image, [255]) + |> Image.to_list() + end + + rgba = List.flatten(rgba) + + Blurhash.encode(x, y, width, height, rgba) + else + _ -> nil + end end end -- cgit v1.2.3 From 27df2c0ce6c214f36db742af702fc239f80764a7 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Mon, 27 Nov 2023 03:34:31 -0700 Subject: Fix #strip_report_status_data --- lib/pleroma/web/activity_pub/utils.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 437220077..073ccd615 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -855,6 +855,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do Enum.map(reported_activities, fn act when is_map(act) -> act["id"] act when is_binary(act) -> act + _other -> nil end) new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) -- cgit v1.2.3 From 4ef56c5b65a4d1e7e90a87f1a1a507df523a4b4b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 Nov 2023 18:44:11 +0400 Subject: ActivityPub.Utils: Only treat object ids as valid while stripping --- lib/pleroma/web/activity_pub/utils.ex | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 073ccd615..b32f19740 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do alias Ecto.UUID alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -852,10 +853,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do [actor | reported_activities] = activity.data["object"] stripped_activities = - Enum.map(reported_activities, fn - act when is_map(act) -> act["id"] - act when is_binary(act) -> act - _other -> nil + Enum.reduce(reported_activities, [], fn act, acc -> + case ObjectID.cast(act) do + {:ok, act} -> [act | acc] + _ -> acc + end end) new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) -- cgit v1.2.3 From 03db495e1d88f34bef8d556b0f88806c3260d403 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 12:23:41 +0400 Subject: AnalyzeMetadata: Switch to rinpatch_blurhash --- lib/pleroma/upload/filter/analyze_metadata.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 92b80b1b1..710fd02d2 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -87,17 +87,17 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, max <- max(height, width), {x, y} <- {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} do - {:ok, rgba} = + {:ok, rgb} = if Image.has_alpha?(resized_image) do - Image.to_list(resized_image) + # remove alpha channel + resized_image + |> Operation.extract_band!(0, n: 3) + |> Image.write_to_binary() else - Operation.bandjoin_const!(resized_image, [255]) - |> Image.to_list() + Image.write_to_binary(resized_image) end - rgba = List.flatten(rgba) - - Blurhash.encode(x, y, width, height, rgba) + Blurhash.encode(rgb, width, height, x, y) else _ -> nil end -- cgit v1.2.3 From b3214be32f83541626b198b86ee065b92862f4cf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 12:33:54 +0400 Subject: AnayzeMetadata: Fix error case that would never match --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 710fd02d2..ef75d73d3 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -78,7 +78,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do %{width: width, height: height} else nil -> {:error, {:ffprobe, :command_not_found}} - {:error, _} = error -> error + error -> {:error, error} end end -- cgit v1.2.3 From ccc2adee4111367d67646f6f2828e03b861dd393 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 13:13:43 +0400 Subject: Linting --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index ef75d73d3..e510ae3e6 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -82,7 +82,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end end - defp vips_blurhash(image = %Vix.Vips.Image{}) do + defp vips_blurhash(%Vix.Vips.Image{} = image) do with {:ok, resized_image} <- Operation.thumbnail_image(image, 100), {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, max <- max(height, width), -- cgit v1.2.3 From e216603477e2c393a586f7eb0bc8183e73bf4cd7 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Wed, 29 Nov 2023 07:55:44 -0700 Subject: Change url to externalLink as requested by hj here: https://shigusegubu.club/notice/AcIjZjackKAt6e522a --- lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex | 8 ++++---- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/pleroma_api/views/scrobble_view.ex | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index 68c586b73..141b60533 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, length: %Schema{type: :integer, description: "The length of the media playing"}, - url: %Schema{type: :string, description: "A URL referencing the media playing"}, + externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, visibility: %Schema{ allOf: [VisibilityScope], default: "public", @@ -71,7 +71,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, - "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title" + "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title" } } end @@ -85,7 +85,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do title: %Schema{type: :string, description: "The title of the media playing"}, album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, - url: %Schema{type: :string, description: "A URL referencing the media playing"}, + externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, length: %Schema{ type: :integer, description: "The length of the media playing", @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, - "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title", + "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at" => "2019-09-28T12:40:45.000Z" } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 00cbacbb4..8910ad5b8 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp listen_object(draft) do object = draft.params - |> Map.take([:album, :artist, :title, :length, :url]) + |> Map.take([:album, :artist, :title, :length, :externalLink]) |> Map.new(fn {key, value} -> {to_string(key), value} end) |> Map.put("type", "Audio") |> Map.put("to", draft.to) diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index 7a983f8b5..edf0a2390 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do title: object.data["title"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(), album: object.data["album"] |> HTML.strip_tags(), - url: object.data["url"], + externalLink: object.data["externalLink"], length: object.data["length"] } end -- cgit v1.2.3 From 15a8acbd6cc6ecc89b49956acfeba49d02270c1d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 30 Nov 2023 09:40:17 +0400 Subject: MRF, Docs.Generator: Ensure code is loaded before checking it --- lib/pleroma/docs/generator.ex | 2 ++ lib/pleroma/web/activity_pub/mrf.ex | 3 +++ 2 files changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 6508f1947..456a8fd54 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Docs.Generator do # This shouldn't be needed as all modules are expected to have module_info/1, # but in test enviroments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). + Code.ensure_loaded(module) + if function_exported?(module, :module_info, 1) do module.module_info(:attributes) |> Keyword.get_values(:behaviour) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index ff9f84497..a0deac300 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -54,6 +54,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do @required_description_keys [:key, :related_policy] def filter_one(policy, message) do + Code.ensure_loaded(policy) + should_plug_history? = if function_exported?(policy, :history_awareness, 0) do policy.history_awareness() @@ -188,6 +190,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do def config_descriptions(policies) do Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc -> + Code.ensure_loaded(policy) if function_exported?(policy, :config_description, 0) do description = @default_description -- cgit v1.2.3 From eb6be30602f95cf86aef465229dacbdcb92c3c58 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 30 Nov 2023 10:05:00 +0400 Subject: Linting --- lib/pleroma/web/activity_pub/mrf.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index a0deac300..69054e25b 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -191,6 +191,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do def config_descriptions(policies) do Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc -> Code.ensure_loaded(policy) + if function_exported?(policy, :config_description, 0) do description = @default_description -- cgit v1.2.3 From 1ad0d94d6f1f11c848f277877cc14b3a47a7ae95 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 5 Dec 2023 16:35:41 -0500 Subject: Change set_reachable/1 to an upsert --- lib/pleroma/instances/instance.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 9756c66dc..101e5dc88 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -97,13 +97,9 @@ defmodule Pleroma.Instances.Instance do def reachable?(url_or_host) when is_binary(url_or_host), do: true def set_reachable(url_or_host) when is_binary(url_or_host) do - with host <- host(url_or_host), - %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do - {:ok, _instance} = - existing_record - |> changeset(%{unreachable_since: nil}) - |> Repo.update() - end + %Instance{host: host(url_or_host)} + |> changeset(%{unreachable_since: nil}) + |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host) end def set_reachable(_), do: {:error, nil} -- cgit v1.2.3 From ef8a2134bc9e2e9f3aa08172b982c347332e14c1 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 7 Dec 2023 11:25:18 +0400 Subject: AccountView: Add test, refactor --- lib/pleroma/web/mastodon_api/views/account_view.ex | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 34478cec8..237de3055 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -250,11 +250,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end last_status_at = - if not is_nil(user.last_status_at) do + user.last_status_at && user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601() - else - nil - end %{ id: to_string(user.id), -- cgit v1.2.3 From b70ca7d54e052abe0cb572e37e1af9c7caa815db Mon Sep 17 00:00:00 2001 From: kPherox Date: Thu, 7 Dec 2023 09:53:24 +0000 Subject: fix: AnalyzeMetadata filter no longer depends on ImageMagick's commands --- lib/pleroma/application_requirements.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 44b1c1705..477eac35b 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -168,8 +168,6 @@ defmodule Pleroma.ApplicationRequirements do check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"), check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"), check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"), - check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"), - check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert"), check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe") ] -- cgit v1.2.3 From 074b31d9ab30bbecba3fff4335dfec39af5cf9b8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 7 Dec 2023 22:27:19 -0500 Subject: Optimistic Inbox Rework inbound federation to accept requests optimistically. The HTTP Signatures Plug will not attempt to fetch the actor or key and will fail early. If the signature cannot be validated we pass the required data into the Oban job with a reduced priority and increase the timeout to 20 seconds. The Oban job will handle the actor and key fetching before attempting to validate the activity again. This job will be retried 5 times by default. Another welcome side effect is that actors who change their keys can federate to Pleroma instances immediately instead of needing to wait the default value of 86400s / 24 hours before the key will be fetched again. --- lib/pleroma/signature.ex | 2 +- lib/pleroma/user.ex | 11 +++++++- .../web/activity_pub/activity_pub_controller.ex | 7 ++--- lib/pleroma/web/federator.ex | 7 +++++ lib/pleroma/workers/receiver_worker.ex | 33 ++++++++++++++++++++-- 5 files changed, 51 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 5cfdae051..42cceba28 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Signature do with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), {:ok, actor_id} <- key_id_to_actor_id(kid), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), - {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do + {:ok, public_key} <- User.get_or_fetch_public_key_for_ap_id(actor_id) do {:ok, public_key} else e -> diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ce125d608..4c9aeffcb 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2135,7 +2135,7 @@ defmodule Pleroma.User do def public_key(_), do: {:error, "key not found"} - def get_public_key_for_ap_id(ap_id) do + def get_or_fetch_public_key_for_ap_id(ap_id) do with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id), {:ok, public_key} <- public_key(user) do {:ok, public_key} @@ -2144,6 +2144,15 @@ defmodule Pleroma.User do end end + def get_public_key_for_ap_id(ap_id) do + with {:ok, %User{} = user} <- get_cached_by_ap_id(ap_id), + {:ok, public_key} <- public_key(user) do + {:ok, public_key} + else + _ -> :error + end + end + @doc "Gets or fetch a user by uri or nickname." @spec get_or_fetch(String.t()) :: {:ok, User.t()} | {:error, String.t()} def get_or_fetch("http://" <> _host = uri), do: get_or_fetch_by_ap_id(uri) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 1357c379c..3b2193ca3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -287,10 +287,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do json(conn, "ok") end - def inbox(%{assigns: %{valid_signature: false}} = conn, _params) do - conn - |> put_status(:bad_request) - |> json("Invalid HTTP Signature") + def inbox(%{assigns: %{valid_signature: false}, req_headers: req_headers} = conn, params) do + Federator.incoming_ap_doc(%{req_headers: req_headers, params: params}) + json(conn, "ok") end # POST /relay/inbox -or- POST /internal/fetch/inbox diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 84b77cda1..1b8b22b7e 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -35,6 +35,13 @@ defmodule Pleroma.Web.Federator do end # Client API + def incoming_ap_doc(%{params: params, req_headers: req_headers}) do + ReceiverWorker.enqueue( + "incoming_ap_doc", + %{"req_headers" => req_headers, "params" => params, "timeout" => :timer.seconds(20)}, + priority: 5 + ) + end def incoming_ap_doc(params) do ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index cf1bb62b4..b04e2974a 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -3,24 +3,51 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ReceiverWorker do + alias Pleroma.Signature + alias Pleroma.User alias Pleroma.Web.Federator use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" @impl Oban.Worker + + def perform(%Job{ + args: %{"op" => "incoming_ap_doc", "req_headers" => req_headers, "params" => params} + }) do + conn_data = %{params: params, req_headers: req_headers} + + with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), + {:ok, _public_key} <- Signature.refetch_public_key(conn_data), + {:signature, true} <- {:signature, HTTPSignatures.validate_conn(conn_data)}, + {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do + {:ok, res} + else + e -> process_errors(e) + end + end + def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do {:ok, res} else + e -> process_errors(e) + end + end + + @impl Oban.Worker + def timeout(%_{args: %{"timeout" => timeout}}), do: timeout + + def timeout(_job), do: :timer.seconds(5) + + defp process_errors(errors) do + case errors do {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} {:error, :already_present} -> {:cancel, :already_present} {:error, {:validate_object, reason}} -> {:cancel, reason} {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} + {:signature, false} -> {:error, :invalid_signature} e -> e end end - - @impl Oban.Worker - def timeout(_job), do: :timer.seconds(5) end -- cgit v1.2.3 From ce5acd4158a351c59a79a4a0692eb2ef9775b554 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 8 Dec 2023 18:10:22 -0500 Subject: get_cached_by_ap_id/1 returns a single result, not a tuple --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 4c9aeffcb..3f729fdcc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2145,7 +2145,7 @@ defmodule Pleroma.User do end def get_public_key_for_ap_id(ap_id) do - with {:ok, %User{} = user} <- get_cached_by_ap_id(ap_id), + with %User{} = user <- get_cached_by_ap_id(ap_id), {:ok, public_key} <- public_key(user) do {:ok, public_key} else -- cgit v1.2.3 From 97cf78f63d312d0475ac8908d0b093cb5eff18d5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 8 Dec 2023 18:24:30 -0500 Subject: Remove unnecessary forced refresh of user --- lib/pleroma/user.ex | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3f729fdcc..c7753ca5d 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1037,16 +1037,6 @@ defmodule Pleroma.User do defp maybe_send_registration_email(_), do: {:ok, :noop} - def needs_update?(%User{local: true}), do: false - - def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true - - def needs_update?(%User{local: false} = user) do - NaiveDateTime.diff(NaiveDateTime.utc_now(), user.last_refreshed_at) >= 86_400 - end - - def needs_update?(_), do: true - @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()} # "Locked" (self-locked) users demand explicit authorization of follow requests @@ -2059,15 +2049,13 @@ defmodule Pleroma.User do def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) def get_or_fetch_by_ap_id(ap_id) do - cached_user = get_cached_by_ap_id(ap_id) - - maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id) + user = get_cached_by_ap_id(ap_id) || fetch_by_ap_id(ap_id) - case {cached_user, maybe_fetched_user} do - {_, {:ok, %User{} = user}} -> + case user do + %User{} = user -> {:ok, user} - {%User{} = user, _} -> + {:ok, %User{} = user} -> {:ok, user} _ -> -- cgit v1.2.3 From 1d417d2a364eb8ae2035275927f6284951fe128b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 8 Dec 2023 21:49:25 -0500 Subject: Our version of Oban only supports priorities 0-3 --- lib/pleroma/web/federator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 1b8b22b7e..669a5bb0b 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Web.Federator do ReceiverWorker.enqueue( "incoming_ap_doc", %{"req_headers" => req_headers, "params" => params, "timeout" => :timer.seconds(20)}, - priority: 5 + priority: 2 ) end -- cgit v1.2.3 From 4039106500ed0adba3ac6dbd480954aa76ce6bc3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 8 Dec 2023 21:51:36 -0500 Subject: Fix the req_headers formatting --- lib/pleroma/workers/receiver_worker.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index b04e2974a..562112dd2 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -14,6 +14,10 @@ defmodule Pleroma.Workers.ReceiverWorker do def perform(%Job{ args: %{"op" => "incoming_ap_doc", "req_headers" => req_headers, "params" => params} }) do + # Oban's serialization converts our tuple headers to lists. + # Revert it for the signature validation. + req_headers = Enum.into(req_headers, [], &List.to_tuple(&1)) + conn_data = %{params: params, req_headers: req_headers} with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), -- cgit v1.2.3 From 82724f6664c05e746eb241fb60aefd9a931b372d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 17:43:54 -0500 Subject: Do not retry fetching deleted objects --- lib/pleroma/workers/receiver_worker.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 562112dd2..d4e8e715e 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -51,6 +51,7 @@ defmodule Pleroma.Workers.ReceiverWorker do {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} {:signature, false} -> {:error, :invalid_signature} + {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason} e -> e end end -- cgit v1.2.3 From 94daa3e8c1ec42f40214dc13f061f4319764f715 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 18:21:00 -0500 Subject: Revert "Remove unnecessary forced refresh of user" This reverts commit 97cf78f63d312d0475ac8908d0b093cb5eff18d5. --- lib/pleroma/user.ex | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index c7753ca5d..3f729fdcc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1037,6 +1037,16 @@ defmodule Pleroma.User do defp maybe_send_registration_email(_), do: {:ok, :noop} + def needs_update?(%User{local: true}), do: false + + def needs_update?(%User{local: false, last_refreshed_at: nil}), do: true + + def needs_update?(%User{local: false} = user) do + NaiveDateTime.diff(NaiveDateTime.utc_now(), user.last_refreshed_at) >= 86_400 + end + + def needs_update?(_), do: true + @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()} # "Locked" (self-locked) users demand explicit authorization of follow requests @@ -2049,13 +2059,15 @@ defmodule Pleroma.User do def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) def get_or_fetch_by_ap_id(ap_id) do - user = get_cached_by_ap_id(ap_id) || fetch_by_ap_id(ap_id) + cached_user = get_cached_by_ap_id(ap_id) + + maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id) - case user do - %User{} = user -> + case {cached_user, maybe_fetched_user} do + {_, {:ok, %User{} = user}} -> {:ok, user} - {:ok, %User{} = user} -> + {%User{} = user, _} -> {:ok, user} _ -> -- cgit v1.2.3 From d417f7321801890381de4daa53c1a78ef4650d2c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 9 Dec 2023 18:40:29 -0500 Subject: Process inbound Delete activities at lowest priority --- lib/pleroma/web/federator.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 669a5bb0b..8621d984c 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -43,6 +43,10 @@ defmodule Pleroma.Web.Federator do ) end + def incoming_ap_doc(%{"type" => "Delete"} = params) do + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) + end + def incoming_ap_doc(params) do ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) end -- cgit v1.2.3 From c068a218eac02044cb5823abb1917051e1815b4f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 10 Dec 2023 18:57:46 +0400 Subject: Backup Tests: Split out async tests, use mox. --- lib/pleroma/user/backup.ex | 60 ++++++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 447fca2a1..c7359313a 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -35,6 +35,8 @@ defmodule Pleroma.User.Backup do timestamps() end + @config_impl Application.compile_env(__MODULE__, :config_impl, Pleroma.Config) + def create(user, admin_id \\ nil) do with :ok <- validate_limit(user, admin_id), {:ok, backup} <- user |> new() |> Repo.insert() do @@ -124,7 +126,10 @@ defmodule Pleroma.User.Backup do |> Repo.update() end - def process(%__MODULE__{} = backup) do + def process( + %__MODULE__{} = backup, + processor_module \\ __MODULE__.Processor + ) do set_state(backup, :running, 0) current_pid = self() @@ -132,7 +137,7 @@ defmodule Pleroma.User.Backup do task = Task.Supervisor.async_nolink( Pleroma.TaskSupervisor, - __MODULE__, + processor_module, :do_process, [backup, current_pid] ) @@ -140,25 +145,8 @@ defmodule Pleroma.User.Backup do wait_backup(backup, backup.processed_number, task) end - def do_process(backup, current_pid) do - with {:ok, zip_file} <- export(backup, current_pid), - {:ok, %{size: size}} <- File.stat(zip_file), - {:ok, _upload} <- upload(backup, zip_file) do - backup - |> cast( - %{ - file_size: size, - processed: true, - state: :complete - }, - [:file_size, :processed, :state] - ) - |> Repo.update() - end - end - defp wait_backup(backup, current_processed, task) do - wait_time = Pleroma.Config.get([__MODULE__, :process_wait_time]) + wait_time = @config_impl.get([__MODULE__, :process_wait_time]) receive do {:progress, new_processed} -> @@ -365,3 +353,35 @@ defmodule Pleroma.User.Backup do ) end end + +defmodule Pleroma.User.Backup.ProcessorAPI do + @callback do_process(%Pleroma.User.Backup{}, pid()) :: + {:ok, %Pleroma.User.Backup{}} | {:error, any()} +end + +defmodule Pleroma.User.Backup.Processor do + @behaviour Pleroma.User.Backup.ProcessorAPI + + alias Pleroma.Repo + alias Pleroma.User.Backup + + import Ecto.Changeset + + @impl true + def do_process(backup, current_pid) do + with {:ok, zip_file} <- Backup.export(backup, current_pid), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- Backup.upload(backup, zip_file) do + backup + |> cast( + %{ + file_size: size, + processed: true, + state: :complete + }, + [:file_size, :processed, :state] + ) + |> Repo.update() + end + end +end -- cgit v1.2.3 From 06fc196772a967a44593a10ceaf00396ce6b8de3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 10 Dec 2023 19:46:25 +0400 Subject: Backup: Fix config --- lib/pleroma/user/backup.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index c7359313a..04dc8e0ce 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -35,7 +35,7 @@ defmodule Pleroma.User.Backup do timestamps() end - @config_impl Application.compile_env(__MODULE__, :config_impl, Pleroma.Config) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) def create(user, admin_id \\ nil) do with :ok <- validate_limit(user, admin_id), -- cgit v1.2.3 From 223c1bac8dcbfd6b2454b79077545d3465688d7f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 10 Dec 2023 12:55:41 -0500 Subject: Cancel the job if the signature is still invalid after a refetch of the public key --- lib/pleroma/workers/receiver_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index d4e8e715e..1dddd8d2e 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -50,7 +50,7 @@ defmodule Pleroma.Workers.ReceiverWorker do {:error, {:validate_object, reason}} -> {:cancel, reason} {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} - {:signature, false} -> {:error, :invalid_signature} + {:signature, false} -> {:cancel, :invalid_signature} {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason} e -> e end -- cgit v1.2.3 From 18deea59b441a89c1e6870d29dd0a6c0f3070f55 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 10 Dec 2023 13:22:55 -0500 Subject: ActivityPub.make_user_from_ap_id/1 fetches the whole actor object including updating the public key for us --- lib/pleroma/signature.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 42cceba28..5cfdae051 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Signature do with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), {:ok, actor_id} <- key_id_to_actor_id(kid), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), - {:ok, public_key} <- User.get_or_fetch_public_key_for_ap_id(actor_id) do + {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} else e -> -- cgit v1.2.3 From c0a50b7c3e340cd621827922200daa0f29dc6e15 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 10 Dec 2023 13:24:04 -0500 Subject: User.get_or_fetch_public_key_for_ap_id/1 is no longer required. --- lib/pleroma/user.ex | 9 --------- 1 file changed, 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 3f729fdcc..0706f5607 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2135,15 +2135,6 @@ defmodule Pleroma.User do def public_key(_), do: {:error, "key not found"} - def get_or_fetch_public_key_for_ap_id(ap_id) do - with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id), - {:ok, public_key} <- public_key(user) do - {:ok, public_key} - else - _ -> :error - end - end - def get_public_key_for_ap_id(ap_id) do with %User{} = user <- get_cached_by_ap_id(ap_id), {:ok, public_key} <- public_key(user) do -- cgit v1.2.3 From 90a47ca050c5839e8b4dc3bac315dc436d49152d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 11 Dec 2023 09:25:05 +0400 Subject: S3 Test: Remove global state dependencies --- lib/pleroma/upload.ex | 15 ++++++++------- lib/pleroma/uploaders/s3.ex | 15 ++++++++++----- 2 files changed, 18 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 4aee9326f..bedd7889a 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -34,7 +34,6 @@ defmodule Pleroma.Upload do """ alias Ecto.UUID - alias Pleroma.Config alias Pleroma.Maps alias Pleroma.Web.ActivityPub.Utils require Logger @@ -76,6 +75,8 @@ defmodule Pleroma.Upload do :path ] + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + defp get_description(upload) do case {upload.description, Pleroma.Config.get([Pleroma.Upload, :default_description])} do {description, _} when is_binary(description) -> description @@ -244,18 +245,18 @@ defmodule Pleroma.Upload do defp url_from_spec(_upload, _base_url, {:url, url}), do: url def base_url do - uploader = Config.get([Pleroma.Upload, :uploader]) - upload_base_url = Config.get([Pleroma.Upload, :base_url]) - public_endpoint = Config.get([uploader, :public_endpoint]) + uploader = @config_impl.get([Pleroma.Upload, :uploader]) + upload_base_url = @config_impl.get([Pleroma.Upload, :base_url]) + public_endpoint = @config_impl.get([uploader, :public_endpoint]) case uploader do Pleroma.Uploaders.Local -> upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" Pleroma.Uploaders.S3 -> - bucket = Config.get([Pleroma.Uploaders.S3, :bucket]) - truncated_namespace = Config.get([Pleroma.Uploaders.S3, :truncated_namespace]) - namespace = Config.get([Pleroma.Uploaders.S3, :bucket_namespace]) + bucket = @config_impl.get([Pleroma.Uploaders.S3, :bucket]) + truncated_namespace = @config_impl.get([Pleroma.Uploaders.S3, :truncated_namespace]) + namespace = @config_impl.get([Pleroma.Uploaders.S3, :bucket_namespace]) bucket_with_namespace = cond do diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index 19287c532..7b32bd8a5 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -6,7 +6,8 @@ defmodule Pleroma.Uploaders.S3 do @behaviour Pleroma.Uploaders.Uploader require Logger - alias Pleroma.Config + @ex_aws_impl Application.compile_env(:pleroma, [__MODULE__, :ex_aws_impl], ExAws) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) # The file name is re-encoded with S3's constraints here to comply with previous # links with less strict filenames @@ -22,7 +23,7 @@ defmodule Pleroma.Uploaders.S3 do @impl true def put_file(%Pleroma.Upload{} = upload) do - config = Config.get([__MODULE__]) + config = @config_impl.get([__MODULE__]) bucket = Keyword.get(config, :bucket) streaming = Keyword.get(config, :streaming_enabled) @@ -56,7 +57,7 @@ defmodule Pleroma.Uploaders.S3 do ]) end - case ExAws.request(op) do + case @ex_aws_impl.request(op) do {:ok, _} -> {:ok, {:file, s3_name}} @@ -69,9 +70,9 @@ defmodule Pleroma.Uploaders.S3 do @impl true def delete_file(file) do [__MODULE__, :bucket] - |> Config.get() + |> @config_impl.get() |> ExAws.S3.delete_object(file) - |> ExAws.request() + |> @ex_aws_impl.request() |> case do {:ok, %{status_code: 204}} -> :ok error -> {:error, inspect(error)} @@ -83,3 +84,7 @@ defmodule Pleroma.Uploaders.S3 do String.replace(name, @regex, "-") end end + +defmodule Pleroma.Uploaders.S3.ExAwsAPI do + @callback request(op :: ExAws.Operation.t()) :: {:ok, ExAws.Operation.t()} | {:error, term()} +end -- cgit v1.2.3 From 228966e6dd1bd4a01d0c01609d9f6ad2279d278f Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 11:09:54 +0400 Subject: Exiftool.ReadDescription: Remove wrong spec. --- lib/pleroma/upload/filter/exiftool/read_description.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/exiftool/read_description.ex b/lib/pleroma/upload/filter/exiftool/read_description.ex index 543b22031..8c1ed82f8 100644 --- a/lib/pleroma/upload/filter/exiftool/read_description.ex +++ b/lib/pleroma/upload/filter/exiftool/read_description.ex @@ -10,8 +10,6 @@ defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do """ @behaviour Pleroma.Upload.Filter - @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()} - def filter(%Pleroma.Upload{description: description}) when is_binary(description), do: {:ok, :noop} -- cgit v1.2.3 From 22c4d89dbbc45169dfc325e64527e6dd30f7c422 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 12:48:55 +0400 Subject: ScheduledActivity: Use config mocking --- lib/pleroma/scheduled_activity.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 0ed51ad07..63c6cb45b 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -6,7 +6,6 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema alias Ecto.Multi - alias Pleroma.Config alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User @@ -20,6 +19,8 @@ defmodule Pleroma.ScheduledActivity do @min_offset :timer.minutes(5) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + schema "scheduled_activities" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:scheduled_at, :naive_datetime) @@ -87,7 +88,7 @@ defmodule Pleroma.ScheduledActivity do |> where([sa], type(sa.scheduled_at, :date) == type(^scheduled_at, :date)) |> select([sa], count(sa.id)) |> Repo.one() - |> Kernel.>=(Config.get([ScheduledActivity, :daily_user_limit])) + |> Kernel.>=(@config_impl.get([ScheduledActivity, :daily_user_limit])) end def exceeds_total_user_limit?(user_id) do @@ -95,7 +96,7 @@ defmodule Pleroma.ScheduledActivity do |> where(user_id: ^user_id) |> select([sa], count(sa.id)) |> Repo.one() - |> Kernel.>=(Config.get([ScheduledActivity, :total_user_limit])) + |> Kernel.>=(@config_impl.get([ScheduledActivity, :total_user_limit])) end def far_enough?(scheduled_at) when is_binary(scheduled_at) do @@ -123,7 +124,7 @@ defmodule Pleroma.ScheduledActivity do def create(%User{} = user, attrs) do Multi.new() |> Multi.insert(:scheduled_activity, new(user, attrs)) - |> maybe_add_jobs(Config.get([ScheduledActivity, :enabled])) + |> maybe_add_jobs(@config_impl.get([ScheduledActivity, :enabled])) |> Repo.transaction() |> transaction_response end -- cgit v1.2.3 From 00def0875b2014ab18a7b159de855a3c05dd39a4 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 13:28:11 +0400 Subject: RichMediaTest: Use mocked config --- lib/pleroma/web/rich_media/helpers.ex | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 0488df30e..61000bb9b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -4,11 +4,12 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + @options [ pool: :media, max_body: 2_000_000, @@ -17,7 +18,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Config.get([Pleroma.Formatter, :validate_tld]) + validate_tld = @config_impl.get([Pleroma.Formatter, :validate_tld]) page_url |> Linkify.Parser.url?(validate_tld: validate_tld) @@ -27,10 +28,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp validate_page_url(%URI{host: host, scheme: "https", authority: authority}) when is_binary(authority) do cond do - host in Config.get([:rich_media, :ignore_hosts], []) -> + host in @config_impl.get([:rich_media, :ignore_hosts], []) -> :error - get_tld(host) in Config.get([:rich_media, :ignore_tld], []) -> + get_tld(host) in @config_impl.get([:rich_media, :ignore_tld], []) -> :error true -> @@ -56,7 +57,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do end def fetch_data_for_object(object) do - with true <- Config.get([:rich_media, :enabled]), + with true <- @config_impl.get([:rich_media, :enabled]), {:ok, page_url} <- HTML.extract_first_external_url_from_object(object), :ok <- validate_page_url(page_url), @@ -68,7 +69,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do end def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do - with true <- Config.get([:rich_media, :enabled]), + with true <- @config_impl.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity, fetch: false) do fetch_data_for_object(object) else -- cgit v1.2.3 From dca41cc4a37999a6971b70ef5e0996e528b79bf5 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 15:25:52 +0400 Subject: EmailTest: use config mock --- lib/pleroma/emails/mailer.ex | 3 ++- lib/pleroma/user.ex | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 101442130..02c148e9f 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -14,9 +14,10 @@ defmodule Pleroma.Emails.Mailer do @otp_app :pleroma @mailer_config [otp: :pleroma] + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) @spec enabled?() :: boolean() - def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) + def enabled?, do: @config_impl.get([__MODULE__, :enabled]) @doc "add email to queue" def deliver_async(email, config \\ []) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0706f5607..f939687eb 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -86,6 +86,7 @@ defmodule Pleroma.User do ] @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) schema "users" do field(:bio, :string, default: "") @@ -1000,7 +1001,7 @@ defmodule Pleroma.User do @spec maybe_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop} def maybe_send_confirmation_email(%User{is_confirmed: false, email: email} = user) when is_binary(email) do - if Config.get([:instance, :account_activation_required]) do + if @config_impl.get([:instance, :account_activation_required]) do send_confirmation_email(user) {:ok, :enqueued} else -- cgit v1.2.3 From e7af2addd85c66edc62262e5e799649fef6a9911 Mon Sep 17 00:00:00 2001 From: Alexander Tumin Date: Tue, 12 Dec 2023 15:08:20 +0300 Subject: Add media proxy to opengraph rich media cards --- lib/pleroma/web/mastodon_api/views/status_view.ex | 27 ++++++++++++++--------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index d070262cc..8825f3715 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -562,25 +562,24 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do page_url = page_url_data |> to_string - image_url_data = - if is_binary(rich_media["image"]) do - URI.parse(rich_media["image"]) - else - nil - end - - image_url = build_image_url(image_url_data, page_url_data) + image_url = proxied_url(rich_media["image"], page_url_data) + audio_url = proxied_url(rich_media["audio"], page_url_data) + video_url = proxied_url(rich_media["video"], page_url_data) %{ type: "link", provider_name: page_url_data.host, provider_url: page_url_data.scheme <> "://" <> page_url_data.host, url: page_url, - image: image_url |> MediaProxy.url(), + image: image_url, title: rich_media["title"] || "", description: rich_media["description"] || "", pleroma: %{ - opengraph: rich_media + opengraph: + rich_media + |> Maps.put_if_present("image", image_url) + |> Maps.put_if_present("audio", audio_url) + |> Maps.put_if_present("video", video_url) } } end @@ -817,4 +816,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp get_source_content_type(_source) do Utils.get_content_type(nil) end + + defp proxied_url(url, page_url_data) do + if is_binary(url) do + build_image_url(URI.parse(url), page_url_data) |> MediaProxy.url() + else + nil + end + end end -- cgit v1.2.3 From cca6c20eb6acd0f510054d1a1050d12ea2a32482 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 12 Dec 2023 19:35:19 +0400 Subject: Revert "EmailTest: use config mock" This reverts commit dca41cc4a37999a6971b70ef5e0996e528b79bf5. --- lib/pleroma/emails/mailer.ex | 3 +-- lib/pleroma/user.ex | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 02c148e9f..101442130 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -14,10 +14,9 @@ defmodule Pleroma.Emails.Mailer do @otp_app :pleroma @mailer_config [otp: :pleroma] - @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) @spec enabled?() :: boolean() - def enabled?, do: @config_impl.get([__MODULE__, :enabled]) + def enabled?, do: Pleroma.Config.get([__MODULE__, :enabled]) @doc "add email to queue" def deliver_async(email, config \\ []) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f939687eb..0706f5607 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -86,7 +86,6 @@ defmodule Pleroma.User do ] @cachex Pleroma.Config.get([:cachex, :provider], Cachex) - @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) schema "users" do field(:bio, :string, default: "") @@ -1001,7 +1000,7 @@ defmodule Pleroma.User do @spec maybe_send_confirmation_email(User.t()) :: {:ok, :enqueued | :noop} def maybe_send_confirmation_email(%User{is_confirmed: false, email: email} = user) when is_binary(email) do - if @config_impl.get([:instance, :account_activation_required]) do + if Config.get([:instance, :account_activation_required]) do send_confirmation_email(user) {:ok, :enqueued} else -- cgit v1.2.3 From cb1b52d980e23c7d4f06dda5891bd2cbbdafeb95 Mon Sep 17 00:00:00 2001 From: Yonle Date: Fri, 15 Dec 2023 07:13:31 +0700 Subject: ap userview: add outbox field. Signed-off-by: Yonle --- lib/pleroma/web/activity_pub/views/user_view.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index f69fca075..24ee683ae 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -46,6 +46,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "following" => "#{user.ap_id}/following", "followers" => "#{user.ap_id}/followers", "inbox" => "#{user.ap_id}/inbox", + "outbox" => "#{user.ap_id}/outbox", "name" => "Pleroma", "summary" => "An internal service actor for this Pleroma instance. No user-serviceable parts inside.", -- cgit v1.2.3 From 3fbc80eb585b44e2f315b9bf02b655b50ee473d3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 16 Dec 2023 20:26:08 +0400 Subject: B ActivityPub.Publisher: Prioritize direct mentions --- lib/pleroma/web/activity_pub/publisher.ex | 104 +++++++++++++++++------------- lib/pleroma/web/federator/publisher.ex | 7 +- 2 files changed, 64 insertions(+), 47 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index af6aa0781..373ceef32 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -118,7 +118,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end - @spec recipients(User.t(), Activity.t()) :: list(User.t()) | [] + @spec recipients(User.t(), Activity.t()) :: {[User.t()], [User.t()]} defp recipients(actor, activity) do followers = if actor.follower_address in activity.recipients do @@ -138,7 +138,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [] end - Pleroma.Web.Federator.Publisher.remote_users(actor, activity) ++ followers ++ fetchers + mentioned = Pleroma.Web.Federator.Publisher.remote_users(actor, activity) + non_mentioned = (followers ++ fetchers) -- mentioned + + {mentioned, non_mentioned} end defp get_cc_ap_ids(ap_id, recipients) do @@ -195,34 +198,39 @@ defmodule Pleroma.Web.ActivityPub.Publisher do public = is_public?(activity) {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - recipients = recipients(actor, activity) + {priority_recipients, recipients} = recipients(actor, activity) inboxes = - recipients - |> Enum.map(fn actor -> actor.inbox end) - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - |> Instances.filter_reachable() + [priority_recipients, recipients] + |> Enum.map(fn recipients -> + recipients + |> Enum.map(fn actor -> actor.inbox end) + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Instances.filter_reachable() + end) Repo.checkout(fn -> - Enum.each(inboxes, fn {inbox, unreachable_since} -> - %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) - - # Get all the recipients on the same host and add them to cc. Otherwise, a remote - # instance would only accept a first message for the first recipient and ignore the rest. - cc = get_cc_ap_ids(ap_id, recipients) - - json = - data - |> Map.put("cc", cc) - |> Jason.encode!() - - Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ - inbox: inbox, - json: json, - actor_id: actor.id, - id: activity.data["id"], - unreachable_since: unreachable_since - }) + Enum.each(inboxes, fn inboxes -> + Enum.each(inboxes, fn {inbox, unreachable_since} -> + %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) + + # Get all the recipients on the same host and add them to cc. Otherwise, a remote + # instance would only accept a first message for the first recipient and ignore the rest. + cc = get_cc_ap_ids(ap_id, recipients) + + json = + data + |> Map.put("cc", cc) + |> Jason.encode!() + + Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ + inbox: inbox, + json: json, + actor_id: actor.id, + id: activity.data["id"], + unreachable_since: unreachable_since + }) + end) end) end) end @@ -239,25 +247,33 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) - recipients(actor, activity) - |> Enum.map(fn %User{} = user -> - determine_inbox(activity, user) - end) - |> Enum.uniq() - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - |> Instances.filter_reachable() - |> Enum.each(fn {inbox, unreachable_since} -> - Pleroma.Web.Federator.Publisher.enqueue_one( - __MODULE__, - %{ - inbox: inbox, - json: json, - actor_id: actor.id, - id: activity.data["id"], - unreachable_since: unreachable_since - } - ) + {priority_recipients, recipients} = recipients(actor, activity) + + [{priority_recipients, 0}, {recipients, 1}] + |> Enum.map(fn {recipients, priority} -> + recipients + |> Enum.map(fn %User{} = user -> + determine_inbox(activity, user) + end) + |> Enum.uniq() + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Instances.filter_reachable() + |> Enum.each(fn {inbox, unreachable_since} -> + Pleroma.Web.Federator.Publisher.enqueue_one( + __MODULE__, + %{ + inbox: inbox, + json: json, + actor_id: actor.id, + id: activity.data["id"], + unreachable_since: unreachable_since + }, + priority: priority + ) + end) end) + + :ok end def gather_webfinger_links(%User{} = user) do diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index a45796e9d..8c6547208 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -29,11 +29,12 @@ defmodule Pleroma.Web.Federator.Publisher do @doc """ Enqueue publishing a single activity. """ - @spec enqueue_one(module(), Map.t()) :: :ok - def enqueue_one(module, %{} = params) do + @spec enqueue_one(module(), Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} + def enqueue_one(module, %{} = params, worker_args \\ []) do PublisherWorker.enqueue( "publish_one", - %{"module" => to_string(module), "params" => params} + %{"module" => to_string(module), "params" => params}, + worker_args ) end -- cgit v1.2.3 From c212fc1dcfae578e83a0a8fb036da30ca0b92f56 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sat, 16 Dec 2023 20:32:13 +0400 Subject: User: Ignore non-local users when setting 'last active at' --- lib/pleroma/user.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0706f5607..ddeaf0624 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2681,6 +2681,8 @@ defmodule Pleroma.User do |> update_and_set_cache() end + def update_last_active_at(user), do: user + def active_user_count(days \\ 30) do active_after = Timex.shift(NaiveDateTime.utc_now(), days: -days) -- cgit v1.2.3 From c1423ddca3c82140e537037fdf133ee057c2dac0 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Sun, 17 Dec 2023 11:15:58 +0400 Subject: ActivityPub.Publisher: Filter inboxes --- lib/pleroma/web/activity_pub/publisher.ex | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 373ceef32..a580994b1 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -118,7 +118,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end - @spec recipients(User.t(), Activity.t()) :: {[User.t()], [User.t()]} + @spec recipients(User.t(), Activity.t()) :: [[User.t()]] defp recipients(actor, activity) do followers = if actor.follower_address in activity.recipients do @@ -141,7 +141,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do mentioned = Pleroma.Web.Federator.Publisher.remote_users(actor, activity) non_mentioned = (followers ++ fetchers) -- mentioned - {mentioned, non_mentioned} + [mentioned, non_mentioned] end defp get_cc_ap_ids(ap_id, recipients) do @@ -198,7 +198,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do public = is_public?(activity) {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - {priority_recipients, recipients} = recipients(actor, activity) + [priority_recipients, recipients] = recipients(actor, activity) inboxes = [priority_recipients, recipients] @@ -247,16 +247,19 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) - {priority_recipients, recipients} = recipients(actor, activity) - - [{priority_recipients, 0}, {recipients, 1}] - |> Enum.map(fn {recipients, priority} -> - recipients - |> Enum.map(fn %User{} = user -> - determine_inbox(activity, user) + [priority_inboxes, inboxes] = + recipients(actor, activity) + |> Enum.map(fn recipients -> + recipients + |> Enum.map(fn actor -> actor.inbox end) + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) end) - |> Enum.uniq() - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + + inboxes = inboxes -- priority_inboxes + + [{priority_inboxes, 0}, {inboxes, 1}] + |> Enum.each(fn {inboxes, priority} -> + inboxes |> Instances.filter_reachable() |> Enum.each(fn {inbox, unreachable_since} -> Pleroma.Web.Federator.Publisher.enqueue_one( -- cgit v1.2.3 From e2066994b1e64481e0e74350688d91c71d03e230 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Dec 2023 10:18:29 -0500 Subject: Fix Web Push notification delivery Finch does not automatically append header content-type: octet-stream for binary payloads. --- lib/pleroma/http/web_push.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/http/web_push.ex b/lib/pleroma/http/web_push.ex index ca399b6c8..888079c1e 100644 --- a/lib/pleroma/http/web_push.ex +++ b/lib/pleroma/http/web_push.ex @@ -6,7 +6,11 @@ defmodule Pleroma.HTTP.WebPush do @moduledoc false def post(url, payload, headers, options \\ []) do - list_headers = Map.to_list(headers) + list_headers = + headers + |> Map.to_list() + |> Kernel.++([{"content-type", "octet-stream"}]) + Pleroma.HTTP.post(url, payload, list_headers, options) end end -- cgit v1.2.3 From f43f33e3078385084136295d2a3320efa6cb4134 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 19 Dec 2023 13:53:10 -0500 Subject: Return a 400 from a bad delivery attempt to the inbox This stops the backend from generating 500 errors from these events. --- lib/pleroma/web/activity_pub/activity_pub_controller.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3b2193ca3..b1a118160 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -273,12 +273,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do - with %User{} = recipient <- User.get_cached_by_nickname(nickname), - {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]), + with %User{is_active: true} = recipient <- User.get_cached_by_nickname(nickname), + {:ok, %User{is_active: true} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]), true <- Utils.recipient_in_message(recipient, actor, params), params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do Federator.incoming_ap_doc(params) json(conn, "ok") + else + _ -> + conn + |> put_status(:bad_request) + |> json("Invalid request.") end end -- cgit v1.2.3 From 9896b64f54d4568a7b42ddd11e647021a20c1378 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 28 Nov 2023 17:17:46 +0000 Subject: Elixir 1.15: Chase the Logger.warn deprecation --- lib/pleroma/application.ex | 2 +- lib/pleroma/application_requirements.ex | 4 ++-- lib/pleroma/config/deprecation_warnings.ex | 18 +++++++++--------- lib/pleroma/config/oban.ex | 2 +- lib/pleroma/config/transfer_task.ex | 6 +++--- lib/pleroma/emoji/loader.ex | 2 +- lib/pleroma/gun/conn.ex | 6 +++--- lib/pleroma/http/adapter_helper.ex | 8 ++++---- lib/pleroma/instances/instance.ex | 6 +++--- lib/pleroma/maintenance.ex | 2 +- lib/pleroma/migrators/support/base_migrator.ex | 6 +++--- lib/pleroma/reverse_proxy.ex | 2 +- lib/pleroma/telemetry/logger.ex | 2 +- lib/pleroma/upload/filter/analyze_metadata.ex | 4 ++-- lib/pleroma/user.ex | 2 +- lib/pleroma/user/backup.ex | 2 +- .../web/activity_pub/activity_pub_controller.ex | 2 +- lib/pleroma/web/activity_pub/mrf.ex | 2 +- lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex | 4 ++-- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++-- lib/pleroma/web/common_api.ex | 2 +- lib/pleroma/web/common_api/utils.ex | 4 ++-- lib/pleroma/web/plugs/http_security_plug.ex | 2 +- lib/pleroma/web/plugs/rate_limiter.ex | 2 +- lib/pleroma/web/push.ex | 2 +- lib/pleroma/web/push/impl.ex | 2 +- lib/pleroma/web/rich_media/parser.ex | 2 +- lib/pleroma/web/web_finger.ex | 2 +- 29 files changed, 53 insertions(+), 53 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 52cd6e9a9..8fa6f3fae 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -138,7 +138,7 @@ defmodule Pleroma.Application do num else e -> - Logger.warn( + Logger.warning( "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6" ) diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 477eac35b..1dbfea3e2 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -34,7 +34,7 @@ defmodule Pleroma.ApplicationRequirements do defp check_welcome_message_config!(:ok) do if Pleroma.Config.get([:welcome, :email, :enabled], false) and not Pleroma.Emails.Mailer.enabled?() do - Logger.warn(""" + Logger.warning(""" To send welcome emails, you need to enable the mailer. Welcome emails will NOT be sent with the current config. @@ -53,7 +53,7 @@ defmodule Pleroma.ApplicationRequirements do def check_confirmation_accounts!(:ok) do if Pleroma.Config.get([:instance, :account_activation_required]) && not Pleroma.Emails.Mailer.enabled?() do - Logger.warn(""" + Logger.warning(""" Account activation is required, but the mailer is disabled. Users will NOT be able to confirm their accounts with this config. Either disable account activation or enable the mailer. diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index b53b15d95..1cd3241ea 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Config.DeprecationWarnings do filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) if Pleroma.Upload.Filter.Exiftool in filters do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -63,7 +63,7 @@ defmodule Pleroma.Config.DeprecationWarnings do |> Enum.any?(fn {_, v} -> Enum.any?(v, &is_binary/1) end) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the SimplePolicy configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -121,7 +121,7 @@ defmodule Pleroma.Config.DeprecationWarnings do has_strings = Config.get([:instance, :quarantined_instances]) |> Enum.any?(&is_binary/1) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the quarantined_instances configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -158,7 +158,7 @@ defmodule Pleroma.Config.DeprecationWarnings do has_strings = Config.get([:mrf, :transparency_exclusions]) |> Enum.any?(&is_binary/1) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the transparency_exclusions configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -193,7 +193,7 @@ defmodule Pleroma.Config.DeprecationWarnings do def check_hellthread_threshold do if Config.get([:mrf_hellthread, :threshold]) do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! You are using the old configuration mechanism for the hellthread filter. Please check config.md. """) @@ -274,7 +274,7 @@ defmodule Pleroma.Config.DeprecationWarnings do if warning == "" do :ok else - Logger.warn(warning_preface <> warning) + Logger.warning(warning_preface <> warning) :error end end @@ -284,7 +284,7 @@ defmodule Pleroma.Config.DeprecationWarnings do whitelist = Config.get([:media_proxy, :whitelist]) if Enum.any?(whitelist, &(not String.starts_with?(&1, "http"))) do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. """) @@ -299,7 +299,7 @@ defmodule Pleroma.Config.DeprecationWarnings do pool_config = Config.get(:connections_pool) if timeout = pool_config[:await_up_timeout] do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using old setting `config :pleroma, :connections_pool, await_up_timeout`. Please change to `config :pleroma, :connections_pool, connect_timeout` to ensure compatibility with future releases. """) @@ -331,7 +331,7 @@ defmodule Pleroma.Config.DeprecationWarnings do "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`" end) - Logger.warn(Enum.join([warning_preface | pool_warnings])) + Logger.warning(Enum.join([warning_preface | pool_warnings])) Config.put(:pools, updated_config) :error diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex index 483d2bb79..836f0c1a7 100644 --- a/lib/pleroma/config/oban.ex +++ b/lib/pleroma/config/oban.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Config.Oban do You are using old workers in Oban crontab settings, which were removed. Please, remove setting from crontab in your config file (prod.secret.exs): #{inspect(setting)} """ - |> Logger.warn() + |> Logger.warning() List.delete(acc, setting) else diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 6fd05b0e0..91885347f 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -144,7 +144,7 @@ defmodule Pleroma.Config.TransferTask do error_msg = "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{inspect(value)} error: #{inspect(error)}" - Logger.warn(error_msg) + Logger.warning(error_msg) nil end @@ -178,12 +178,12 @@ defmodule Pleroma.Config.TransferTask do :ok = Application.start(app) else nil -> - Logger.warn("#{app} is not started.") + Logger.warning("#{app} is not started.") error -> error |> inspect() - |> Logger.warn() + |> Logger.warning() end end diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index 97d4b8f70..eb6f6816b 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Emoji.Loader do Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") if not Enum.empty?(files) do - Logger.warn( + Logger.warning( "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{Enum.join(files, ", ")}" ) end diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 7c5785def..804cd11c7 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) @@ -90,7 +90,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) @@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 252a6aba5..e9bb2023a 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -70,15 +70,15 @@ defmodule Pleroma.HTTP.AdapterHelper do {:ok, parse_host(host), port} else {_, _} -> - Logger.warn("Parsing port failed #{inspect(proxy)}") + Logger.warning("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} :error -> - Logger.warn("Parsing port failed #{inspect(proxy)}") + Logger.warning("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") + Logger.warning("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end @@ -88,7 +88,7 @@ defmodule Pleroma.HTTP.AdapterHelper do {:ok, type, parse_host(host), port} else _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") + Logger.warning("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 101e5dc88..c497a4fb7 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -173,7 +173,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") + Logger.warning("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") nil end @@ -201,7 +201,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn( + Logger.warning( "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" ) @@ -284,7 +284,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn( + Logger.warning( "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" ) diff --git a/lib/pleroma/maintenance.ex b/lib/pleroma/maintenance.ex index eb5a6ef42..1e39b03e6 100644 --- a/lib/pleroma/maintenance.ex +++ b/lib/pleroma/maintenance.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Maintenance do "full" -> Logger.info("Running VACUUM FULL.") - Logger.warn( + Logger.warning( "Re-packing your entire database may take a while and will consume extra disk space during the process." ) diff --git a/lib/pleroma/migrators/support/base_migrator.ex b/lib/pleroma/migrators/support/base_migrator.ex index 3bcd59fd0..ce88caac7 100644 --- a/lib/pleroma/migrators/support/base_migrator.ex +++ b/lib/pleroma/migrators/support/base_migrator.ex @@ -73,7 +73,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do data_migration.state == :manual or data_migration.name in manual_migrations -> message = "Data migration is in manual execution or manual fix mode." update_status(:manual, message) - Logger.warn("#{__MODULE__}: #{message}") + Logger.warning("#{__MODULE__}: #{message}") data_migration.state == :complete -> on_complete(data_migration) @@ -109,7 +109,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`. """ - Logger.warn("#{__MODULE__}: #{message}") + Logger.warning("#{__MODULE__}: #{message}") update_status(:manual, message) on_complete(data_migration()) @@ -125,7 +125,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do defp on_complete(data_migration) do if data_migration.feature_lock || feature_state() == :disabled do - Logger.warn( + Logger.warning( "#{__MODULE__}: migration complete but feature is locked; consider enabling." ) diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 2248c2713..880940d07 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -192,7 +192,7 @@ defmodule Pleroma.ReverseProxy do halt(conn) {:error, error, conn} -> - Logger.warn( + Logger.warning( "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}" ) diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 384c70fbc..92d395394 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Telemetry.Logger do %{key: key}, _ ) do - Logger.warn(fn -> + Logger.warning(fn -> "Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{inspect(reason)}" end) end diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index e510ae3e6..7ee643277 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {:ok, :filtered, upload} rescue e in ErlangError -> - Logger.warn("#{__MODULE__}: #{inspect(e)}") + Logger.warning("#{__MODULE__}: #{inspect(e)}") {:ok, :noop} end end @@ -46,7 +46,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {:ok, :filtered, upload} rescue e in ErlangError -> - Logger.warn("#{__MODULE__}: #{inspect(e)}") + Logger.warning("#{__MODULE__}: #{inspect(e)}") {:ok, :noop} end end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ddeaf0624..5f98935b3 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1560,7 +1560,7 @@ defmodule Pleroma.User do unmute(muter, mutee) else {who, result} = error -> - Logger.warn( + Logger.warning( "User.unmute/2 failed. #{who}: #{result}, muter_id: #{muter_id}, mutee_id: #{mutee_id}" ) diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 04dc8e0ce..74e0ec073 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -293,7 +293,7 @@ defmodule Pleroma.User.Backup do acc + 1 else {:error, e} -> - Logger.warn( + Logger.warning( "Error processing backup item: #{inspect(e)}\n The item is: #{inspect(i)}" ) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 3b2193ca3..3f76531c6 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -475,7 +475,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json(message) e -> - Logger.warn(fn -> "AP C2S: #{inspect(e)}" end) + Logger.warning(fn -> "AP C2S: #{inspect(e)}" end) conn |> put_status(:bad_request) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 69054e25b..7f6dce925 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -203,7 +203,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do if Enum.all?(@required_description_keys, &Map.has_key?(description, &1)) do [description | acc] else - Logger.warn( + Logger.warning( "#{policy} config description doesn't have one or all required keys #{inspect(@required_description_keys)}" ) diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 5b6adbb4b..5a4a97626 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do try_follow(follower, message) else nil -> - Logger.warn( + Logger.warning( "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname account does not exist, or the account is not correctly configured as a bot." ) diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index f66c379b5..28c2cf3b3 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do shortcode e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") + Logger.warning("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") nil end else @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do end else e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") + Logger.warning("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") nil end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 86d3ac60f..35f3aea03 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -156,7 +156,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.drop(["conversation", "inReplyToAtomUri"]) else e -> - Logger.warn("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") + Logger.warning("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") object end else @@ -182,7 +182,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do object e -> - Logger.warn("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") + Logger.warning("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") object end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 82c7f70d2..dfc0a625d 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -550,7 +550,7 @@ defmodule Pleroma.Web.CommonAPI do remove_mute(user, activity) else {what, result} = error -> - Logger.warn( + Logger.warning( "CommonAPI.remove_mute/2 failed. #{what}: #{result}, user_id: #{user_id}, activity_id: #{activity_id}" ) diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 0f394e951..dcda3e0e8 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -321,13 +321,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do format_asctime(date) else _e -> - Logger.warn("Date #{date} in wrong format, must be ISO 8601") + Logger.warning("Date #{date} in wrong format, must be ISO 8601") "" end end def date_to_asctime(date) do - Logger.warn("Date #{date} in wrong format, must be ISO 8601") + Logger.warning("Date #{date} in wrong format, must be ISO 8601") "" end diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 5093414c4..a27dcd0ab 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -201,7 +201,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do def warn_if_disabled do unless Config.get([:http_security, :enabled]) do - Logger.warn(" + Logger.warning(" .i;;;;i. iYcviii;vXY: .YXi .i1c. diff --git a/lib/pleroma/web/plugs/rate_limiter.ex b/lib/pleroma/web/plugs/rate_limiter.ex index 2080b06bd..aa79dbf6b 100644 --- a/lib/pleroma/web/plugs/rate_limiter.ex +++ b/lib/pleroma/web/plugs/rate_limiter.ex @@ -89,7 +89,7 @@ defmodule Pleroma.Web.Plugs.RateLimiter do end defp handle_disabled(conn) do - Logger.warn( + Logger.warning( "Rate limiter disabled due to forwarded IP not being found. Please ensure your reverse proxy is providing the X-Forwarded-For header or disable the RemoteIP plug/rate limiter." ) diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex index 9665b0b4a..0d43f402e 100644 --- a/lib/pleroma/web/push.ex +++ b/lib/pleroma/web/push.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.Push do def init do unless enabled() do - Logger.warn(""" + Logger.warning(""" VAPID key pair is not found. If you wish to enabled web push, please run mix web_push.gen.keypair diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 3c5f00764..36f44d8e8 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -57,7 +57,7 @@ defmodule Pleroma.Web.Push.Impl do end def perform(_) do - Logger.warn("Unknown notification type") + Logger.warning("Unknown notification type") {:error, :unknown_type} end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index dbe81eabb..c37c45963 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -75,7 +75,7 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp log_error(url, reason) do - Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) + Logger.warning(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) end end diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index f95dc2458..9466ad430 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -163,7 +163,7 @@ defmodule Pleroma.Web.WebFinger do get_template_from_xml(body) else error -> - Logger.warn("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}") + Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}") {:error, :lrdd_not_found} end end -- cgit v1.2.3 From 39d3df86c8e2ee05d409865ebc866c543a604ad9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 21 Dec 2023 00:10:30 +0100 Subject: Use consistent terminology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/user.ex | 2 +- lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 +- lib/pleroma/web/web_finger.ex | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index a0d4aca66..523b362d6 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -2252,7 +2252,7 @@ defmodule Pleroma.User do if String.contains?(user.nickname, "@") do user.nickname else - host = Pleroma.Web.WebFinger.domain() + host = Pleroma.Web.WebFinger.host() user.nickname <> "@" <> host end end diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index aef02a418..ba7836e51 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -14,7 +14,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do instance = Config.get(:instance) %{ - uri: Pleroma.Web.WebFinger.domain(), + uri: Pleroma.Web.WebFinger.host(), title: Keyword.get(instance, :name), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index b28fad8d1..c6840f10b 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Web.WebFinger do def represent_user(user, "JSON") do %{ - "subject" => "acct:#{user.nickname}@#{domain()}", + "subject" => "acct:#{user.nickname}@#{host()}", "aliases" => gather_aliases(user), "links" => gather_links(user) } @@ -90,13 +90,13 @@ defmodule Pleroma.Web.WebFinger do :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{domain()}"} + {:Subject, "acct:#{user.nickname}@#{host()}"} ] ++ aliases ++ links } |> XmlBuilder.to_doc() end - def domain do + def host do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end -- cgit v1.2.3 From 4f2fb8dc5e2b4ca88d42565ccd811e1028e8c720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 21 Dec 2023 00:13:13 +0100 Subject: Use consistent terminology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 4 ++-- lib/pleroma/web/web_finger.ex | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index e03025a00..f95b5360a 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do common_information(instance) |> Map.merge(%{ - uri: Pleroma.Web.WebFinger.domain(), + uri: Pleroma.Web.WebFinger.host(), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), email: Keyword.get(instance, :email), @@ -49,7 +49,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do common_information(instance) |> Map.merge(%{ - domain: Pleroma.Web.WebFinger.domain(), + domain: Pleroma.Web.WebFinger.host(), source_url: Pleroma.Application.repository(), description: Keyword.get(instance, :short_description), usage: %{users: %{active_month: Pleroma.User.active_user_count()}}, diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index b28fad8d1..c6840f10b 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Web.WebFinger do def represent_user(user, "JSON") do %{ - "subject" => "acct:#{user.nickname}@#{domain()}", + "subject" => "acct:#{user.nickname}@#{host()}", "aliases" => gather_aliases(user), "links" => gather_links(user) } @@ -90,13 +90,13 @@ defmodule Pleroma.Web.WebFinger do :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{domain()}"} + {:Subject, "acct:#{user.nickname}@#{host()}"} ] ++ aliases ++ links } |> XmlBuilder.to_doc() end - def domain do + def host do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end -- cgit v1.2.3 From ea0ec5fbcf2e17d97f323bcb6872df48d5e42714 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 14:20:33 -0500 Subject: Remove Fetcher.fetch_object_from_id!/2 It was only being called once and can be replaced with a case statement. --- lib/pleroma/object.ex | 5 ++++- lib/pleroma/object/fetcher.ex | 26 +++++--------------------- 2 files changed, 9 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index fa5baf1a4..f2ba24abf 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -177,7 +177,10 @@ defmodule Pleroma.Object do ap_id Keyword.get(options, :fetch) -> - Fetcher.fetch_object_from_id!(ap_id, options) + case Fetcher.fetch_object_from_id(ap_id, options) do + {:ok, object} -> object + _ -> nil + end true -> get_cached_by_ap_id(ap_id) diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index cc3772563..508ee7e62 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -79,9 +79,11 @@ defmodule Pleroma.Object.Fetcher do {:error, "Object containment failed."} {:transmogrifier, {:error, {:reject, e}}} -> + Logger.info("Rejected #{id} while fetching: #{inspect(e)}") {:reject, e} {:transmogrifier, {:reject, e}} -> + Logger.info("Rejected #{id} while fetching: #{inspect(e)}") {:reject, e} {:transmogrifier, _} = e -> @@ -97,10 +99,12 @@ defmodule Pleroma.Object.Fetcher do {:ok, object} {:fetch, {:error, error}} -> + Logger.error("Error while fetching #{id}: #{inspect(error)}") {:error, error} e -> - e + Logger.error("Error while fetching #{id}: #{inspect(e)}") + {:error, e} end end @@ -117,26 +121,6 @@ defmodule Pleroma.Object.Fetcher do |> Maps.put_if_present("bcc", data["bcc"]) end - def fetch_object_from_id!(id, options \\ []) do - with {:ok, object} <- fetch_object_from_id(id, options) do - object - else - {:error, %Tesla.Mock.Error{}} -> - nil - - {:error, "Object has been deleted"} -> - nil - - {:reject, reason} -> - Logger.info("Rejected #{id} while fetching: #{inspect(reason)}") - nil - - e -> - Logger.error("Error while fetching #{id}: #{inspect(e)}") - nil - end - end - defp make_signature(id, date) do uri = URI.parse(id) -- cgit v1.2.3 From 603e9f6a92e8ea2d2d34d2af26ec2b12fe4b8543 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 14:22:04 -0500 Subject: Fix Transmogrifier tests These tests relied on the removed Fetcher.fetch_object_from_id!/2 function injecting the error tuple into a log message with the exact words "Object containment failed." We will keep this behavior by generating a similar log message, but perhaps this should do a better job of matching on the error tuple returned by Transmogrifier.handle_incoming/1 --- lib/pleroma/object/fetcher.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 508ee7e62..13fb50639 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -75,8 +75,9 @@ defmodule Pleroma.Object.Fetcher do {:allowed_depth, false} -> {:error, "Max thread distance exceeded."} - {:containment, _} -> - {:error, "Object containment failed."} + {:containment, e} -> + Logger.info("Error while fetching #{id}: Object containment failed. #{inspect(e)}") + {:error, e} {:transmogrifier, {:error, {:reject, e}}} -> Logger.info("Rejected #{id} while fetching: #{inspect(e)}") -- cgit v1.2.3 From d472bafec19cee269e7c943bafae7c805785acd7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 15:54:14 -0500 Subject: Mark instances as unreachable when returning a 403 from an object fetch This is a definite sign the instance is blocked and they are enforcing authorized_fetch --- lib/pleroma/object/fetcher.ex | 9 +++++++++ lib/pleroma/workers/remote_fetcher_worker.ex | 11 +++++++++++ 2 files changed, 20 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 13fb50639..a07b35d6f 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -99,6 +99,15 @@ defmodule Pleroma.Object.Fetcher do {:fetch_object, %Object{} = object} -> {:ok, object} + {:fetch, {:error, {:ok, %Tesla.Env{status: 403}}}} -> + Instances.set_consistently_unreachable(id) + + Logger.error( + "Error while fetching #{id}: HTTP 403 likely due to instance block rejecting the signed fetch." + ) + + {:error, "Object fetch has been denied"} + {:fetch, {:error, error}} -> Logger.error("Error while fetching #{id}: #{inspect(error)}") {:error, error} diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index d2a77aa17..70d074c1c 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -10,6 +10,17 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do {:ok, _object} = Fetcher.fetch_object_from_id(id, depth: args["depth"]) + + case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do + {:ok, _object} -> + :ok + + {:error, reason = "Object fetch has been denied"} -> + {:cancel, reason} + + _ -> + :error + end end @impl Oban.Worker -- cgit v1.2.3 From 67dd81e8254576950fc20ed84d3da3a3d24c1706 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 16:05:28 -0500 Subject: Consolidate the HTTP status code checking into the private get_object/1 --- lib/pleroma/object/fetcher.ex | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index a07b35d6f..75f39fb6a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -99,15 +99,6 @@ defmodule Pleroma.Object.Fetcher do {:fetch_object, %Object{} = object} -> {:ok, object} - {:fetch, {:error, {:ok, %Tesla.Env{status: 403}}}} -> - Instances.set_consistently_unreachable(id) - - Logger.error( - "Error while fetching #{id}: HTTP 403 likely due to instance block rejecting the signed fetch." - ) - - {:error, "Object fetch has been denied"} - {:fetch, {:error, error}} -> Logger.error("Error while fetching #{id}: #{inspect(error)}") {:error, error} @@ -221,6 +212,10 @@ defmodule Pleroma.Object.Fetcher do {:error, {:content_type, nil}} end + {:ok, %{status: 403}} -> + Instances.set_consistently_unreachable(id) + {:error, "Object fetch has been denied"} + {:ok, %{status: code}} when code in [404, 410] -> {:error, "Object has been deleted"} -- cgit v1.2.3 From c6b38441f15a4f3bb925bc4ffc68ac8930bc29b2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 16:05:44 -0500 Subject: Cancel remote fetch jobs for deleted objects --- lib/pleroma/workers/remote_fetcher_worker.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 70d074c1c..1a7bd2586 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -18,6 +18,9 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:error, reason = "Object fetch has been denied"} -> {:cancel, reason} + {:error, reason = "Object has been deleted"} -> + {:cancel, reason} + _ -> :error end -- cgit v1.2.3 From 5f6966cd9ff9c728d97a8e4075cba738ceca0aeb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 16:23:41 -0500 Subject: Remove mistaken duplicate fetch --- lib/pleroma/workers/remote_fetcher_worker.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 1a7bd2586..ff0286e54 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do - {:ok, _object} = Fetcher.fetch_object_from_id(id, depth: args["depth"]) - case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do {:ok, _object} -> :ok -- cgit v1.2.3 From 9c0040124a9ea68cedca4959d32105ee6a6c3ee1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 16:28:05 -0500 Subject: Skip remote fetch jobs for unreachable instances --- lib/pleroma/workers/remote_fetcher_worker.ex | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index ff0286e54..d4fae33df 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -3,24 +3,29 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.RemoteFetcherWorker do + alias Pleroma.Instances alias Pleroma.Object.Fetcher use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do - case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do - {:ok, _object} -> - :ok + if Instances.reachable?(id) do + case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do + {:ok, _object} -> + :ok - {:error, reason = "Object fetch has been denied"} -> - {:cancel, reason} + {:error, reason = "Object fetch has been denied"} -> + {:cancel, reason} - {:error, reason = "Object has been deleted"} -> - {:cancel, reason} + {:error, reason = "Object has been deleted"} -> + {:cancel, reason} - _ -> - :error + _ -> + :error + end + else + {:cancel, "Unreachable instance"} end end -- cgit v1.2.3 From 73c4c6d7de6d33c68cf663e65df8525ce8eef4f5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 26 Dec 2023 17:12:58 -0500 Subject: Revert "Mark instances as unreachable when returning a 403 from an object fetch" This reverts commit d472bafec19cee269e7c943bafae7c805785acd7. --- lib/pleroma/object/fetcher.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 75f39fb6a..f1a0a483b 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -213,7 +213,6 @@ defmodule Pleroma.Object.Fetcher do end {:ok, %{status: 403}} -> - Instances.set_consistently_unreachable(id) {:error, "Object fetch has been denied"} {:ok, %{status: code}} when code in [404, 410] -> -- cgit v1.2.3 From 7a58ddfa486048d17fa653662914e0f02d11fadb Mon Sep 17 00:00:00 2001 From: tusooa Date: Sun, 5 Nov 2023 18:49:31 -0500 Subject: Allow local user to have group actor type https://git.pleroma.social/pleroma/pleroma/-/issues/3205 --- lib/pleroma/constants.ex | 8 ++++++++ lib/pleroma/user.ex | 3 ++- lib/pleroma/web/mastodon_api/views/account_view.ex | 10 +++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index 77bc4bfac..d814b4931 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -76,6 +76,14 @@ defmodule Pleroma.Constants do ] ) + const(allowed_user_actor_types, + do: [ + "Person", + "Service", + "Group" + ] + ) + # basic regex, just there to weed out potential mistakes # https://datatracker.ietf.org/doc/html/rfc2045#section-5.1 const(mime_regex, diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 10dafbe6f..0fd1b6365 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -39,6 +39,7 @@ defmodule Pleroma.User do alias Pleroma.Workers.BackgroundWorker require Logger + require Pleroma.Constants @type t :: %__MODULE__{} @type account_status :: @@ -579,7 +580,7 @@ defmodule Pleroma.User do |> validate_format(:nickname, local_nickname_regex()) |> validate_length(:bio, max: bio_limit) |> validate_length(:name, min: 1, max: name_limit) - |> validate_inclusion(:actor_type, ["Person", "Service"]) + |> validate_inclusion(:actor_type, Pleroma.Constants.allowed_user_actor_types()) |> put_fields() |> put_emoji() |> put_change_if_present(:bio, &{:ok, parse_bio(&1, struct)}) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 237de3055..e7c555eb2 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -212,7 +212,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do do: user.follower_count, else: 0 - bot = user.actor_type == "Service" + bot = is_bot?(user) emojis = Enum.map(user.emoji, fn {shortcode, raw_url} -> @@ -468,4 +468,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do defp image_url(%{"url" => [%{"href" => href} | _]}), do: href defp image_url(_), do: nil + + defp is_bot?(user) do + # Because older and/or Mastodon clients may not recognize a Group actor properly, + # and currently the group actor can only boost things, we should let these clients + # think groups are bots. + # See https://git.pleroma.social/pleroma/pleroma-meta/-/issues/14 + user.actor_type == "Service" || user.actor_type == "Group" + end end -- cgit v1.2.3 From 5459bbc1efabc373e5619990be8d811df072e5e6 Mon Sep 17 00:00:00 2001 From: tusooa Date: Mon, 6 Nov 2023 19:59:05 -0500 Subject: Allow group actors to boost posts --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + lib/pleroma/web/activity_pub/side_effects.ex | 2 ++ lib/pleroma/web/activity_pub/utils.ex | 17 +++++++++++++++++ 3 files changed, 20 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 32d1a1037..3b0140d96 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -319,6 +319,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do {:ok, _actor} <- update_last_status_at_if_public(actor, activity), _ <- notify_and_stream(activity), :ok <- maybe_schedule_poll_notifications(activity), + :ok <- maybe_handle_group_posts(activity), :ok <- maybe_federate(activity) do {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 10f268f05..59b19180d 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -233,6 +233,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + Utils.maybe_handle_group_posts(activity) + meta = meta |> add_notifications(notifications) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index b32f19740..e4edbd5ee 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -935,4 +935,21 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> where([a, object: o], fragment("(?)->>'type' = 'Answer'", o.data)) |> Repo.all() end + + def maybe_handle_group_posts(activity) do + mentions = + activity.data["to"] + |> Enum.filter(&(&1 != activity.actor)) + + mentioned_local_groups = + User.get_all_by_ap_id(mentions) + |> Enum.filter(&(&1.actor_type == "Group" and &1.local)) + + mentioned_local_groups + |> Enum.each(fn group -> + Pleroma.Web.CommonAPI.repeat(activity.id, group) + end) + + :ok + end end -- cgit v1.2.3 From e34a975dd946cc609638d85c20a57e2bfed6ebc7 Mon Sep 17 00:00:00 2001 From: tusooa Date: Tue, 7 Nov 2023 21:16:24 -0500 Subject: Do not boost if group is blocking poster --- lib/pleroma/web/activity_pub/utils.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index e4edbd5ee..e2fc2640d 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -937,13 +937,19 @@ defmodule Pleroma.Web.ActivityPub.Utils do end def maybe_handle_group_posts(activity) do + poster = User.get_cached_by_ap_id(activity.actor) + mentions = activity.data["to"] |> Enum.filter(&(&1 != activity.actor)) mentioned_local_groups = User.get_all_by_ap_id(mentions) - |> Enum.filter(&(&1.actor_type == "Group" and &1.local)) + |> Enum.filter(fn user -> + user.actor_type == "Group" and + user.local and + not User.blocks?(user, poster) + end) mentioned_local_groups |> Enum.each(fn group -> -- cgit v1.2.3 From b273025fd7612eb869c5a14a8cca81cbc45bad42 Mon Sep 17 00:00:00 2001 From: tusooa Date: Wed, 27 Dec 2023 12:21:05 -0500 Subject: Add pleroma:group_actors to instance features --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index f95b5360a..e514d0e91 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -125,7 +125,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do if Config.get([:instance, :profile_directory]) do "profile_directory" end, - "pleroma:get:main/ostatus" + "pleroma:get:main/ostatus", + "pleroma:group_actors" ] |> Enum.filter(& &1) end -- cgit v1.2.3 From 1a337dcc18f3dabb97bf480f5569e8787e5ce2cf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 12:43:12 -0500 Subject: These functions in Pleroma.Instances should be defdelegates to Pleroma.Instances.Instance --- lib/pleroma/instances.ex | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/instances.ex b/lib/pleroma/instances.ex index 782948f83..b6d83f591 100644 --- a/lib/pleroma/instances.ex +++ b/lib/pleroma/instances.ex @@ -7,16 +7,15 @@ defmodule Pleroma.Instances do alias Pleroma.Instances.Instance - def filter_reachable(urls_or_hosts), do: Instance.filter_reachable(urls_or_hosts) + defdelegate filter_reachable(urls_or_hosts), to: Instance - def reachable?(url_or_host), do: Instance.reachable?(url_or_host) + defdelegate reachable?(url_or_host), to: Instance - def set_reachable(url_or_host), do: Instance.set_reachable(url_or_host) + defdelegate set_reachable(url_or_host), to: Instance - def set_unreachable(url_or_host, unreachable_since \\ nil), - do: Instance.set_unreachable(url_or_host, unreachable_since) + defdelegate set_unreachable(url_or_host, unreachable_since \\ nil), to: Instance - def get_consistently_unreachable, do: Instance.get_consistently_unreachable() + defdelegate get_consistently_unreachable, to: Instance def set_consistently_unreachable(url_or_host), do: set_unreachable(url_or_host, reachability_datetime_threshold()) -- cgit v1.2.3 From d4c77103d1fe5df9b2ea8bd3429a8fc0240645c1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 15:27:37 -0500 Subject: Fix detection of user follower collection being private We were overzealous with matching on a raw error from the object fetch that should have never been relied on like this. If we can't fetch successfully we should assume that the collection is private. Building a more expressive and universal error struct to match on may be something to consider. --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 32d1a1037..39159b1c4 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1697,9 +1697,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Fetcher.fetch_and_contain_remote_object_from_id(first) do {:ok, false} else - {:error, {:ok, %{status: code}}} when code in [401, 403] -> {:ok, true} - {:error, _} = e -> e - e -> {:error, e} + {:error, _} -> {:ok, true} end end -- cgit v1.2.3 From f53197c82a90533c9152d7d8ed57c2604a2d6685 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 27 Dec 2023 23:52:46 +0100 Subject: Fix operation name typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec.ex | 6 +++--- .../web/api_spec/operations/admin/frontend_operation.ex | 4 ++-- .../web/api_spec/operations/admin/o_auth_app_operation.ex | 8 ++++---- lib/pleroma/web/api_spec/operations/admin/report_operation.ex | 10 +++++----- 4 files changed, 14 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 163226ce5..8a759185b 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -94,14 +94,14 @@ defmodule Pleroma.Web.ApiSpec do "tags" => [ "Chat administration", "Emoji pack administration", - "Frontend managment", + "Frontend management", "Instance configuration", "Instance documents", "Invites", "MediaProxy cache", - "OAuth application managment", + "OAuth application management", "Relays", - "Report managment", + "Report management", "Status administration", "User administration", "Announcement management" diff --git a/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex b/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex index 3e85c44d2..e17881b49 100644 --- a/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/frontend_operation.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do def index_operation do %Operation{ - tags: ["Frontend managment"], + tags: ["Frontend management"], summary: "Retrieve a list of available frontends", operationId: "AdminAPI.FrontendController.index", security: [%{"oAuth" => ["admin:read"]}], @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.FrontendOperation do def install_operation do %Operation{ - tags: ["Frontend managment"], + tags: ["Frontend management"], summary: "Install a frontend", operationId: "AdminAPI.FrontendController.install", security: [%{"oAuth" => ["admin:read"]}], diff --git a/lib/pleroma/web/api_spec/operations/admin/o_auth_app_operation.ex b/lib/pleroma/web/api_spec/operations/admin/o_auth_app_operation.ex index 1a05aff6a..2b2496c26 100644 --- a/lib/pleroma/web/api_spec/operations/admin/o_auth_app_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/o_auth_app_operation.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do def index_operation do %Operation{ summary: "Retrieve a list of OAuth applications", - tags: ["OAuth application managment"], + tags: ["OAuth application management"], operationId: "AdminAPI.OAuthAppController.index", security: [%{"oAuth" => ["admin:write"]}], parameters: [ @@ -69,7 +69,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do def create_operation do %Operation{ - tags: ["OAuth application managment"], + tags: ["OAuth application management"], summary: "Create an OAuth application", operationId: "AdminAPI.OAuthAppController.create", requestBody: request_body("Parameters", create_request()), @@ -84,7 +84,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do def update_operation do %Operation{ - tags: ["OAuth application managment"], + tags: ["OAuth application management"], summary: "Update OAuth application", operationId: "AdminAPI.OAuthAppController.update", parameters: [id_param() | admin_api_params()], @@ -102,7 +102,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.OAuthAppOperation do def delete_operation do %Operation{ - tags: ["OAuth application managment"], + tags: ["OAuth application management"], summary: "Delete OAuth application", operationId: "AdminAPI.OAuthAppController.delete", parameters: [id_param() | admin_api_params()], diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index 312e091a5..d7a74f665 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def index_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Retrieve a list of reports", operationId: "AdminAPI.ReportController.index", security: [%{"oAuth" => ["admin:read:reports"]}], @@ -69,7 +69,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def show_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Retrieve a report", operationId: "AdminAPI.ReportController.show", parameters: [id_param() | admin_api_params()], @@ -83,7 +83,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def update_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Change state of specified reports", operationId: "AdminAPI.ReportController.update", security: [%{"oAuth" => ["admin:write:reports"]}], @@ -99,7 +99,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def notes_create_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Add a note to the report", operationId: "AdminAPI.ReportController.notes_create", parameters: [id_param() | admin_api_params()], @@ -120,7 +120,7 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do def notes_delete_operation do %Operation{ - tags: ["Report managment"], + tags: ["Report management"], summary: "Delete note attached to the report", operationId: "AdminAPI.ReportController.notes_delete", parameters: [ -- cgit v1.2.3 From 017e35fbf128d47c033275a70b76b72f24d7c754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 28 Dec 2023 00:15:32 +0100 Subject: Fix some more typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/mix/tasks/pleroma/database.ex | 2 +- lib/mix/tasks/pleroma/digest.ex | 2 +- lib/mix/tasks/pleroma/ecto/rollback.ex | 2 +- lib/mix/tasks/pleroma/instance.ex | 2 +- lib/pleroma/captcha/kocaptcha.ex | 2 +- lib/pleroma/config/deprecation_warnings.ex | 8 ++++---- lib/pleroma/docs/generator.ex | 2 +- lib/pleroma/search/search_backend.ex | 2 +- lib/pleroma/user/query.ex | 2 +- lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex | 2 +- lib/pleroma/web/api_spec.ex | 2 +- lib/pleroma/web/api_spec/operations/account_operation.ex | 2 +- lib/pleroma/web/api_spec/operations/instance_operation.ex | 4 ++-- lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex | 2 +- lib/pleroma/web/api_spec/operations/status_operation.ex | 4 ++-- lib/pleroma/web/api_spec/operations/twitter_util_operation.ex | 6 +++--- lib/pleroma/web/api_spec/schemas/attachment.ex | 2 +- lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 2 +- lib/pleroma/web/o_auth/o_auth_controller.ex | 2 +- lib/pleroma/web/plugs/cache.ex | 2 +- lib/pleroma/web/plugs/uploaded_media.ex | 2 +- lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/streamer.ex | 2 +- lib/pleroma/web/twitter_api/controllers/password_controller.ex | 2 +- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 4 ++-- 25 files changed, 33 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ed560c177..93ee57dc3 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -193,7 +193,7 @@ defmodule Mix.Tasks.Pleroma.Database do "ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';" ) - # non-exist config will not raise excpetion but only give >0 messages + # non-exist config will not raise exception but only give >0 messages if length(msg) > 0 do shell_info("Error: #{inspect(msg, pretty: true)}") else diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex index aea9c8ac5..53cac0b94 100644 --- a/lib/mix/tasks/pleroma/digest.ex +++ b/lib/mix/tasks/pleroma/digest.ex @@ -30,7 +30,7 @@ defmodule Mix.Tasks.Pleroma.Digest do shell_info("Digest email have been sent to #{nickname} (#{user.email})") else _ -> - shell_info("Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}") + shell_info("Couldn't find any mentions for #{nickname} since #{last_digest_emailed_at}") end end end diff --git a/lib/mix/tasks/pleroma/ecto/rollback.ex b/lib/mix/tasks/pleroma/ecto/rollback.ex index 3d78eaec4..121890f39 100644 --- a/lib/mix/tasks/pleroma/ecto/rollback.ex +++ b/lib/mix/tasks/pleroma/ecto/rollback.ex @@ -61,7 +61,7 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do Logger.configure(level: :info) if opts[:env] == "test" do - Logger.info("Rollback succesfully") + Logger.info("Rollback successfully") else {:ok, _, _} = Ecto.Migrator.with_repo(Pleroma.Repo, &Ecto.Migrator.run(&1, path, :down, opts)) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 5d8b254a2..fd1809a42 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -292,7 +292,7 @@ defmodule Mix.Tasks.Pleroma.Instance do if db_configurable? do shell_info( - " Please transfer your config to the database after running database migrations. Refer to \"Transfering the config to/from the database\" section of the docs for more information." + " Please transfer your config to the database after running database migrations. Refer to \"Transferring the config to/from the database\" section of the docs for more information." ) end else diff --git a/lib/pleroma/captcha/kocaptcha.ex b/lib/pleroma/captcha/kocaptcha.ex index e786e28b9..c4987d4fd 100644 --- a/lib/pleroma/captcha/kocaptcha.ex +++ b/lib/pleroma/captcha/kocaptcha.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Captcha.Kocaptcha do @impl Service def validate(_token, captcha, answer_data) do - # Here the token is unsed, because the unencrypted captcha answer is just passed to method + # Here the token is unused, because the unencrypted captcha answer is just passed to method if not is_nil(captcha) and :crypto.hash(:md5, captcha) |> Base.encode16() == String.upcase(answer_data), do: :ok, diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index 1cd3241ea..a77923264 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -172,7 +172,7 @@ defmodule Pleroma.Config.DeprecationWarnings do ``` config :pleroma, :mrf, - transparency_exclusions: [{"instance.tld", "Reason to exlude transparency"}] + transparency_exclusions: [{"instance.tld", "Reason to exclude transparency"}] ``` """) @@ -213,7 +213,7 @@ defmodule Pleroma.Config.DeprecationWarnings do check_gun_pool_options(), check_activity_expiration_config(), check_remote_ip_plug_name(), - check_uploders_s3_public_endpoint(), + check_uploaders_s3_public_endpoint(), check_old_chat_shoutbox(), check_quarantined_instances_tuples(), check_transparency_exclusions_tuples(), @@ -372,8 +372,8 @@ defmodule Pleroma.Config.DeprecationWarnings do ) end - @spec check_uploders_s3_public_endpoint() :: :ok | nil - def check_uploders_s3_public_endpoint do + @spec check_uploaders_s3_public_endpoint() :: :ok | nil + def check_uploaders_s3_public_endpoint do s3_config = Pleroma.Config.get([Pleroma.Uploaders.S3]) use_old_config = Keyword.has_key?(s3_config, :public_endpoint) diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 456a8fd54..93b19484f 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Docs.Generator do :code.all_loaded() |> Enum.filter(fn {module, _} -> # This shouldn't be needed as all modules are expected to have module_info/1, - # but in test enviroments some transient modules `:elixir_compiler_XX` + # but in test environments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). Code.ensure_loaded(module) diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index a42e2f5f6..46be84551 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Search.SearchBackend do Remove the object from the index. Just the object, as opposed to the whole activity, is passed, since the object - is what contains the actual content and there is no need for fitlering when removing + is what contains the actual content and there is no need for filtering when removing from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: {:ok, any()} | {:error, any()} diff --git a/lib/pleroma/user/query.ex b/lib/pleroma/user/query.ex index 3e090cac0..874e8ec2b 100644 --- a/lib/pleroma/user/query.ex +++ b/lib/pleroma/user/query.ex @@ -22,7 +22,7 @@ defmodule Pleroma.User.Query do - pass non empty string - e.g. Pleroma.User.Query.build(%{email: "email@example.com"}) - *contains criteria* - - add field to @containns_criteria list + - add field to @contains_criteria list - pass values list - e.g. Pleroma.User.Query.build(%{ap_id: ["http://ap_id1", "http://ap_id2"]}) """ diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index b73fd974c..d13d980cc 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do alias Pleroma.Object @moduledoc """ - Reject, TWKN-remove or Set-Sensitive messsages with specific hashtags (without the leading #) + Reject, TWKN-remove or Set-Sensitive messages with specific hashtags (without the leading #) Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. """ diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 8a759185b..3588608f2 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.ApiSpec do - [Mastodon API documentation](https://docs.joinmastodon.org/client/intro/) - [Differences in Mastodon API responses from vanilla Mastodon](https://docs-develop.pleroma.social/backend/development/API/differences_in_mastoapi_responses/) - Please report such occurences on our [issue tracker](https://git.pleroma.social/pleroma/pleroma/-/issues). Feel free to submit API questions or proposals there too! + Please report such occurrences on our [issue tracker](https://git.pleroma.social/pleroma/pleroma/-/issues). Feel free to submit API questions or proposals there too! """, # Strip environment from the version version: Application.spec(:pleroma, :vsn) |> to_string() |> String.replace(~r/\+.*$/, ""), diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index f2897a3a3..75cea2184 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -347,7 +347,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do summary: "Endorse", operationId: "AccountController.endorse", security: [%{"oAuth" => ["follow", "write:accounts"]}], - description: "Addds the given account to endorsed accounts list.", + description: "Adds the given account to endorsed accounts list.", parameters: [%Reference{"$ref": "#/components/parameters/accountIdOrNickname"}], responses: %{ 200 => Operation.response("Relationship", "application/json", AccountRelationship), diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 8e395bde8..708b74b12 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -101,7 +101,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do languages: %Schema{ type: :array, items: %Schema{type: :string}, - description: "Primary langauges of the website and its staff" + description: "Primary languages of the website and its staff" }, registrations: %Schema{type: :boolean, description: "Whether registrations are enabled"}, # Extra (not present in Mastodon): @@ -252,7 +252,7 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do languages: %Schema{ type: :array, items: %Schema{type: :string}, - description: "Primary langauges of the website and its staff" + description: "Primary languages of the website and its staff" }, registrations: %Schema{ type: :object, diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index 141b60533..f595583b6 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do security: [%{"oAuth" => ["write"]}], operationId: "PleromaAPI.ScrobbleController.create", deprecated: true, - requestBody: request_body("Parameters", create_request(), requried: true), + requestBody: request_body("Parameters", create_request(), required: true), responses: %{ 200 => Operation.response("Scrobble", "application/json", scrobble()) } diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index c133a3aac..f52372c18 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -534,7 +534,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do format: :"date-time", nullable: true, description: - "ISO 8601 Datetime at which to schedule a status. Providing this paramter will cause ScheduledStatus to be returned instead of Status. Must be at least 5 minutes in the future." + "ISO 8601 Datetime at which to schedule a status. Providing this parameter will cause ScheduledStatus to be returned instead of Status. Must be at least 5 minutes in the future." }, language: %Schema{ type: :string, @@ -546,7 +546,7 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do allOf: [BooleanLike], nullable: true, description: - "If set to `true` the post won't be actually posted, but the status entitiy would still be rendered back. This could be useful for previewing rich text/custom emoji, for example" + "If set to `true` the post won't be actually posted, but the status entity would still be rendered back. This could be useful for previewing rich text/custom emoji, for example" }, content_type: %Schema{ type: :string, diff --git a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex index 084329ad7..87af0d526 100644 --- a/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex +++ b/lib/pleroma/web/api_spec/operations/twitter_util_operation.ex @@ -87,7 +87,7 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do defp change_password_request do %Schema{ title: "ChangePasswordRequest", - description: "POST body for changing the account's passowrd", + description: "POST body for changing the account's password", type: :object, required: [:password, :new_password, :new_password_confirmation], properties: %{ @@ -136,12 +136,12 @@ defmodule Pleroma.Web.ApiSpec.TwitterUtilOperation do } end - def update_notificaton_settings_operation do + def update_notification_settings_operation do %Operation{ tags: ["Settings"], summary: "Update Notification Settings", security: [%{"oAuth" => ["write:accounts"]}], - operationId: "UtilController.update_notificaton_settings", + operationId: "UtilController.update_notification_settings", parameters: [ Operation.parameter( :block_from_strangers, diff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex index 48634a14f..2871b5f99 100644 --- a/lib/pleroma/web/api_spec/schemas/attachment.ex +++ b/lib/pleroma/web/api_spec/schemas/attachment.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do title: "Attachment", description: "Represents a file or media attachment that can be added to a status.", type: :object, - requried: [:id, :url, :preview_url], + required: [:id, :url, :preview_url], properties: %{ id: %Schema{type: :string, description: "The ID of the attachment in the database."}, url: %Schema{ diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 9a4b56301..1b5de4b45 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -235,7 +235,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do # So we first build the normal local changeset, then apply it to the # user data, but don't persist it. With this, we generate the object # data for our update activity. We feed this and the changeset as meta - # inforation into the pipeline, where they will be properly updated and + # information into the pipeline, where they will be properly updated and # federated. with changeset <- User.update_changeset(user, user_params), {:ok, unpersisted_user} <- Ecto.Changeset.apply_action(changeset, :update), diff --git a/lib/pleroma/web/o_auth/o_auth_controller.ex b/lib/pleroma/web/o_auth/o_auth_controller.ex index c1fb4f378..2bbe5d5fa 100644 --- a/lib/pleroma/web/o_auth/o_auth_controller.ex +++ b/lib/pleroma/web/o_auth/o_auth_controller.ex @@ -310,7 +310,7 @@ defmodule Pleroma.Web.OAuth.OAuthController do after_token_exchange(conn, %{token: token}) else _error -> - handle_token_exchange_error(conn, :invalid_credentails) + handle_token_exchange_error(conn, :invalid_credentials) end end diff --git a/lib/pleroma/web/plugs/cache.ex b/lib/pleroma/web/plugs/cache.ex index 667477857..5a7e86ef7 100644 --- a/lib/pleroma/web/plugs/cache.ex +++ b/lib/pleroma/web/plugs/cache.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.Plugs.Cache do - `ttl`: An expiration time (time-to-live). This value should be in milliseconds or `nil` to disable expiration. Defaults to `nil`. - `query_params`: Take URL query string into account (`true`), ignore it (`false`) or limit to specific params only (list). Defaults to `true`. - - `tracking_fun`: A function that is called on successfull responses, no matter if the request is cached or not. It should accept a conn as the first argument and the value assigned to `tracking_fun_data` as the second. + - `tracking_fun`: A function that is called on successful responses, no matter if the request is cached or not. It should accept a conn as the first argument and the value assigned to `tracking_fun_data` as the second. Additionally, you can overwrite the TTL inside a controller action by assigning `cache_ttl` to the connection struct: diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index 8b3bc9acb..f1076da1b 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -105,7 +105,7 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do end defp get_media(conn, unknown, _, _) do - Logger.error("#{__MODULE__}: Unknown get startegy: #{inspect(unknown)}") + Logger.error("#{__MODULE__}: Unknown get strategy: #{inspect(unknown)}") conn |> send_resp(:internal_server_error, dgettext("errors", "Internal Error")) diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 22c85e34b..00268b121 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -481,7 +481,7 @@ defmodule Pleroma.Web.Router do post("/change_email", UtilController, :change_email) post("/change_password", UtilController, :change_password) post("/delete_account", UtilController, :delete_account) - put("/notification_settings", UtilController, :update_notificaton_settings) + put("/notification_settings", UtilController, :update_notification_settings) post("/disable_account", UtilController, :disable_account) post("/move_account", UtilController, :move_account) diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 48ca82421..aaee79d8e 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -396,7 +396,7 @@ defmodule Pleroma.Web.Streamer do end end - # In test environement, only return true if the registry is started. + # In test environment, only return true if the registry is started. # In benchmark environment, returns false. # In any other environment, always returns true. cond do diff --git a/lib/pleroma/web/twitter_api/controllers/password_controller.ex b/lib/pleroma/web/twitter_api/controllers/password_controller.ex index 31b7dd728..e5482de9d 100644 --- a/lib/pleroma/web/twitter_api/controllers/password_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/password_controller.ex @@ -4,7 +4,7 @@ defmodule Pleroma.Web.TwitterAPI.PasswordController do @moduledoc """ - The module containts functions for reset password. + The module contains functions for password reset. """ use Pleroma.Web, :controller diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index ca8a98960..50f8f803f 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -35,7 +35,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do :change_email, :change_password, :delete_account, - :update_notificaton_settings, + :update_notification_settings, :disable_account, :move_account, :add_alias, @@ -181,7 +181,7 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do json(conn, emoji) end - def update_notificaton_settings(%{assigns: %{user: user}} = conn, params) do + def update_notification_settings(%{assigns: %{user: user}} = conn, params) do with {:ok, _} <- User.update_notification_settings(user, params) do json(conn, %{status: "success"}) end -- cgit v1.2.3 From 6c9929b809290d9caca6a7a40eb0f59af6dd558d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 20:18:14 -0500 Subject: Set Logger level to error --- lib/pleroma/object/fetcher.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index f1a0a483b..9ba0ac0ca 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -76,15 +76,15 @@ defmodule Pleroma.Object.Fetcher do {:error, "Max thread distance exceeded."} {:containment, e} -> - Logger.info("Error while fetching #{id}: Object containment failed. #{inspect(e)}") + Logger.error("Error while fetching #{id}: Object containment failed. #{inspect(e)}") {:error, e} {:transmogrifier, {:error, {:reject, e}}} -> - Logger.info("Rejected #{id} while fetching: #{inspect(e)}") + Logger.error("Rejected #{id} while fetching: #{inspect(e)}") {:reject, e} {:transmogrifier, {:reject, e}} -> - Logger.info("Rejected #{id} while fetching: #{inspect(e)}") + Logger.error("Rejected #{id} while fetching: #{inspect(e)}") {:reject, e} {:transmogrifier, _} = e -> -- cgit v1.2.3 From becb070603bc65d5302698a2b6cf5f89b3d5c1f0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 20:47:18 -0500 Subject: Conslidate log messages for object fetcher failures and leverage Logger.metadata --- lib/pleroma/object/fetcher.ex | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 9ba0ac0ca..72420519a 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -72,23 +72,25 @@ defmodule Pleroma.Object.Fetcher do {:object, data, Object.normalize(activity, fetch: false)} do {:ok, object} else - {:allowed_depth, false} -> + {:allowed_depth, false} = e -> + log_fetch_error(id, e) {:error, "Max thread distance exceeded."} - {:containment, e} -> - Logger.error("Error while fetching #{id}: Object containment failed. #{inspect(e)}") - {:error, e} + {:containment, reason} = e -> + log_fetch_error(id, e) + {:error, reason} - {:transmogrifier, {:error, {:reject, e}}} -> - Logger.error("Rejected #{id} while fetching: #{inspect(e)}") - {:reject, e} + {:transmogrifier, {:error, {:reject, reason}}} = e -> + log_fetch_error(id, e) + {:reject, reason} - {:transmogrifier, {:reject, e}} -> - Logger.error("Rejected #{id} while fetching: #{inspect(e)}") - {:reject, e} + {:transmogrifier, {:reject, reason}} = e -> + log_fetch_error(id, e) + {:reject, reason} - {:transmogrifier, _} = e -> - {:error, e} + {:transmogrifier, reason} = e -> + log_fetch_error(id, e) + {:error, reason} {:object, data, nil} -> reinject_object(%Object{}, data) @@ -99,16 +101,21 @@ defmodule Pleroma.Object.Fetcher do {:fetch_object, %Object{} = object} -> {:ok, object} - {:fetch, {:error, error}} -> - Logger.error("Error while fetching #{id}: #{inspect(error)}") - {:error, error} + {:fetch, {:error, reason}} = e -> + log_fetch_error(id, e) + {:error, reason} e -> - Logger.error("Error while fetching #{id}: #{inspect(e)}") + log_fetch_error(id, e) {:error, e} end end + defp log_fetch_error(id, error) do + Logger.metadata([object: id]) + Logger.error("Object rejected while fetching #{id} #{inspect(error)}") + end + defp prepare_activity_params(data) do %{ "type" => "Create", -- cgit v1.2.3 From 882267e3eceaa61cff16a9e0dee231d54a80d927 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 21:39:17 -0500 Subject: Remove duplicate log messages from Transmogrifier Object fetch errors are logged in the fetcher module --- lib/pleroma/web/activity_pub/transmogrifier.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 35f3aea03..80f64cf1e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -23,7 +23,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do import Ecto.Query - require Logger require Pleroma.Constants @doc """ @@ -155,8 +154,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.put("context", replied_object.data["context"] || object["conversation"]) |> Map.drop(["conversation", "inReplyToAtomUri"]) else - e -> - Logger.warning("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") + _ -> object end else @@ -181,8 +179,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do {:quoting?, _} -> object - e -> - Logger.warning("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") + _ -> object end end @@ -852,8 +849,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do relative_object do Map.put(data, "object", external_url) else - {:fetch, e} -> - Logger.error("Couldn't fetch #{object} #{inspect(e)}") + {:fetch, _} -> data _ -> -- cgit v1.2.3 From 287f2c9719310f96ce37b445f26e069f804c34b2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 21:55:07 -0500 Subject: Formatting --- lib/pleroma/object/fetcher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 72420519a..5bc1cfd5e 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -112,7 +112,7 @@ defmodule Pleroma.Object.Fetcher do end defp log_fetch_error(id, error) do - Logger.metadata([object: id]) + Logger.metadata(object: id) Logger.error("Object rejected while fetching #{id} #{inspect(error)}") end -- cgit v1.2.3 From a2708f7fe31c6009b0c9954e5de3b74c45f9818f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 21:57:47 -0500 Subject: Leverage existing atoms as return errors for the object fetcher --- lib/pleroma/object/fetcher.ex | 4 ++-- lib/pleroma/workers/remote_fetcher_worker.ex | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 5bc1cfd5e..0c90e0eaa 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -220,10 +220,10 @@ defmodule Pleroma.Object.Fetcher do end {:ok, %{status: 403}} -> - {:error, "Object fetch has been denied"} + {:error, :forbidden} {:ok, %{status: code}} when code in [404, 410] -> - {:error, "Object has been deleted"} + {:error, :not_found} {:error, e} -> {:error, e} diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index d4fae33df..dca4530d7 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -15,11 +15,11 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:ok, _object} -> :ok - {:error, reason = "Object fetch has been denied"} -> - {:cancel, reason} + {:error, :forbidden} -> + {:cancel, :forbidden} - {:error, reason = "Object has been deleted"} -> - {:cancel, reason} + {:error, :not_found} -> + {:cancel, :not_found} _ -> :error -- cgit v1.2.3 From ad0a5deb67f454b0529a4faf72399cd9ecc9c0e6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 27 Dec 2023 22:28:41 -0500 Subject: Prevent requeuing Remote Fetcher jobs that exceed thread depth --- lib/pleroma/object/fetcher.ex | 2 +- lib/pleroma/workers/remote_fetcher_worker.ex | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 0c90e0eaa..972dfaf28 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -74,7 +74,7 @@ defmodule Pleroma.Object.Fetcher do else {:allowed_depth, false} = e -> log_fetch_error(id, e) - {:error, "Max thread distance exceeded."} + {:error, :allowed_depth} {:containment, reason} = e -> log_fetch_error(id, e) diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index dca4530d7..fef8a26b2 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -21,6 +21,9 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:error, :not_found} -> {:cancel, :not_found} + {:error, :allowed_depth} -> + {:cancel, :allowed_depth} + _ -> :error end -- cgit v1.2.3 From 1d816222e00741324fe3a068fb69f16675067a56 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 11:15:43 -0500 Subject: Remove support for multiple federation publisher modules This also unravels some needless indirection. --- lib/pleroma/web/federator.ex | 2 +- lib/pleroma/web/federator/publisher.ex | 46 ---------------------------------- lib/pleroma/web/nodeinfo/nodeinfo.ex | 2 +- lib/pleroma/web/web_finger.ex | 2 +- 4 files changed, 3 insertions(+), 49 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 8621d984c..6deebb963 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -6,9 +6,9 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Activity alias Pleroma.Object.Containment alias Pleroma.User + alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Federator.Publisher alias Pleroma.Workers.PublisherWorker alias Pleroma.Workers.ReceiverWorker diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index 8c6547208..03509bbaf 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -3,8 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Federator.Publisher do - alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.User alias Pleroma.Workers.PublisherWorker @@ -38,50 +36,6 @@ defmodule Pleroma.Web.Federator.Publisher do ) end - @doc """ - Relays an activity to all specified peers. - """ - @callback publish(User.t(), Activity.t()) :: :ok | {:error, any()} - - @spec publish(User.t(), Activity.t()) :: :ok - def publish(%User{} = user, %Activity{} = activity) do - Config.get([:instance, :federation_publisher_modules]) - |> Enum.each(fn module -> - if module.is_representable?(activity) do - Logger.debug("Publishing #{activity.data["id"]} using #{inspect(module)}") - module.publish(user, activity) - end - end) - - :ok - end - - @doc """ - Gathers links used by an outgoing federation module for WebFinger output. - """ - @callback gather_webfinger_links(User.t()) :: list() - - @spec gather_webfinger_links(User.t()) :: list() - def gather_webfinger_links(%User{} = user) do - Config.get([:instance, :federation_publisher_modules]) - |> Enum.reduce([], fn module, links -> - links ++ module.gather_webfinger_links(user) - end) - end - - @doc """ - Gathers nodeinfo protocol names supported by the federation module. - """ - @callback gather_nodeinfo_protocol_names() :: list() - - @spec gather_nodeinfo_protocol_names() :: list() - def gather_nodeinfo_protocol_names do - Config.get([:instance, :federation_publisher_modules]) - |> Enum.reduce([], fn module, links -> - links ++ module.gather_nodeinfo_protocol_names() - end) - end - @doc """ Gathers a set of remote users given an IR envelope. """ diff --git a/lib/pleroma/web/nodeinfo/nodeinfo.ex b/lib/pleroma/web/nodeinfo/nodeinfo.ex index 9e27ac26c..4d5a9a57f 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.Nodeinfo.Nodeinfo do alias Pleroma.Config alias Pleroma.Stats alias Pleroma.User - alias Pleroma.Web.Federator.Publisher + alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.MastodonAPI.InstanceView # returns a nodeinfo 2.0 map, since 2.1 just adds a repository field diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 0684a770c..26fb8af84 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.WebFinger do alias Pleroma.HTTP alias Pleroma.User + alias Pleroma.Web.ActivityPub.Publisher alias Pleroma.Web.Endpoint - alias Pleroma.Web.Federator.Publisher alias Pleroma.Web.XML alias Pleroma.XmlBuilder require Jason -- cgit v1.2.3 From 3acfdb6f8a8e242b87d7f81255c1c2ff8cd3171d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 14:53:09 -0500 Subject: Retire the Pleroma.Web.Federator.Publisher module --- lib/pleroma/web/activity_pub/publisher.ex | 48 ++++++++++++++++++++--- lib/pleroma/web/federator/publisher.ex | 64 ------------------------------- lib/pleroma/workers/publisher_worker.ex | 4 +- 3 files changed, 44 insertions(+), 72 deletions(-) delete mode 100644 lib/pleroma/web/federator/publisher.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index a580994b1..bb8209232 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -13,19 +13,56 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.User alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Workers.PublisherWorker require Pleroma.Constants import Pleroma.Web.ActivityPub.Visibility - @behaviour Pleroma.Web.Federator.Publisher - require Logger @moduledoc """ ActivityPub outgoing federation module. """ + @doc """ + Enqueue publishing a single activity. + """ + @spec enqueue_one(Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} + def enqueue_one(%{} = params, worker_args \\ []) do + PublisherWorker.enqueue( + "publish_one", + %{"params" => params}, + worker_args + ) + end + + @doc """ + Gathers a set of remote users given an IR envelope. + """ + def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do + cc = Map.get(data, "cc", []) + + bcc = + data + |> Map.get("bcc", []) + |> Enum.reduce([], fn ap_id, bcc -> + case Pleroma.List.get_by_ap_id(ap_id) do + %Pleroma.List{user_id: ^user_id} = list -> + {:ok, following} = Pleroma.List.get_following(list) + bcc ++ Enum.map(following, & &1.ap_id) + + _ -> + bcc + end + end) + + [to, cc, bcc] + |> Enum.concat() + |> Enum.map(&User.get_cached_by_ap_id/1) + |> Enum.filter(fn user -> user && !user.local end) + end + @doc """ Determine if an activity can be represented by running it through Transmogrifier. """ @@ -138,7 +175,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [] end - mentioned = Pleroma.Web.Federator.Publisher.remote_users(actor, activity) + mentioned = remote_users(actor, activity) non_mentioned = (followers ++ fetchers) -- mentioned [mentioned, non_mentioned] @@ -223,7 +260,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do |> Map.put("cc", cc) |> Jason.encode!() - Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ + __MODULE__.enqueue_one(%{ inbox: inbox, json: json, actor_id: actor.id, @@ -262,8 +299,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do inboxes |> Instances.filter_reachable() |> Enum.each(fn {inbox, unreachable_since} -> - Pleroma.Web.Federator.Publisher.enqueue_one( - __MODULE__, + __MODULE__.enqueue_one( %{ inbox: inbox, json: json, diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex deleted file mode 100644 index 03509bbaf..000000000 --- a/lib/pleroma/web/federator/publisher.ex +++ /dev/null @@ -1,64 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.Federator.Publisher do - alias Pleroma.User - alias Pleroma.Workers.PublisherWorker - - require Logger - - @moduledoc """ - Defines the contract used by federation implementations to publish messages to - their peers. - """ - - @doc """ - Determine whether an activity can be relayed using the federation module. - """ - @callback is_representable?(Pleroma.Activity.t()) :: boolean() - - @doc """ - Relays an activity to a specified peer, determined by the parameters. The - parameters used are controlled by the federation module. - """ - @callback publish_one(Map.t()) :: {:ok, Map.t()} | {:error, any()} - - @doc """ - Enqueue publishing a single activity. - """ - @spec enqueue_one(module(), Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} - def enqueue_one(module, %{} = params, worker_args \\ []) do - PublisherWorker.enqueue( - "publish_one", - %{"module" => to_string(module), "params" => params}, - worker_args - ) - end - - @doc """ - Gathers a set of remote users given an IR envelope. - """ - def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do - cc = Map.get(data, "cc", []) - - bcc = - data - |> Map.get("bcc", []) - |> Enum.reduce([], fn ap_id, bcc -> - case Pleroma.List.get_by_ap_id(ap_id) do - %Pleroma.List{user_id: ^user_id} = list -> - {:ok, following} = Pleroma.List.get_following(list) - bcc ++ Enum.map(following, & &1.ap_id) - - _ -> - bcc - end - end) - - [to, cc, bcc] - |> Enum.concat() - |> Enum.map(&User.get_cached_by_ap_id/1) - |> Enum.filter(fn user -> user && !user.local end) - end -end diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index 598ae3779..63fcf4ac2 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -18,9 +18,9 @@ defmodule Pleroma.Workers.PublisherWorker do Federator.perform(:publish, activity) end - def perform(%Job{args: %{"op" => "publish_one", "module" => module_name, "params" => params}}) do + def perform(%Job{args: %{"op" => "publish_one", "params" => params}}) do params = Map.new(params, fn {k, v} -> {String.to_atom(k), v} end) - Federator.perform(:publish_one, String.to_atom(module_name), params) + Federator.perform(:publish_one, params) end @impl Oban.Worker -- cgit v1.2.3 From 08ba9a15b2b5fe6896197ab947bc59d57f4845c8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 22:51:47 -0500 Subject: Fix the Federator perform/2 Oban callback --- lib/pleroma/web/federator.ex | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 6deebb963..1f2c3835a 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -68,10 +68,8 @@ defmodule Pleroma.Web.Federator do # Job Worker Callbacks - @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()} - def perform(:publish_one, module, params) do - apply(module, :publish_one, [params]) - end + @spec perform(atom(), any()) :: {:ok, any()} | {:error, any()} + def perform(:publish_one, params), do: Publisher.publish_one(params) def perform(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) -- cgit v1.2.3 From efd50759d587977a52db01ef6ececc961c736604 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 22:59:27 -0500 Subject: Log errors when publishing activities --- lib/pleroma/web/activity_pub/publisher.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index bb8209232..e0dd2e7c9 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -117,8 +117,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do result else - {_post_result, response} -> + {_post_result, response} = e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) + Logger.error("Failed to publish activity #{id} #{inspect(e)}") {:error, response} end end -- cgit v1.2.3 From aa070c7dafbceb33b9656f54aa552672497942f6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 23:09:33 -0500 Subject: Handle 401s as I have observed it in the wild --- lib/pleroma/object/fetcher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index 972dfaf28..af5642af4 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -219,7 +219,7 @@ defmodule Pleroma.Object.Fetcher do {:error, {:content_type, nil}} end - {:ok, %{status: 403}} -> + {:ok, %{status: code}} when code in [401, 403] -> {:error, :forbidden} {:ok, %{status: code}} when code in [404, 410] -> -- cgit v1.2.3 From 39dc6c65ef7a95412a985a3edce019914af12df8 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 29 Dec 2023 05:23:01 +0100 Subject: ChatMessage: Tolerate attachment field set to an empty array Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3224 --- .../web/activity_pub/object_validators/chat_message_validator.ex | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index efae48cae..09e25be89 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -57,6 +57,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do |> Map.put("attachment", attachment) end + def fix_attachment(%{"attachment" => attachment} = data) when attachment == [] do + data + |> Map.drop(["attachment"]) + end + def fix_attachment(data), do: data def changeset(struct, data) do -- cgit v1.2.3 From f17f92105bff555d2d372ff2ec053fe40fa1b41b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 28 Dec 2023 23:52:05 -0500 Subject: Oban jobs should be discarded on permanent errors --- lib/pleroma/workers/remote_fetcher_worker.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index fef8a26b2..7919969aa 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -16,19 +16,19 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do :ok {:error, :forbidden} -> - {:cancel, :forbidden} + {:discard, :forbidden} {:error, :not_found} -> - {:cancel, :not_found} + {:discard, :not_found} {:error, :allowed_depth} -> - {:cancel, :allowed_depth} + {:discard, :allowed_depth} _ -> :error end else - {:cancel, "Unreachable instance"} + {:discard, "Unreachable instance"} end end -- cgit v1.2.3 From 77949d4590b2a82ef6bb4c79f0777962991e28b1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 00:25:11 -0500 Subject: Make the Publisher log error less noisy --- lib/pleroma/web/activity_pub/publisher.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e0dd2e7c9..103b00f11 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -119,7 +119,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do else {_post_result, response} = e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) - Logger.error("Failed to publish activity #{id} #{inspect(e)}") + Logger.metadata(activity: id, inbox: inbox, status: code) + Logger.error("Publisher failed to inbox #{inbox} with status #{code}") {:error, response} end end -- cgit v1.2.3 From 7ebca7ecfa93f41d9eac2dcefa4b2e55f1b0c4ac Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 00:25:33 -0500 Subject: Activity publishing failures will prevent the job from retrying if the publishing request returns a 403 or 410 --- lib/pleroma/web/activity_pub/publisher.ex | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 103b00f11..1ce920e81 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -117,11 +117,16 @@ defmodule Pleroma.Web.ActivityPub.Publisher do result else - {_post_result, response} = e -> + {_post_result, %{status: code} = response} = e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) Logger.metadata(activity: id, inbox: inbox, status: code) Logger.error("Publisher failed to inbox #{inbox} with status #{code}") - {:error, response} + + case response do + %{status: 403} -> {:discard, :forbidden} + %{status: 410} -> {:discard, :not_found} + _ -> {:error, response} + end end end -- cgit v1.2.3 From 141702538b7ebdc501425b09928332d345c2b577 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 00:31:05 -0500 Subject: Discard on a 404 as well --- lib/pleroma/web/activity_pub/publisher.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 1ce920e81..282cc672a 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -124,6 +124,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do case response do %{status: 403} -> {:discard, :forbidden} + %{status: 404} -> {:discard, :not_found} %{status: 410} -> {:discard, :not_found} _ -> {:error, response} end -- cgit v1.2.3 From 2950397d476b0fd015b28182572927539b88e8fb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 00:50:50 -0500 Subject: Fix following redirects with Finch --- lib/pleroma/http.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index d41061538..eec61cf14 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -106,6 +106,10 @@ defmodule Pleroma.HTTP do [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] end + defp adapter_middlewares({Tesla.Adapter.Finch, _}) do + [Tesla.Middleware.FollowRedirects] + end + defp adapter_middlewares(_) do if Pleroma.Config.get(:env) == :test do # Emulate redirects in test env, which are handled by adapters in other environments -- cgit v1.2.3 From 4afe211e50466606c898e3fbd08b617413ef2e16 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 01:04:05 -0500 Subject: Return the full tuple from Tesla --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 282cc672a..e44e6eac1 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -126,7 +126,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do %{status: 403} -> {:discard, :forbidden} %{status: 404} -> {:discard, :not_found} %{status: 410} -> {:discard, :not_found} - _ -> {:error, response} + _ -> {:error, e} end end end -- cgit v1.2.3 From 833117f5738c286864d525aea9a87d1a7c193ff3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 12:18:23 -0500 Subject: Fix tests Need to handle the edge case of no valid HTTP response which has no status code --- lib/pleroma/web/activity_pub/publisher.ex | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e44e6eac1..cc47b3d27 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -128,6 +128,12 @@ defmodule Pleroma.Web.ActivityPub.Publisher do %{status: 410} -> {:discard, :not_found} _ -> {:error, e} end + + e -> + unless params[:unreachable_since], do: Instances.set_unreachable(inbox) + Logger.metadata(activity: id, inbox: inbox) + Logger.error("Publisher failed to inbox #{inbox} #{inspect(e)}") + {:error, e} end end -- cgit v1.2.3 From 50edef5bc13d0407aaaf26c951ce7a4a8cd4db58 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 29 Dec 2023 14:12:44 -0500 Subject: Change QTFastStart to recover gracefully if it encounters an error during bitstring matching This fixes issues with internal errors when trying to serve the video --- lib/pleroma/helpers/qt_fast_start.ex | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/helpers/qt_fast_start.ex b/lib/pleroma/helpers/qt_fast_start.ex index 5711c7162..919fec84a 100644 --- a/lib/pleroma/helpers/qt_fast_start.ex +++ b/lib/pleroma/helpers/qt_fast_start.ex @@ -40,16 +40,21 @@ defmodule Pleroma.Helpers.QtFastStart do got_mdat, acc ) do - full_size = (size - 8) * 8 - <> = rest - - acc = [ - {fourcc, pos, pos + size, size, - <>} - | acc - ] - - fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc) + try do + full_size = (size - 8) * 8 + <> = rest + + acc = [ + {fourcc, pos, pos + size, size, + <>} + | acc + ] + + fix(rest, pos + size, got_moov || fourcc == "moov", got_mdat || fourcc == "mdat", acc) + rescue + _ -> + :abort + end end defp fix(<<>>, _pos, _, _, acc) do -- cgit v1.2.3 From 1b5168ae021fa9d6b154b25444dd96c812083ce7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 28 Nov 2023 20:46:41 +0000 Subject: Phoenix detects the webfinger requests with content-type application/jrd+json as "jrd" now --- lib/pleroma/web/router.ex | 2 +- lib/pleroma/web/web_finger/web_finger_controller.ex | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 00268b121..8ba845364 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -182,7 +182,7 @@ defmodule Pleroma.Web.Router do end pipeline :well_known do - plug(:accepts, ["json", "jrd+json", "xml", "xrd+xml"]) + plug(:accepts, ["json", "jrd", "jrd+json", "xml", "xrd+xml"]) end pipeline :config do diff --git a/lib/pleroma/web/web_finger/web_finger_controller.ex b/lib/pleroma/web/web_finger/web_finger_controller.ex index 9e5efb77f..021df9bc5 100644 --- a/lib/pleroma/web/web_finger/web_finger_controller.ex +++ b/lib/pleroma/web/web_finger/web_finger_controller.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Web.WebFinger.WebFingerController do end def webfinger(%{assigns: %{format: format}} = conn, %{"resource" => resource}) - when format in ["json", "jrd+json"] do + when format in ["jrd", "json", "jrd+json"] do with {:ok, response} <- WebFinger.webfinger(resource, "JSON") do json(conn, response) else -- cgit v1.2.3 From 6af49270a9b1ddbdf8836139597b66695d8e1606 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 2 Jan 2024 15:12:33 +0100 Subject: MRF: Log sensible error for subdomains_regex --- lib/pleroma/web/activity_pub/mrf.ex | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 7f6dce925..1071f8e6e 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors +# Copyright © 2017-2023 Pleroma Authors # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF do @@ -139,7 +139,16 @@ defmodule Pleroma.Web.ActivityPub.MRF do @spec subdomains_regex([String.t()]) :: [Regex.t()] def subdomains_regex(domains) when is_list(domains) do - for domain <- domains, do: ~r(^#{String.replace(domain, "*.", "(.*\\.)*")}$)i + for domain <- domains do + try do + target = String.replace(domain, "*.", "(.*\\.)*") + ~r<^#{target}$>i + rescue + e -> + Logger.error("MRF: Invalid subdomain Regex: #{domain}") + reraise e, __STACKTRACE__ + end + end end @spec subdomain_match?([Regex.t()], String.t()) :: boolean() -- cgit v1.2.3 From 32d8e0d496265f2ebe15199bede63a1f57f043cd Mon Sep 17 00:00:00 2001 From: Alexander Tumin Date: Thu, 4 Jan 2024 16:41:27 +0300 Subject: Fix authentication check on account rendering when bio is defined --- lib/pleroma/web/mastodon_api/views/account_view.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index e7c555eb2..df8fdc8b8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -194,6 +194,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do end defp do_render("show.json", %{user: user} = opts) do + self = opts[:for] == user + user = User.sanitize_html(user, User.html_filter_policy(opts[:for])) display_name = user.name || user.nickname @@ -203,12 +205,12 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do header_static = User.banner_url(user) |> MediaProxy.preview_url(static: true) following_count = - if !user.hide_follows_count or !user.hide_follows or opts[:for] == user, + if !user.hide_follows_count or !user.hide_follows or self, do: user.following_count, else: 0 followers_count = - if !user.hide_followers_count or !user.hide_followers or opts[:for] == user, + if !user.hide_followers_count or !user.hide_followers or self, do: user.follower_count, else: 0 -- cgit v1.2.3 From 69e4ebbb8ea5d273c4a82ee8dcc275da8ee1dace Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 7 Jan 2024 15:28:40 +0300 Subject: Make remote emoji packs API use specifically the V1 URL Akkoma does not understand it without V1, and it works either way with normal pleroma, so no reason to not do this --- lib/pleroma/emoji/pack.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 6e58f8898..49bb93799 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -209,7 +209,9 @@ defmodule Pleroma.Emoji.Pack do with :ok <- validate_shareable_packs_available(uri) do uri - |> URI.merge("/api/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}") + |> URI.merge( + "/api/v1/pleroma/emoji/packs?page=#{opts[:page]}&page_size=#{opts[:page_size]}" + ) |> http_get() end end @@ -250,7 +252,7 @@ defmodule Pleroma.Emoji.Pack do with :ok <- validate_shareable_packs_available(uri), {:ok, remote_pack} <- - uri |> URI.merge("/api/pleroma/emoji/pack?name=#{name}") |> http_get(), + uri |> URI.merge("/api/v1/pleroma/emoji/pack?name=#{name}") |> http_get(), {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name), {:ok, archive} <- download_archive(url, sha), pack <- copy_as(remote_pack, as || name), @@ -592,7 +594,7 @@ defmodule Pleroma.Emoji.Pack do {:ok, %{ sha: sha, - url: URI.merge(uri, "/api/pleroma/emoji/packs/archive?name=#{name}") |> to_string() + url: URI.merge(uri, "/api/v1/pleroma/emoji/packs/archive?name=#{name}") |> to_string() }} %{"fallback-src" => src, "fallback-src-sha256" => sha} when is_binary(src) -> -- cgit v1.2.3 From 29158681f9f6d3f16bcab011a3338a60f11afddc Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 7 Jan 2024 17:05:30 +0300 Subject: Fetch count before downloading the pack and use that as page size --- lib/pleroma/emoji/pack.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index 49bb93799..85f7e1877 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -251,8 +251,12 @@ defmodule Pleroma.Emoji.Pack do uri = url |> String.trim() |> URI.parse() with :ok <- validate_shareable_packs_available(uri), + {:ok, %{"files_count" => files_count}} <- + uri |> URI.merge("/api/v1/pleroma/emoji/pack?name=#{name}&page_size=0") |> http_get(), {:ok, remote_pack} <- - uri |> URI.merge("/api/v1/pleroma/emoji/pack?name=#{name}") |> http_get(), + uri + |> URI.merge("/api/v1/pleroma/emoji/pack?name=#{name}&page_size=#{files_count}") + |> http_get(), {:ok, %{sha: sha, url: url} = pack_info} <- fetch_pack_info(remote_pack, uri, name), {:ok, archive} <- download_archive(url, sha), pack <- copy_as(remote_pack, as || name), -- cgit v1.2.3 From 4ca65c6182e9a575de3f360a3dac2861ad4e9960 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Sun, 7 Jan 2024 19:36:13 +0100 Subject: MRF.StealEmojiPolicy: Properly add fallback extension to filenames missing one --- lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 28c2cf3b3..237dfefa5 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -34,7 +34,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do |> Path.basename() |> Path.extname() - file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png")) + extension = if extension == "", do: ".png", else: extension + + file_path = Path.join(emoji_dir_path, shortcode <> extension) case File.write(file_path, response.body) do :ok -> -- cgit v1.2.3 From 7651198508cc40adfab27969f196e4496bd441da Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Thu, 11 Jan 2024 07:13:13 +0100 Subject: Support objects with a nil contentMap (firefish) Closes: https://git.pleroma.social/pleroma/pleroma/-/issues/3231 --- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 35f3aea03..68f3e1399 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -339,6 +339,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_tag(object), do: object + def fix_content_map(%{"contentMap" => nil} = object) do + Map.drop(object, ["contentMap"]) + end + # content map usually only has one language so this will do for now. def fix_content_map(%{"contentMap" => content_map} = object) do content_groups = Map.to_list(content_map) -- cgit v1.2.3 From 3c30eadd5ede822d6310d3ce6534d26d7caf41f5 Mon Sep 17 00:00:00 2001 From: Mint Date: Thu, 11 Jan 2024 20:38:12 +0300 Subject: Fix duplicate inbox deliveries --- lib/pleroma/web/activity_pub/publisher.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index cc47b3d27..fb7c6f005 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -255,7 +255,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [priority_recipients, recipients] |> Enum.map(fn recipients -> recipients - |> Enum.map(fn actor -> actor.inbox end) + |> Enum.map(fn %User{} = user -> + determine_inbox(activity, user) + end) + |> Enum.uniq() |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) |> Instances.filter_reachable() end) @@ -302,7 +305,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do recipients(actor, activity) |> Enum.map(fn recipients -> recipients - |> Enum.map(fn actor -> actor.inbox end) + |> Enum.map(fn %User{} = user -> + determine_inbox(activity, user) + end) + |> Enum.uniq() |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) end) -- cgit v1.2.3 From 12c052551bcd6b7871ccde5b9228315b89f45e01 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 14 Jan 2024 13:23:17 -0500 Subject: Allow the Remote Fetcher to attempt fetching an unreachable instance --- lib/pleroma/workers/remote_fetcher_worker.ex | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 7919969aa..d526a99cb 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -3,32 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.RemoteFetcherWorker do - alias Pleroma.Instances alias Pleroma.Object.Fetcher use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do - if Instances.reachable?(id) do - case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do - {:ok, _object} -> - :ok + case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do + {:ok, _object} -> + :ok - {:error, :forbidden} -> - {:discard, :forbidden} + {:error, :forbidden} -> + {:discard, :forbidden} - {:error, :not_found} -> - {:discard, :not_found} + {:error, :not_found} -> + {:discard, :not_found} - {:error, :allowed_depth} -> - {:discard, :allowed_depth} + {:error, :allowed_depth} -> + {:discard, :allowed_depth} - _ -> - :error - end - else - {:discard, "Unreachable instance"} + _ -> + :error end end -- cgit v1.2.3 From b39403a48fdb861b905bea16febba6d1660bb8df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 17 Jan 2024 17:12:40 +0100 Subject: Update API docs for my changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/views/instance_view.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index e514d0e91..7f73917d6 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -40,6 +40,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), + chat_limit: Keyword.get(instance, :chat_limit), pleroma: pleroma_configuration(instance) }) end @@ -222,6 +223,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do banner_upload_limit: Keyword.get(instance, :banner_upload_limit), background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), + chat_limit: Keyword.get(instance, :chat_limit), description_limit: Keyword.get(instance, :description_limit), shout_limit: Config.get([:shout, :limit]) }) -- cgit v1.2.3 From 2a28377be06d286a13c69d6735734d5cfd503309 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 15:55:33 -0500 Subject: Fix mix task pleroma.instance dialyzer error lib/mix/tasks/pleroma/instance.ex:356:pattern_match_cov The pattern :variable_ can never match, because previous clauses completely cover the type %{ :anonymize => boolean(), :dedupe => boolean(), :read_description => boolean(), :strip_location => boolean() }. --- lib/mix/tasks/pleroma/instance.ex | 2 -- .../web/twitter_api/controllers/.util_controller.ex.swp | Bin 0 -> 4096 bytes 2 files changed, 2 deletions(-) create mode 100644 lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index fd1809a42..0dc30549c 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -352,6 +352,4 @@ defmodule Mix.Tasks.Pleroma.Instance do enabled_filters end - - defp upload_filters(_), do: [] end diff --git a/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp b/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp new file mode 100644 index 000000000..822db48a7 Binary files /dev/null and b/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp differ -- cgit v1.2.3 From df1d390a49f3f632fbf99b0ffc4724ba03885e7e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 16:01:30 -0500 Subject: Pleroma.Activity.Queries: fix dialyzer error lib/pleroma/activity/queries.ex:12:unknown_type Unknown type: Activity.t/0. --- lib/pleroma/activity/queries.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index 81c44ac05..d770b9ff3 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Activity.Queries do import Ecto.Query, only: [from: 2, where: 3] - @type query :: Ecto.Queryable.t() | Activity.t() + @type query :: Ecto.Queryable.t() | Pleroma.Activity.t() alias Pleroma.Activity alias Pleroma.User -- cgit v1.2.3 From b16a01ba9d59ff1b5b64f799874dc646fd87aee9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 16:08:38 -0500 Subject: Pleroma.ApplicationRequirements: fix dialyzer errors lib/pleroma/application_requirements.ex:19:unknown_type Unknown type: Pleroma.ApplicationRequirements.VerifyError.t/0. lib/pleroma/application_requirements.ex:199:pattern_match_cov The pattern variable_result can never match, because previous clauses completely cover the type :ok. --- lib/pleroma/application_requirements.ex | 7 ++++--- .../web/twitter_api/controllers/.util_controller.ex.swp | Bin 4096 -> 0 bytes 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp (limited to 'lib') diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 1dbfea3e2..819245481 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -7,7 +7,10 @@ defmodule Pleroma.ApplicationRequirements do The module represents the collection of validations to runs before start server. """ - defmodule VerifyError, do: defexception([:message]) + defmodule VerifyError do + defexception([:message]) + @type t :: %__MODULE__{} + end alias Pleroma.Config alias Pleroma.Helpers.MediaHelper @@ -193,8 +196,6 @@ defmodule Pleroma.ApplicationRequirements do end end - defp check_system_commands!(result), do: result - defp check_repo_pool_size!(:ok) do if Pleroma.Config.get([Pleroma.Repo, :pool_size], 10) != 10 and not Pleroma.Config.get([:dangerzone, :override_repo_pool_size], false) do diff --git a/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp b/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp deleted file mode 100644 index 822db48a7..000000000 Binary files a/lib/pleroma/web/twitter_api/controllers/.util_controller.ex.swp and /dev/null differ -- cgit v1.2.3 From dc8045d7664327a2ada9c2a14ec46a0e84bfd6bc Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 16:36:01 -0500 Subject: FlakeId.Ecto.CompatType.t() does not exist This type is not exported and usable. FlakeId intends to return the type as :uuid, so we replace it in the typespecs with Ecto.UUID.t() which assuages the dialyzer errors e.g., lib/pleroma/bookmark.ex:25:unknown_type Unknown type: FlakeId.Ecto.CompatType.t/0. --- lib/pleroma/bookmark.ex | 6 +++--- lib/pleroma/chat.ex | 12 ++++++------ lib/pleroma/report_note.ex | 4 ++-- lib/pleroma/web/activity_pub/activity_pub.ex | 2 +- 4 files changed, 12 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index 187749e86..5119701f5 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Bookmark do timestamps() end - @spec create(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) :: + @spec create(Ecto.UUID.t(), Ecto.UUID.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()} def create(user_id, activity_id) do attrs = %{ @@ -37,7 +37,7 @@ defmodule Pleroma.Bookmark do |> Repo.insert() end - @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t() + @spec for_user_query(Ecto.UUID.t()) :: Ecto.Query.t() def for_user_query(user_id) do Bookmark |> where(user_id: ^user_id) @@ -52,7 +52,7 @@ defmodule Pleroma.Bookmark do |> Repo.one() end - @spec destroy(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t()) :: + @spec destroy(Ecto.UUID.t(), Ecto.UUID.t()) :: {:ok, Bookmark.t()} | {:error, Changeset.t()} def destroy(user_id, activity_id) do from(b in Bookmark, diff --git a/lib/pleroma/chat.ex b/lib/pleroma/chat.ex index fe32ec08c..5c4dbc1ff 100644 --- a/lib/pleroma/chat.ex +++ b/lib/pleroma/chat.ex @@ -42,7 +42,7 @@ defmodule Pleroma.Chat do |> unique_constraint(:user_id, name: :chats_user_id_recipient_index) end - @spec get_by_user_and_id(User.t(), FlakeId.Ecto.CompatType.t()) :: + @spec get_by_user_and_id(User.t(), Ecto.UUID.t()) :: {:ok, t()} | {:error, :not_found} def get_by_user_and_id(%User{id: user_id}, id) do from(c in __MODULE__, @@ -52,17 +52,17 @@ defmodule Pleroma.Chat do |> Repo.find_resource() end - @spec get_by_id(FlakeId.Ecto.CompatType.t()) :: t() | nil + @spec get_by_id(Ecto.UUID.t()) :: t() | nil def get_by_id(id) do Repo.get(__MODULE__, id) end - @spec get(FlakeId.Ecto.CompatType.t(), String.t()) :: t() | nil + @spec get(Ecto.UUID.t(), String.t()) :: t() | nil def get(user_id, recipient) do Repo.get_by(__MODULE__, user_id: user_id, recipient: recipient) end - @spec get_or_create(FlakeId.Ecto.CompatType.t(), String.t()) :: + @spec get_or_create(Ecto.UUID.t(), String.t()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} def get_or_create(user_id, recipient) do %__MODULE__{} @@ -75,7 +75,7 @@ defmodule Pleroma.Chat do ) end - @spec bump_or_create(FlakeId.Ecto.CompatType.t(), String.t()) :: + @spec bump_or_create(Ecto.UUID.t(), String.t()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} def bump_or_create(user_id, recipient) do %__MODULE__{} @@ -87,7 +87,7 @@ defmodule Pleroma.Chat do ) end - @spec for_user_query(FlakeId.Ecto.CompatType.t()) :: Ecto.Query.t() + @spec for_user_query(Ecto.UUID.t()) :: Ecto.Query.t() def for_user_query(user_id) do from(c in Chat, where: c.user_id == ^user_id, diff --git a/lib/pleroma/report_note.ex b/lib/pleroma/report_note.ex index f2ad76fa8..1a45b8e99 100644 --- a/lib/pleroma/report_note.ex +++ b/lib/pleroma/report_note.ex @@ -23,7 +23,7 @@ defmodule Pleroma.ReportNote do timestamps() end - @spec create(FlakeId.Ecto.CompatType.t(), FlakeId.Ecto.CompatType.t(), String.t()) :: + @spec create(Ecto.UUID.t(), Ecto.UUID.t(), String.t()) :: {:ok, ReportNote.t()} | {:error, Changeset.t()} def create(user_id, activity_id, content) do attrs = %{ @@ -38,7 +38,7 @@ defmodule Pleroma.ReportNote do |> Repo.insert() end - @spec destroy(FlakeId.Ecto.CompatType.t()) :: + @spec destroy(Ecto.UUID.t()) :: {:ok, ReportNote.t()} | {:error, Changeset.t()} def destroy(id) do from(r in ReportNote, where: r.id == ^id) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3980ef7b8..a12438f56 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -499,7 +499,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end @spec fetch_latest_direct_activity_id_for_context(String.t(), keyword() | map()) :: - FlakeId.Ecto.CompatType.t() | nil + Ecto.UUID.t() | nil def fetch_latest_direct_activity_id_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(Map.merge(%{skip_preload: true}, opts)) -- cgit v1.2.3 From 0b7d2142111fa6ff6d0d9a462322f4bc88f15175 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 16:47:50 -0500 Subject: Fix invalid typespec references to Ecto.Changeset.t() --- lib/pleroma/bookmark.ex | 4 ++-- lib/pleroma/config_db.ex | 6 +++--- lib/pleroma/report_note.ex | 4 ++-- lib/pleroma/user.ex | 16 ++++++++-------- lib/pleroma/user_invite_token.ex | 2 +- lib/pleroma/web/o_auth/authorization.ex | 8 ++++---- lib/pleroma/web/o_auth/token.ex | 4 ++-- 7 files changed, 22 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/bookmark.ex b/lib/pleroma/bookmark.ex index 5119701f5..b83d72446 100644 --- a/lib/pleroma/bookmark.ex +++ b/lib/pleroma/bookmark.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Bookmark do end @spec create(Ecto.UUID.t(), Ecto.UUID.t()) :: - {:ok, Bookmark.t()} | {:error, Changeset.t()} + {:ok, Bookmark.t()} | {:error, Ecto.Changeset.t()} def create(user_id, activity_id) do attrs = %{ user_id: user_id, @@ -53,7 +53,7 @@ defmodule Pleroma.Bookmark do end @spec destroy(Ecto.UUID.t(), Ecto.UUID.t()) :: - {:ok, Bookmark.t()} | {:error, Changeset.t()} + {:ok, Bookmark.t()} | {:error, Ecto.Changeset.t()} def destroy(user_id, activity_id) do from(b in Bookmark, where: b.user_id == ^user_id, diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index 846cede04..e28fcb124 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -54,7 +54,7 @@ defmodule Pleroma.ConfigDB do @spec get_by_params(map()) :: ConfigDB.t() | nil def get_by_params(%{group: _, key: _} = params), do: Repo.get_by(ConfigDB, params) - @spec changeset(ConfigDB.t(), map()) :: Changeset.t() + @spec changeset(ConfigDB.t(), map()) :: Ecto.Changeset.t() def changeset(config, params \\ %{}) do config |> cast(params, [:key, :group, :value]) @@ -138,7 +138,7 @@ defmodule Pleroma.ConfigDB do end end - @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} + @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Ecto.Changeset.t()} def update_or_create(params) do params = Map.put(params, :value, to_elixir_types(params[:value])) search_opts = Map.take(params, [:group, :key]) @@ -175,7 +175,7 @@ defmodule Pleroma.ConfigDB do end) end - @spec delete(ConfigDB.t() | map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} + @spec delete(ConfigDB.t() | map()) :: {:ok, ConfigDB.t()} | {:error, Ecto.Changeset.t()} def delete(%ConfigDB{} = config), do: Repo.delete(config) def delete(params) do diff --git a/lib/pleroma/report_note.ex b/lib/pleroma/report_note.ex index 1a45b8e99..f59e5451b 100644 --- a/lib/pleroma/report_note.ex +++ b/lib/pleroma/report_note.ex @@ -24,7 +24,7 @@ defmodule Pleroma.ReportNote do end @spec create(Ecto.UUID.t(), Ecto.UUID.t(), String.t()) :: - {:ok, ReportNote.t()} | {:error, Changeset.t()} + {:ok, ReportNote.t()} | {:error, Ecto.Changeset.t()} def create(user_id, activity_id, content) do attrs = %{ user_id: user_id, @@ -39,7 +39,7 @@ defmodule Pleroma.ReportNote do end @spec destroy(Ecto.UUID.t()) :: - {:ok, ReportNote.t()} | {:error, Changeset.t()} + {:ok, ReportNote.t()} | {:error, Ecto.Changeset.t()} def destroy(id) do from(r in ReportNote, where: r.id == ^id) |> Repo.one() diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0fd1b6365..45288bb44 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -672,7 +672,7 @@ defmodule Pleroma.User do |> validate_inclusion(:actor_type, ["Person", "Service"]) end - @spec update_as_admin(User.t(), map()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec update_as_admin(User.t(), map()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def update_as_admin(user, params) do params = Map.put(params, "password_confirmation", params["password"]) changeset = update_as_admin_changeset(user, params) @@ -693,7 +693,7 @@ defmodule Pleroma.User do |> put_change(:password_reset_pending, false) end - @spec reset_password(User.t(), map()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec reset_password(User.t(), map()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def reset_password(%User{} = user, params) do reset_password(user, user, params) end @@ -1783,14 +1783,14 @@ defmodule Pleroma.User do BackgroundWorker.enqueue("user_activation", %{"user_id" => user.id, "status" => status}) end - @spec set_activation([User.t()], boolean()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec set_activation([User.t()], boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def set_activation(users, status) when is_list(users) do Repo.transaction(fn -> for user <- users, do: set_activation(user, status) end) end - @spec set_activation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec set_activation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def set_activation(%User{} = user, status) do with {:ok, user} <- set_activation_status(user, status) do user @@ -1868,7 +1868,7 @@ defmodule Pleroma.User do |> update_and_set_cache() end - @spec purge_user_changeset(User.t()) :: Changeset.t() + @spec purge_user_changeset(User.t()) :: Ecto.Changeset.t() def purge_user_changeset(user) do # "Right to be forgotten" # https://gdpr.eu/right-to-be-forgotten/ @@ -2359,7 +2359,7 @@ defmodule Pleroma.User do updated_user end - @spec set_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Changeset.t()} + @spec set_confirmation(User.t(), boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} def set_confirmation(%User{} = user, bool) do user |> confirmation_changeset(set_confirmation: bool) @@ -2537,7 +2537,7 @@ defmodule Pleroma.User do |> update_and_set_cache() end - @spec confirmation_changeset(User.t(), keyword()) :: Changeset.t() + @spec confirmation_changeset(User.t(), keyword()) :: Ecto.Changeset.t() def confirmation_changeset(user, set_confirmation: confirmed?) do params = if confirmed? do @@ -2555,7 +2555,7 @@ defmodule Pleroma.User do cast(user, params, [:is_confirmed, :confirmation_token]) end - @spec approval_changeset(User.t(), keyword()) :: Changeset.t() + @spec approval_changeset(User.t(), keyword()) :: Ecto.Changeset.t() def approval_changeset(user, set_approval: approved?) do cast(user, %{is_approved: approved?}, [:is_approved]) end diff --git a/lib/pleroma/user_invite_token.ex b/lib/pleroma/user_invite_token.ex index b242a8848..4bfb3a6a7 100644 --- a/lib/pleroma/user_invite_token.ex +++ b/lib/pleroma/user_invite_token.ex @@ -64,7 +64,7 @@ defmodule Pleroma.UserInviteToken do end @spec update_invite(UserInviteToken.t(), map()) :: - {:ok, UserInviteToken.t()} | {:error, Changeset.t()} + {:ok, UserInviteToken.t()} | {:error, Ecto.Changeset.t()} def update_invite(invite, changes) do change(invite, changes) |> Repo.update() end diff --git a/lib/pleroma/web/o_auth/authorization.ex b/lib/pleroma/web/o_auth/authorization.ex index 593d2d66f..3340b4b12 100644 --- a/lib/pleroma/web/o_auth/authorization.ex +++ b/lib/pleroma/web/o_auth/authorization.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.OAuth.Authorization do end @spec create_authorization(App.t(), User.t() | %{}, [String.t()] | nil) :: - {:ok, Authorization.t()} | {:error, Changeset.t()} + {:ok, Authorization.t()} | {:error, Ecto.Changeset.t()} def create_authorization(%App{} = app, %User{} = user, scopes \\ nil) do %{ scopes: scopes || app.scopes, @@ -39,7 +39,7 @@ defmodule Pleroma.Web.OAuth.Authorization do |> Repo.insert() end - @spec create_changeset(map()) :: Changeset.t() + @spec create_changeset(map()) :: Ecto.Changeset.t() def create_changeset(attrs \\ %{}) do %Authorization{} |> cast(attrs, [:user_id, :app_id, :scopes, :valid_until]) @@ -58,7 +58,7 @@ defmodule Pleroma.Web.OAuth.Authorization do put_change(changeset, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), lifespan)) end - @spec use_changeset(Authtorizatiton.t(), map()) :: Changeset.t() + @spec use_changeset(Authtorizatiton.t(), map()) :: Ecto.Changeset.t() def use_changeset(%Authorization{} = auth, params) do auth |> cast(params, [:used]) @@ -66,7 +66,7 @@ defmodule Pleroma.Web.OAuth.Authorization do end @spec use_token(Authorization.t()) :: - {:ok, Authorization.t()} | {:error, Changeset.t()} | {:error, String.t()} + {:ok, Authorization.t()} | {:error, Ecto.Changeset.t()} | {:error, String.t()} def use_token(%Authorization{used: false, valid_until: valid_until} = auth) do if NaiveDateTime.diff(NaiveDateTime.utc_now(), valid_until) < 0 do Repo.update(use_changeset(auth, %{used: true})) diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 26de7bb10..4369b3164 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Web.OAuth.Token do |> Repo.find_resource() end - @spec exchange_token(App.t(), Authorization.t()) :: {:ok, Token.t()} | {:error, Changeset.t()} + @spec exchange_token(App.t(), Authorization.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()} def exchange_token(app, auth) do with {:ok, auth} <- Authorization.use_token(auth), true <- auth.app_id == app.id do @@ -95,7 +95,7 @@ defmodule Pleroma.Web.OAuth.Token do |> validate_required([:valid_until]) end - @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Changeset.t()} + @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Ecto.Changeset.t()} def create(%App{} = app, %User{} = user, attrs \\ %{}) do with {:ok, token} <- do_create(app, user, attrs) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do -- cgit v1.2.3 From 8ed506a370f0b4a7743353cd3c96c9dacd290765 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:05:55 -0500 Subject: Fix invalid type lib/pleroma/docs/json.ex:21:unknown_type Unknown type: Map.t/0. --- lib/pleroma/docs/json.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/docs/json.ex b/lib/pleroma/docs/json.ex index 05f46f39b..f69854935 100644 --- a/lib/pleroma/docs/json.ex +++ b/lib/pleroma/docs/json.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Docs.JSON do :persistent_term.put(@term, Pleroma.Docs.Generator.convert_to_strings(descriptions)) end - @spec compiled_descriptions :: Map.t() + @spec compiled_descriptions :: map() def compiled_descriptions do :persistent_term.get(@term) end -- cgit v1.2.3 From 559aeb5dd0cb66e57fedce9ea20eff505786d0cb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:08:18 -0500 Subject: Add missing type Pleroma.Emoji.t() lib/pleroma/emoji/loader.ex:23:unknown_type Unknown type: Pleroma.Emoji.t/0. --- lib/pleroma/emoji.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/emoji.ex b/lib/pleroma/emoji.ex index 43a3447c3..48302f396 100644 --- a/lib/pleroma/emoji.ex +++ b/lib/pleroma/emoji.ex @@ -24,6 +24,8 @@ defmodule Pleroma.Emoji do defstruct [:code, :file, :tags, :safe_code, :safe_file] + @type t :: %__MODULE__{} + @doc "Build emoji struct" def build({code, file, tags}) do %__MODULE__{ -- cgit v1.2.3 From 593c7e26d4769196a3405b61147dd88aa42e6012 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:13:27 -0500 Subject: Fix invalid type lib/pleroma/migrators/hashtags_table_migrator.ex:103:unknown_type Unknown type: Map.t/0. --- lib/pleroma/migrators/hashtags_table_migrator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/migrators/hashtags_table_migrator.ex b/lib/pleroma/migrators/hashtags_table_migrator.ex index dca4bfa6f..bd4dd2f1d 100644 --- a/lib/pleroma/migrators/hashtags_table_migrator.ex +++ b/lib/pleroma/migrators/hashtags_table_migrator.ex @@ -100,7 +100,7 @@ defmodule Pleroma.Migrators.HashtagsTableMigrator do |> where([_o, hashtags_objects], is_nil(hashtags_objects.object_id)) end - @spec transfer_object_hashtags(Map.t()) :: {:noop | :ok | :error, integer()} + @spec transfer_object_hashtags(map()) :: {:noop | :ok | :error, integer()} defp transfer_object_hashtags(object) do embedded_tags = if Map.has_key?(object, :tag), do: object.tag, else: object.data["tag"] hashtags = Object.object_data_hashtags(%{"tag" => embedded_tags}) -- cgit v1.2.3 From e3f52ee13fbda3b45b5d443ef9070e84cb7c1518 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:14:10 -0500 Subject: Fix invalid types lib/pleroma/web/streamer.ex:37:unknown_type Unknown type: Map.t/0. ________________________________________________________________________________ lib/pleroma/web/streamer.ex:63:unknown_type Unknown type: Map.t/0. --- lib/pleroma/web/streamer.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index aaee79d8e..0c9f04f82 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -34,7 +34,7 @@ defmodule Pleroma.Web.Streamer do stream :: String.t(), User.t() | nil, Token.t() | nil, - Map.t() | nil + map() | nil ) :: {:ok, topic :: String.t()} | {:error, :bad_topic} | {:error, :unauthorized} def get_topic_and_add_socket(stream, user, oauth_token, params \\ %{}) do @@ -60,7 +60,7 @@ defmodule Pleroma.Web.Streamer do end @doc "Expand and authorizes a stream" - @spec get_topic(stream :: String.t() | nil, User.t() | nil, Token.t() | nil, Map.t()) :: + @spec get_topic(stream :: String.t() | nil, User.t() | nil, Token.t() | nil, map()) :: {:ok, topic :: String.t() | nil} | {:error, :bad_topic} def get_topic(stream, user, oauth_token, params \\ %{}) -- cgit v1.2.3 From 467a65af90e4ee7893cd40cefe4931f2738219d4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:14:56 -0500 Subject: Fix invalid types lib/pleroma/web/rich_media/parser/ttl.ex:6:unknown_type Unknown type: Integer.t/0. lib/pleroma/web/rich_media/parser/ttl.ex:6:unknown_type Unknown type: Map.t/0. --- lib/pleroma/web/rich_media/parser/ttl.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/parser/ttl.ex b/lib/pleroma/web/rich_media/parser/ttl.ex index 59d7f87ab..b51298bd8 100644 --- a/lib/pleroma/web/rich_media/parser/ttl.ex +++ b/lib/pleroma/web/rich_media/parser/ttl.ex @@ -3,5 +3,5 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser.TTL do - @callback ttl(Map.t(), String.t()) :: Integer.t() | nil + @callback ttl(map(), String.t()) :: integer() | nil end -- cgit v1.2.3 From 09ae0ab24a30eebbb4880ff44300ee1774bffe93 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:16:10 -0500 Subject: Fix invalid type lib/pleroma/web/rich_media/parser.ex:105:unknown_type Unknown type: Integer.t/0. --- lib/pleroma/web/rich_media/parser.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index c37c45963..17a6ad617 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -102,7 +102,7 @@ defmodule Pleroma.Web.RichMedia.Parser do ttl_setters: [MyModule] """ @spec set_ttl_based_on_image(map(), String.t()) :: - {:ok, Integer.t() | :noop} | {:error, :no_key} + {:ok, integer() | :noop} | {:error, :no_key} def set_ttl_based_on_image(data, url) do case get_ttl_from_image(data, url) do {:ok, ttl} when is_number(ttl) -> -- cgit v1.2.3 From 65dfaa6cb9f6aa5bf7f66b5f02d63141a93137ba Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:18:16 -0500 Subject: Fix invalid type due to late aliasing lib/pleroma/web/o_auth/token/query.ex:12:unknown_type Unknown type: Token.t/0. --- lib/pleroma/web/o_auth/token/query.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/o_auth/token/query.ex b/lib/pleroma/web/o_auth/token/query.ex index 4a4d2d3ef..6853ec8dd 100644 --- a/lib/pleroma/web/o_auth/token/query.ex +++ b/lib/pleroma/web/o_auth/token/query.ex @@ -9,10 +9,10 @@ defmodule Pleroma.Web.OAuth.Token.Query do import Ecto.Query, only: [from: 2] - @type query :: Ecto.Queryable.t() | Token.t() - alias Pleroma.Web.OAuth.Token + @type query :: Ecto.Queryable.t() | Token.t() + @spec get_by_refresh_token(query, String.t()) :: query def get_by_refresh_token(query \\ Token, refresh_token) do from(q in query, where: q.refresh_token == ^refresh_token) -- cgit v1.2.3 From e5120a27031c6423504cb15ea1ccd3ef9d47e6e4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:21:12 -0500 Subject: Fix invalid type due to typos lib/pleroma/web/o_auth/authorization.ex:61:unknown_type Unknown type: Authtorizatiton.t/0. --- lib/pleroma/web/o_auth/authorization.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/o_auth/authorization.ex b/lib/pleroma/web/o_auth/authorization.ex index 3340b4b12..22e5bfc53 100644 --- a/lib/pleroma/web/o_auth/authorization.ex +++ b/lib/pleroma/web/o_auth/authorization.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Web.OAuth.Authorization do put_change(changeset, :valid_until, NaiveDateTime.add(NaiveDateTime.utc_now(), lifespan)) end - @spec use_changeset(Authtorizatiton.t(), map()) :: Ecto.Changeset.t() + @spec use_changeset(Authorization.t(), map()) :: Ecto.Changeset.t() def use_changeset(%Authorization{} = auth, params) do auth |> cast(params, [:used]) -- cgit v1.2.3 From f050a75b92546968f9ec351f78a48e6b6ac75a6b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:22:40 -0500 Subject: Fix invalid types due to typos lib/pleroma/web/feed/feed_view.ex:135:unknown_type Unknown type: NativeDateTime.t/0. lib/pleroma/web/feed/feed_view.ex:148:unknown_type Unknown type: NativeDateTime.t/0. --- lib/pleroma/web/feed/feed_view.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex index 034722eb2..e1ee33d62 100644 --- a/lib/pleroma/web/feed/feed_view.ex +++ b/lib/pleroma/web/feed/feed_view.ex @@ -132,7 +132,7 @@ defmodule Pleroma.Web.Feed.FeedView do |> safe_to_string() end - @spec to_rfc3339(String.t() | NativeDateTime.t()) :: String.t() + @spec to_rfc3339(String.t() | NaiveDateTime.t()) :: String.t() def to_rfc3339(date) when is_binary(date) do date |> Timex.parse!("{ISO:Extended}") @@ -145,7 +145,7 @@ defmodule Pleroma.Web.Feed.FeedView do |> Timex.format!("{RFC3339}") end - @spec to_rfc2822(String.t() | DateTime.t() | NativeDateTime.t()) :: String.t() + @spec to_rfc2822(String.t() | DateTime.t() | NaiveDateTime.t()) :: String.t() def to_rfc2822(datestr) when is_binary(datestr) do datestr |> Timex.parse!("{ISO:Extended}") -- cgit v1.2.3 From 551e90cd52a11dcb11840c6c4074db4a3c580703 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:28:54 -0500 Subject: Fix invalid type lib/pleroma/upload.ex:89:unknown_type Unknown type: Map.t/0. --- lib/pleroma/upload.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index bedd7889a..3c355e64d 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -86,7 +86,7 @@ defmodule Pleroma.Upload do end end - @spec store(source, options :: [option()]) :: {:ok, Map.t()} | {:error, any()} + @spec store(source, options :: [option()]) :: {:ok, map()} | {:error, any()} @doc "Store a file. If using a `Plug.Upload{}` as the source, be sure to use `Majic.Plug` to ensure its content_type and filename is correct." def store(upload, opts \\ []) do opts = get_opts(opts) -- cgit v1.2.3 From 2061a1d917a15bfe12b767c665c05750597e6145 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:29:27 -0500 Subject: Fix invalid type lib/pleroma/uploaders/uploader.ex:43:unknown_type Unknown type: Map.t/0. --- lib/pleroma/uploaders/uploader.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/uploaders/uploader.ex b/lib/pleroma/uploaders/uploader.ex index 77f6f02dd..23caaff1a 100644 --- a/lib/pleroma/uploaders/uploader.ex +++ b/lib/pleroma/uploaders/uploader.ex @@ -40,7 +40,7 @@ defmodule Pleroma.Uploaders.Uploader do @callback delete_file(file :: String.t()) :: :ok | {:error, String.t()} - @callback http_callback(Plug.Conn.t(), Map.t()) :: + @callback http_callback(Plug.Conn.t(), map()) :: {:ok, Plug.Conn.t()} | {:ok, Plug.Conn.t(), file_spec()} | {:error, Plug.Conn.t(), String.t()} -- cgit v1.2.3 From ec5ae83da6f3b630f174288cf3687bf12cee9787 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:31:07 -0500 Subject: Fix invalid types lib/pleroma/web/activity_pub/mrf/policy.ex:6:unknown_type Unknown type: Map.t/0. lib/pleroma/web/activity_pub/mrf/policy.ex:7:unknown_type Unknown type: Map.t/0. --- lib/pleroma/web/activity_pub/mrf/policy.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 0234de4d5..1f34883e7 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -3,8 +3,8 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.Policy do - @callback filter(Map.t()) :: {:ok | :reject, Map.t()} - @callback describe() :: {:ok | :error, Map.t()} + @callback filter(map()) :: {:ok | :reject, map()} + @callback describe() :: {:ok | :error, map()} @callback config_description() :: %{ optional(:children) => [map()], key: atom(), -- cgit v1.2.3 From 4f0711610896cb76322d0f527cbea5ed9d0ef474 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:32:19 -0500 Subject: Fix invalid type lib/pleroma/web/activity_pub/publisher.ex:31:unknown_type Unknown type: Map.t/0. --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index fb7c6f005..2449a3a3e 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do @doc """ Enqueue publishing a single activity. """ - @spec enqueue_one(Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} + @spec enqueue_one(map(), Keyword.t()) :: {:ok, %Oban.Job{}} def enqueue_one(%{} = params, worker_args \\ []) do PublisherWorker.enqueue( "publish_one", -- cgit v1.2.3 From 83eece7764243d5902df96d2a8a6b99d7465ddba Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:33:37 -0500 Subject: Fix invalid type lib/pleroma/web/auth/authenticator.ex:8:unknown_type Unknown type: User.t/0. --- lib/pleroma/web/auth/authenticator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/auth/authenticator.ex b/lib/pleroma/web/auth/authenticator.ex index a0bd154db..01bf1575c 100644 --- a/lib/pleroma/web/auth/authenticator.ex +++ b/lib/pleroma/web/auth/authenticator.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.Auth.Authenticator do @callback get_user(Plug.Conn.t()) :: {:ok, user :: struct()} | {:error, any()} @callback create_from_registration(Plug.Conn.t(), registration :: struct()) :: - {:ok, User.t()} | {:error, any()} + {:ok, Pleroma.User.t()} | {:error, any()} @callback get_registration(Plug.Conn.t()) :: {:ok, registration :: struct()} | {:error, any()} @callback handle_error(Plug.Conn.t(), any()) :: any() @callback auth_template() :: String.t() | nil -- cgit v1.2.3 From 38d01ff511ecd7223787772044ca6e3b20578c05 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:37:27 -0500 Subject: Fix invalid types --- lib/pleroma/reverse_proxy.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 880940d07..ec7732946 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -81,9 +81,9 @@ defmodule Pleroma.ReverseProxy do import Plug.Conn @type option() :: - {:max_read_duration, :timer.time() | :infinity} + {:max_read_duration, non_neg_integer() | :infinity} | {:max_body_length, non_neg_integer() | :infinity} - | {:failed_request_ttl, :timer.time() | :infinity} + | {:failed_request_ttl, non_neg_integer() | :infinity} | {:http, []} | {:req_headers, [{String.t(), String.t()}]} | {:resp_headers, [{String.t(), String.t()}]} -- cgit v1.2.3 From ea26add540b1da91ebc266b0c0532e1f8c54f4d7 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:43:34 -0500 Subject: Fix incorrect type definition for maybe_direct_follow/2 --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 45288bb44..416a5b5c9 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1048,7 +1048,7 @@ defmodule Pleroma.User do def needs_update?(_), do: true - @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t()} | {:error, String.t()} + @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t(), User.t()} | {:error, String.t()} # "Locked" (self-locked) users demand explicit authorization of follow requests def maybe_direct_follow(%User{} = follower, %User{local: true, is_locked: true} = followed) do -- cgit v1.2.3 From 2fbb67add70e77ba7da388623064bb5a671ce349 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:48:12 -0500 Subject: Fix typo in typespec --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 416a5b5c9..ac049ec17 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1011,7 +1011,7 @@ defmodule Pleroma.User do def maybe_send_confirmation_email(_), do: {:ok, :noop} - @spec send_confirmation_email(Uset.t()) :: User.t() + @spec send_confirmation_email(User.t()) :: User.t() def send_confirmation_email(%User{} = user) do user |> Pleroma.Emails.UserEmail.account_confirmation_email() -- cgit v1.2.3 From 7f649a7a19ae5f236dc21aaa11bb9a33c72dabbf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:50:21 -0500 Subject: Dialyzer: remove function that will never match --- lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index 97d75ecf2..df4ba819c 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -56,8 +56,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do nick_score + name_score + actor_type_score end - defp determine_if_followbot(_), do: 0.0 - defp bot_allowed?(%{"object" => target}, bot_actor) do %User{} = user = normalize_by_ap_id(target) -- cgit v1.2.3 From 88042109a3590db25dd9e39f353b1456cbd4e44b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:56:32 -0500 Subject: Dialyzer: fix pattern match coverage --- .../ecto_type/activity_pub/object_validators/bare_uri.ex | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex index 1038296e7..a1af8faa1 100644 --- a/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex @@ -8,10 +8,12 @@ defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.BareUri do def type, do: :string def cast(uri) when is_binary(uri) do - case URI.parse(uri) do - %URI{scheme: nil} -> :error - %URI{} -> {:ok, uri} - _ -> :error + parsed = URI.parse(uri) + + if is_nil(parsed.scheme) do + :error + else + {:ok, uri} end end -- cgit v1.2.3 From 65ac5137768d90ee9d2667e68181037e5d5642d3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 17:58:47 -0500 Subject: Dialyzer: fix pattern match coverage --- lib/pleroma/release_tasks.ex | 6 ------ 1 file changed, 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/release_tasks.ex b/lib/pleroma/release_tasks.ex index f9e8d1948..bcfcd1243 100644 --- a/lib/pleroma/release_tasks.ex +++ b/lib/pleroma/release_tasks.ex @@ -55,12 +55,6 @@ defmodule Pleroma.ReleaseTasks do {:error, term} when is_binary(term) -> IO.puts(:stderr, "The database for #{inspect(@repo)} couldn't be created: #{term}") - - {:error, term} -> - IO.puts( - :stderr, - "The database for #{inspect(@repo)} couldn't be created: #{inspect(term)}" - ) end end end -- cgit v1.2.3 From 029aaf3d744184737666b1e553ed92d7708f1fee Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 18:14:27 -0500 Subject: Use config to control max_restarts --- lib/pleroma/application.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 8fa6f3fae..a01a13b18 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -116,12 +116,7 @@ defmodule Pleroma.Application do # If we have a lot of caches, default max_restarts can cause test # resets to fail. # Go for the default 3 unless we're in test - max_restarts = - if @mix_env == :test do - 100 - else - 3 - end + max_restarts = Application.get_env(:pleroma, __MODULE__)[:max_restarts] opts = [strategy: :one_for_one, name: Pleroma.Supervisor, max_restarts: max_restarts] result = Supervisor.start_link(children, opts) -- cgit v1.2.3 From c7eda0b24ade372e4e167ae560a2debb555a6e02 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 18:22:49 -0500 Subject: Use config to control loading of custom modules --- lib/pleroma/application.ex | 34 ++++++++++++++++------------------ lib/pleroma/user.ex | 3 ++- lib/pleroma/web/o_auth/token.ex | 3 ++- 3 files changed, 20 insertions(+), 20 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index a01a13b18..de7d06974 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -106,7 +106,7 @@ defmodule Pleroma.Application do {Oban, Config.get(Oban)}, Pleroma.Web.Endpoint ] ++ - task_children(@mix_env) ++ + task_children() ++ dont_run_in_test(@mix_env) ++ shout_child(shout_enabled?()) ++ [Pleroma.Gopher.Server] @@ -154,7 +154,7 @@ defmodule Pleroma.Application do raise "Invalid custom modules" {:ok, modules, _warnings} -> - if @mix_env != :test do + if Application.get_env(:pleroma, __MODULE__)[:load_custom_modules] do Enum.each(modules, fn mod -> Logger.info("Custom module loaded: #{inspect(mod)}") end) @@ -237,29 +237,27 @@ defmodule Pleroma.Application do defp shout_child(_), do: [] - defp task_children(:test) do - [ + defp task_children() do + children = [ %{ id: :web_push_init, start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, restart: :temporary } ] - end - defp task_children(_) do - [ - %{ - id: :web_push_init, - start: {Task, :start_link, [&Pleroma.Web.Push.init/0]}, - restart: :temporary - }, - %{ - id: :internal_fetch_init, - start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, - restart: :temporary - } - ] + if Application.get_env(:pleroma, __MODULE__)[:internal_fetch] do + children ++ + [ + %{ + id: :internal_fetch_init, + start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]}, + restart: :temporary + } + ] + else + children + end end # start hackney and gun pools in tests diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ac049ec17..89a95c435 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1048,7 +1048,8 @@ defmodule Pleroma.User do def needs_update?(_), do: true - @spec maybe_direct_follow(User.t(), User.t()) :: {:ok, User.t(), User.t()} | {:error, String.t()} + @spec maybe_direct_follow(User.t(), User.t()) :: + {:ok, User.t(), User.t()} | {:error, String.t()} # "Locked" (self-locked) users demand explicit authorization of follow requests def maybe_direct_follow(%User{} = follower, %User{local: true, is_locked: true} = followed) do diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 4369b3164..d4a7e8999 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -56,7 +56,8 @@ defmodule Pleroma.Web.OAuth.Token do |> Repo.find_resource() end - @spec exchange_token(App.t(), Authorization.t()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()} + @spec exchange_token(App.t(), Authorization.t()) :: + {:ok, Token.t()} | {:error, Ecto.Changeset.t()} def exchange_token(app, auth) do with {:ok, auth} <- Authorization.use_token(auth), true <- auth.app_id == app.id do -- cgit v1.2.3 From 4bb57d4f25bcdc90a63163ba175b6171c9ddbc33 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 18:47:25 -0500 Subject: Use config to control background migrators --- lib/pleroma/application.ex | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index de7d06974..2eda212b3 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -108,6 +108,7 @@ defmodule Pleroma.Application do ] ++ task_children() ++ dont_run_in_test(@mix_env) ++ + background_migrators() ++ shout_child(shout_enabled?()) ++ [Pleroma.Gopher.Server] @@ -218,14 +219,18 @@ defmodule Pleroma.Application do keys: :duplicate, partitions: System.schedulers_online() ]} - ] ++ background_migrators() + ] end defp background_migrators do - [ - Pleroma.Migrators.HashtagsTableMigrator, - Pleroma.Migrators.ContextObjectsDeletionMigrator - ] + if Application.get_env(:pleroma, __MODULE__)[:background_migrators] do + [ + Pleroma.Migrators.HashtagsTableMigrator, + Pleroma.Migrators.ContextObjectsDeletionMigrator + ] + else + [] + end end defp shout_child(true) do -- cgit v1.2.3 From 17877f612e6c655290c5dc8bdb82f4b34e8b5b9f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 18:51:20 -0500 Subject: Use config to control streamer registry --- lib/pleroma/application.ex | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 2eda212b3..272529972 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -107,7 +107,7 @@ defmodule Pleroma.Application do Pleroma.Web.Endpoint ] ++ task_children() ++ - dont_run_in_test(@mix_env) ++ + streamer_registry() ++ background_migrators() ++ shout_child(shout_enabled?()) ++ [Pleroma.Gopher.Server] @@ -209,17 +209,19 @@ defmodule Pleroma.Application do defp shout_enabled?, do: Config.get([:shout, :enabled]) - defp dont_run_in_test(env) when env in [:test, :benchmark], do: [] - - defp dont_run_in_test(_) do - [ - {Registry, - [ - name: Pleroma.Web.Streamer.registry(), - keys: :duplicate, - partitions: System.schedulers_online() - ]} - ] + defp streamer_registry() do + if Application.get_env(:pleroma, __MODULE__)[:streamer_registry] do + [ + {Registry, + [ + name: Pleroma.Web.Streamer.registry(), + keys: :duplicate, + partitions: System.schedulers_online() + ]} + ] + else + [] + end end defp background_migrators do -- cgit v1.2.3 From 23301003717a5e154f54c14a5b8cb10ea2033e1a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 19:10:57 -0500 Subject: Use config to control starting all HTTP pools in test env --- lib/pleroma/application.ex | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 272529972..c0bfb898d 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -14,7 +14,6 @@ defmodule Pleroma.Application do @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] - @mix_env Mix.env() def name, do: @name def version, do: @version @@ -98,7 +97,7 @@ defmodule Pleroma.Application do {Task.Supervisor, name: Pleroma.TaskSupervisor} ] ++ cachex_children() ++ - http_children(adapter, @mix_env) ++ + http_children(adapter) ++ [ Pleroma.Stats, Pleroma.JobQueueMonitor, @@ -268,11 +267,19 @@ defmodule Pleroma.Application do end # start hackney and gun pools in tests - defp http_children(_, :test) do - http_children(Tesla.Adapter.Hackney, nil) ++ http_children(Tesla.Adapter.Gun, nil) + defp http_children(adapter) do + if Application.get_env(:pleroma, __MODULE__)[:test_http_pools] do + http_children_hackney() ++ http_children_gun() + else + cond do + match?(Tesla.Adapter.Hackney, adapter) -> http_children_hackney() + match?(Tesla.Adapter.Gun, adapter) -> http_children_gun() + true -> [] + end + end end - defp http_children(Tesla.Adapter.Hackney, _) do + defp http_children_hackney() do pools = [:federation, :media] pools = @@ -288,13 +295,11 @@ defmodule Pleroma.Application do end end - defp http_children(Tesla.Adapter.Gun, _) do + defp http_children_gun() do Pleroma.Gun.ConnectionPool.children() ++ [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] end - defp http_children(_, _), do: [] - @spec limiters_setup() :: :ok def limiters_setup do config = Config.get(ConcurrentLimiter, []) -- cgit v1.2.3 From dcd010280034fd35254906831402d96fd59f8098 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 19:39:13 -0500 Subject: Credo --- lib/pleroma/application.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index c0bfb898d..de668052f 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -208,7 +208,7 @@ defmodule Pleroma.Application do defp shout_enabled?, do: Config.get([:shout, :enabled]) - defp streamer_registry() do + defp streamer_registry do if Application.get_env(:pleroma, __MODULE__)[:streamer_registry] do [ {Registry, @@ -243,7 +243,7 @@ defmodule Pleroma.Application do defp shout_child(_), do: [] - defp task_children() do + defp task_children do children = [ %{ id: :web_push_init, @@ -279,7 +279,7 @@ defmodule Pleroma.Application do end end - defp http_children_hackney() do + defp http_children_hackney do pools = [:federation, :media] pools = @@ -295,7 +295,7 @@ defmodule Pleroma.Application do end end - defp http_children_gun() do + defp http_children_gun do Pleroma.Gun.ConnectionPool.children() ++ [{Task, &Pleroma.HTTP.AdapterHelper.Gun.limiter_setup/0}] end -- cgit v1.2.3