diff options
Diffstat (limited to 'lib')
136 files changed, 2391 insertions, 752 deletions
diff --git a/lib/mix/pleroma.ex b/lib/mix/pleroma.ex index 2976085ba..c01cf054d 100644 --- a/lib/mix/pleroma.ex +++ b/lib/mix/pleroma.ex @@ -14,7 +14,8 @@ defmodule Mix.Pleroma do :swoosh, :timex, :fast_html, - :oban + :oban, + :logger_backends ] @cachex_children ["object", "user", "scrubber", "web_resp"] @doc "Common functions to be reused in mix tasks" diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 3a2ea44f8..8b3b2f18b 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -205,6 +205,35 @@ defmodule Mix.Tasks.Pleroma.Config do end end + # Removes any policies that are not a real module + # as they will prevent the server from starting + def run(["fix_mrf_policies"]) do + check_configdb(fn -> + start_pleroma() + + group = :pleroma + key = :mrf + + %{value: value} = + group + |> ConfigDB.get_by_group_and_key(key) + + policies = + Keyword.get(value, :policies, []) + |> Enum.filter(&is_atom(&1)) + |> Enum.filter(fn mrf -> + case Code.ensure_compiled(mrf) do + {:module, _} -> true + {:error, _} -> false + end + end) + + value = Keyword.put(value, :policies, policies) + + ConfigDB.update_or_create(%{group: group, key: key, value: value}) + end) + end + @spec migrate_to_db(Path.t() | nil) :: any() def migrate_to_db(file_path \\ nil) do with :ok <- Pleroma.Config.DeprecationWarnings.warn() do diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 93ee57dc3..b82d1f079 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -67,43 +67,168 @@ defmodule Mix.Tasks.Pleroma.Database do OptionParser.parse( args, strict: [ - vacuum: :boolean + vacuum: :boolean, + keep_threads: :boolean, + keep_non_public: :boolean, + prune_orphaned_activities: :boolean ] ) start_pleroma() deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400)) - Logger.info("Pruning objects older than #{deadline} days") + log_message = "Pruning objects older than #{deadline} days" - time_deadline = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(-(deadline * 86_400)) + log_message = + if Keyword.get(options, :keep_non_public) do + log_message <> ", keeping non public posts" + else + log_message + end - from(o in Object, - where: - fragment( - "?->'to' \\? ? OR ?->'cc' \\? ?", - o.data, - ^Pleroma.Constants.as_public(), - o.data, - ^Pleroma.Constants.as_public() - ), - where: o.inserted_at < ^time_deadline, - where: + log_message = + if Keyword.get(options, :keep_threads) do + log_message <> ", keeping threads intact" + else + log_message + end + + log_message = + if Keyword.get(options, :prune_orphaned_activities) do + log_message <> ", pruning orphaned activities" + else + log_message + end + + log_message = + if Keyword.get(options, :vacuum) do + log_message <> + ", doing a full vacuum (you shouldn't do this as a recurring maintanance task)" + else + log_message + end + + Logger.info(log_message) + + if Keyword.get(options, :keep_threads) do + # We want to delete objects from threads where + # 1. the newest post is still old + # 2. none of the activities is local + # 3. none of the activities is bookmarked + # 4. optionally none of the posts is non-public + deletable_context = + if Keyword.get(options, :keep_non_public) do + Pleroma.Activity + |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id) + |> group_by([a], fragment("? ->> 'context'::text", a.data)) + |> having( + [a], + not fragment( + # Posts (checked on Create Activity) is non-public + "bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')", + a.data, + ^Pleroma.Constants.as_public(), + a.data, + ^Pleroma.Constants.as_public(), + a.data + ) + ) + else + Pleroma.Activity + |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id) + |> group_by([a], fragment("? ->> 'context'::text", a.data)) + end + |> having([a], max(a.updated_at) < ^time_deadline) + |> having([a], not fragment("bool_or(?)", a.local)) + |> having([_, b], fragment("max(?::text) is null", b.id)) + |> select([a], fragment("? ->> 'context'::text", a.data)) + + Pleroma.Object + |> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context)) + else + if Keyword.get(options, :keep_non_public) do + Pleroma.Object + |> where( + [o], + fragment( + "?->'to' \\? ? OR ?->'cc' \\? ?", + o.data, + ^Pleroma.Constants.as_public(), + o.data, + ^Pleroma.Constants.as_public() + ) + ) + else + Pleroma.Object + end + |> where([o], o.updated_at < ^time_deadline) + |> where( + [o], fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) - ) + ) + end |> Repo.delete_all(timeout: :infinity) - prune_hashtags_query = """ + if !Keyword.get(options, :keep_threads) do + # Without the --keep-threads option, it's possible that bookmarked + # objects have been deleted. We remove the corresponding bookmarks. + """ + delete from public.bookmarks + where id in ( + select b.id from public.bookmarks b + left join public.activities a on b.activity_id = a.id + left join public.objects o on a."data" ->> 'object' = o.data ->> 'id' + where o.id is null + ) + """ + |> Repo.query([], timeout: :infinity) + end + + if Keyword.get(options, :prune_orphaned_activities) do + # Prune activities who link to a single object + """ + delete from public.activities + where id in ( + select a.id from public.activities a + left join public.objects o on a.data ->> 'object' = o.data ->> 'id' + left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id' + left join public.users u on a.data ->> 'object' = u.ap_id + where not a.local + and jsonb_typeof(a."data" -> 'object') = 'string' + and o.id is null + and a2.id is null + and u.id is null + ) + """ + |> Repo.query([], timeout: :infinity) + + # Prune activities who link to an array of objects + """ + delete from public.activities + where id in ( + select a.id from public.activities a + join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array' + left join public.objects o on j.value = o.data ->> 'id' + left join public.activities a2 on j.value = a2.data ->> 'id' + left join public.users u on j.value = u.ap_id + group by a.id + having max(o.data ->> 'id') is null + and max(a2.data ->> 'id') is null + and max(u.ap_id) is null + ) + """ + |> Repo.query([], timeout: :infinity) + end + + """ DELETE FROM hashtags AS ht WHERE NOT EXISTS ( SELECT 1 FROM hashtags_objects hto WHERE ht.id = hto.hashtag_id) """ - - Repo.query(prune_hashtags_query) + |> Repo.query() if Keyword.get(options, :vacuum) do Maintenance.vacuum("full") @@ -226,7 +351,7 @@ defmodule Mix.Tasks.Pleroma.Database do ) end - shell_info('Done.') + shell_info(~c"Done.") end end diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex new file mode 100644 index 000000000..2a52472f9 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -0,0 +1,83 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Indexer do + import Mix.Pleroma + import Ecto.Query + + alias Pleroma.Workers.SearchIndexingWorker + + def run(["create_index"]) do + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do + IO.puts("Index created") + else + e -> IO.puts("Could not create index: #{inspect(e)}") + end + end + + def run(["drop_index"]) do + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do + IO.puts("Index dropped") + else + e -> IO.puts("Could not drop index: #{inspect(e)}") + end + end + + def run(["index" | options]) do + {options, [], []} = + OptionParser.parse( + options, + strict: [ + chunk: :integer, + limit: :integer, + step: :integer + ] + ) + + start_pleroma() + + chunk_size = Keyword.get(options, :chunk, 100) + limit = Keyword.get(options, :limit, 100_000) + per_step = Keyword.get(options, :step, 1000) + + chunks = max(div(limit, per_step), 1) + + 1..chunks + |> Enum.each(fn step -> + q = + from(a in Pleroma.Activity, + limit: ^per_step, + offset: ^per_step * (^step - 1), + select: [:id], + order_by: [desc: :id] + ) + + {:ok, ids} = + Pleroma.Repo.transaction(fn -> + Pleroma.Repo.stream(q, timeout: :infinity) + |> Enum.map(fn a -> + a.id + end) + end) + + IO.puts("Got #{length(ids)} activities, adding to indexer") + + ids + |> Enum.chunk_every(chunk_size) + |> Enum.each(fn chunk -> + IO.puts("Adding #{length(chunk)} activities to indexing queue") + + chunk + |> Enum.map(fn id -> + SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) + end) + |> Oban.insert_all() + end) + end) + end +end diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex new file mode 100644 index 000000000..69fefb001 --- /dev/null +++ b/lib/mix/tasks/pleroma/test_runner.ex @@ -0,0 +1,25 @@ +defmodule Mix.Tasks.Pleroma.TestRunner do + @shortdoc "Retries tests once if they fail" + + use Mix.Task + + def run(args \\ []) do + case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do + {_, 0} -> + :ok + + _ -> + retry(args) + end + end + + def retry(args) do + case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do + {_, 0} -> + :ok + + _ -> + exit(1) + end + end +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 75154f94c..cb15dc1e9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -51,7 +51,11 @@ defmodule Pleroma.Application do Pleroma.HTML.compile_scrubbers() Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() - Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + + if Config.get([Pleroma.Web.Plugs.HTTPSecurityPlug, :enable], true) do + Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + end + Pleroma.ApplicationRequirements.verify!() load_custom_modules() Pleroma.Docs.JSON.compile() @@ -109,7 +113,8 @@ defmodule Pleroma.Application do streamer_registry() ++ background_migrators() ++ shout_child(shout_enabled?()) ++ - [Pleroma.Gopher.Server] + [Pleroma.Gopher.Server] ++ + [Pleroma.Search.Healthcheck] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options @@ -162,7 +167,8 @@ defmodule Pleroma.Application do expiration: chat_message_id_idempotency_key_expiration(), limit: 500_000 ), - build_cachex("rel_me", limit: 2500) + build_cachex("rel_me", limit: 2500), + build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000) ] end @@ -285,8 +291,6 @@ defmodule Pleroma.Application do config = Config.get(ConcurrentLimiter, []) [ - Pleroma.Web.RichMedia.Helpers, - Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, Pleroma.Search ] |> Enum.each(fn module -> diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 8c0df64fc..a334d12ee 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -241,10 +241,9 @@ defmodule Pleroma.ApplicationRequirements do missing_mrfs = Enum.reduce(mrfs, [], fn x, acc -> - if Code.ensure_compiled(x) do - acc - else - acc ++ [x] + case Code.ensure_compiled(x) do + {:module, _} -> acc + {:error, _} -> acc ++ [x] end end) diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 91885347f..ffc95f144 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Config.TransferTask do @@ -44,14 +44,9 @@ defmodule Pleroma.Config.TransferTask do with {_, true} <- {:configurable, Config.get(:configurable_from_database)} do # We need to restart applications for loaded settings take effect - {logger, other} = + settings = (Repo.all(ConfigDB) ++ deleted_settings) |> Enum.map(&merge_with_default/1) - |> Enum.split_with(fn {group, _, _, _} -> group in [:logger] end) - - logger - |> Enum.sort() - |> Enum.each(&configure/1) started_applications = Application.started_applications() @@ -64,7 +59,7 @@ defmodule Pleroma.Config.TransferTask do [:pleroma | reject] end - other + settings |> Enum.map(&update/1) |> Enum.uniq() |> Enum.reject(&(&1 in reject)) @@ -102,38 +97,6 @@ defmodule Pleroma.Config.TransferTask do {group, key, value, merged} end - # change logger configuration in runtime, without restart - defp configure({_, :backends, _, merged}) do - # removing current backends - Enum.each(Application.get_env(:logger, :backends), &Logger.remove_backend/1) - - Enum.each(merged, &Logger.add_backend/1) - - :ok = update_env(:logger, :backends, merged) - end - - defp configure({_, key, _, merged}) when key in [:console, :ex_syslogger] do - merged = - if key == :console do - put_in(merged[:format], merged[:format] <> "\n") - else - merged - end - - backend = - if key == :ex_syslogger, - do: {ExSyslogger, :ex_syslogger}, - else: key - - Logger.configure_backend(backend, merged) - :ok = update_env(:logger, key, merged) - end - - defp configure({_, key, _, merged}) do - Logger.configure([{key, merged}]) - :ok = update_env(:logger, key, merged) - end - defp update({group, key, value, merged}) do try do :ok = update_env(group, key, merged) diff --git a/lib/pleroma/config_db.ex b/lib/pleroma/config_db.ex index e28fcb124..89d3050d6 100644 --- a/lib/pleroma/config_db.ex +++ b/lib/pleroma/config_db.ex @@ -165,8 +165,7 @@ defmodule Pleroma.ConfigDB do {:pleroma, :ecto_repos}, {:mime, :types}, {:cors_plug, [:max_age, :methods, :expose, :headers]}, - {:swarm, :node_blacklist}, - {:logger, :backends} + {:swarm, :node_blacklist} ] Enum.any?(full_key_update, fn @@ -385,7 +384,12 @@ defmodule Pleroma.ConfigDB do @spec module_name?(String.t()) :: boolean() def module_name?(string) do - Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Ueberauth|Swoosh)\./, string) or - string in ["Oban", "Ueberauth", "ExSyslogger", "ConcurrentLimiter"] + if String.contains?(string, ".") do + [name | _] = String.split(string, ".", parts: 2) + + name in ~w[Pleroma Phoenix Tesla Ueberauth Swoosh Logger LoggerBackends] + else + string in ~w[Oban Ueberauth ExSyslogger ConcurrentLimiter] + end end end diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index 4ed93e5bd..5d3344cc5 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Conversation.Participation do import Ecto.Changeset import Ecto.Query + @type t :: %__MODULE__{} + schema "conversation_participations" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) belongs_to(:conversation, Conversation) diff --git a/lib/pleroma/emoji/pack.ex b/lib/pleroma/emoji/pack.ex index afc341853..785fdb8b2 100644 --- a/lib/pleroma/emoji/pack.ex +++ b/lib/pleroma/emoji/pack.ex @@ -416,10 +416,10 @@ defmodule Pleroma.Emoji.Pack do end defp create_archive_and_cache(pack, hash) do - files = ['pack.json' | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)] + files = [~c"pack.json" | Enum.map(pack.files, fn {_, file} -> to_charlist(file) end)] {:ok, {_, result}} = - :zip.zip('#{pack.name}.zip', files, [:memory, cwd: to_charlist(pack.path)]) + :zip.zip(~c"#{pack.name}.zip", files, [:memory, cwd: to_charlist(pack.path)]) ttl_per_file = Pleroma.Config.get!([:emoji, :shared_pack_cache_seconds_per_file]) overall_ttl = :timer.seconds(ttl_per_file * Enum.count(files)) @@ -586,7 +586,7 @@ defmodule Pleroma.Emoji.Pack do with :ok <- File.mkdir_p!(local_pack.path) do files = Enum.map(remote_pack["files"], fn {_, path} -> to_charlist(path) end) # Fallback cannot contain a pack.json file - files = if pack_info[:fallback], do: files, else: ['pack.json' | files] + files = if pack_info[:fallback], do: files, else: [~c"pack.json" | files] :zip.unzip(archive, cwd: to_charlist(local_pack.path), file_list: files) end diff --git a/lib/pleroma/following_relationship.ex b/lib/pleroma/following_relationship.ex index f38c2fce9..495488dfd 100644 --- a/lib/pleroma/following_relationship.ex +++ b/lib/pleroma/following_relationship.ex @@ -199,8 +199,8 @@ defmodule Pleroma.FollowingRelationship do |> preload([:follower]) |> Repo.all() |> Enum.map(fn following_relationship -> - Pleroma.Web.CommonAPI.follow(following_relationship.follower, target) - Pleroma.Web.CommonAPI.unfollow(following_relationship.follower, origin) + Pleroma.Web.CommonAPI.follow(target, following_relationship.follower) + Pleroma.Web.CommonAPI.unfollow(origin, following_relationship.follower) end) |> case do [] -> diff --git a/lib/pleroma/frontend.ex b/lib/pleroma/frontend.ex index ec72fb6a4..816499917 100644 --- a/lib/pleroma/frontend.ex +++ b/lib/pleroma/frontend.ex @@ -43,10 +43,6 @@ defmodule Pleroma.Frontend do {:download_or_unzip, _} -> Logger.info("Could not download or unzip the frontend") {:error, "Could not download or unzip the frontend"} - - _e -> - Logger.info("Could not install the frontend") - {:error, "Could not install the frontend"} end end diff --git a/lib/pleroma/gun/connection_pool/reclaimer.ex b/lib/pleroma/gun/connection_pool/reclaimer.ex index 35e7f4b2e..3580d38f5 100644 --- a/lib/pleroma/gun/connection_pool/reclaimer.ex +++ b/lib/pleroma/gun/connection_pool/reclaimer.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Gun.ConnectionPool.Reclaimer do def start_monitor do pid = - case GenServer.start_link(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do + case GenServer.start(__MODULE__, [], name: {:via, Registry, {registry(), "reclaimer"}}) do {:ok, pid} -> pid diff --git a/lib/pleroma/gun/connection_pool/worker_supervisor.ex b/lib/pleroma/gun/connection_pool/worker_supervisor.ex index eb83962d8..b9dedf61e 100644 --- a/lib/pleroma/gun/connection_pool/worker_supervisor.ex +++ b/lib/pleroma/gun/connection_pool/worker_supervisor.ex @@ -5,6 +5,9 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do @moduledoc "Supervisor for pool workers. Does not do anything except enforce max connection limit" + alias Pleroma.Config + alias Pleroma.Gun.ConnectionPool.Worker + use DynamicSupervisor def start_link(opts) do @@ -14,21 +17,28 @@ defmodule Pleroma.Gun.ConnectionPool.WorkerSupervisor do def init(_opts) do DynamicSupervisor.init( strategy: :one_for_one, - max_children: Pleroma.Config.get([:connections_pool, :max_connections]) + max_children: Config.get([:connections_pool, :max_connections]) ) end - def start_worker(opts, last_attempt \\ false) do - case DynamicSupervisor.start_child(__MODULE__, {Pleroma.Gun.ConnectionPool.Worker, opts}) do + def start_worker(opts, last_attempt \\ false) + + def start_worker(opts, true) do + case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do + {:error, :max_children} -> + :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) + {:error, :pool_full} + + res -> + res + end + end + + def start_worker(opts, false) do + case DynamicSupervisor.start_child(__MODULE__, {Worker, opts}) do {:error, :max_children} -> - funs = [fn -> last_attempt end, fn -> match?(:error, free_pool()) end] - - if Enum.any?(funs, fn fun -> fun.() end) do - :telemetry.execute([:pleroma, :connection_pool, :provision_failure], %{opts: opts}) - {:error, :pool_full} - else - start_worker(opts, true) - end + free_pool() + start_worker(opts, true) res -> res diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex index 704d37f8a..00e18649e 100644 --- a/lib/pleroma/helpers/inet_helper.ex +++ b/lib/pleroma/helpers/inet_helper.ex @@ -16,4 +16,15 @@ defmodule Pleroma.Helpers.InetHelper do def parse_address(ip) do :inet.parse_address(ip) end + + def parse_cidr(proxy) when is_binary(proxy) do + proxy = + cond do + "/" in String.codepoints(proxy) -> proxy + InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" + InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" + end + + InetCidr.parse_cidr!(proxy, true) + end end diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index e44114d9d..8566ab3ea 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -25,7 +25,7 @@ defmodule Pleroma.Helpers.MediaHelper do end def image_resize(url, options) do - with {:ok, env} <- HTTP.get(url, [], pool: :media), + with {:ok, env} <- HTTP.get(url, [], http_client_opts()), {:ok, resized} <- Operation.thumbnail_buffer(env.body, options.max_width, height: options.max_height, @@ -45,8 +45,8 @@ defmodule Pleroma.Helpers.MediaHelper do @spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()} def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), - false <- @cachex.exists?(:failed_media_helper_cache, url), - {:ok, env} <- HTTP.get(url, [], pool: :media), + {:ok, false} <- @cachex.exists?(:failed_media_helper_cache, url), + {:ok, env} <- HTTP.get(url, [], http_client_opts()), {:ok, pid} <- StringIO.open(env.body) do body_stream = IO.binstream(pid, 1) @@ -71,17 +71,19 @@ defmodule Pleroma.Helpers.MediaHelper do end) case Task.yield(task, 5_000) do - nil -> + {:ok, result} -> + {:ok, result} + + _ -> Task.shutdown(task) @cachex.put(:failed_media_helper_cache, url, nil) {:error, {:ffmpeg, :timeout}} - - result -> - {:ok, result} end else nil -> {:error, {:ffmpeg, :command_not_found}} {:error, _} = error -> error end end + + defp http_client_opts, do: Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) end diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index eec61cf14..c11317850 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -37,7 +37,7 @@ defmodule Pleroma.HTTP do See `Pleroma.HTTP.request/5` """ - @spec post(Request.url(), String.t(), Request.headers(), keyword()) :: + @spec post(Request.url(), Tesla.Env.body(), Request.headers(), keyword()) :: {:ok, Env.t()} | {:error, any()} def post(url, body, headers \\ [], options \\ []), do: request(:post, url, body, headers, options) @@ -56,7 +56,7 @@ defmodule Pleroma.HTTP do `{:ok, %Tesla.Env{}}` or `{:error, error}` """ - @spec request(method(), Request.url(), String.t(), Request.headers(), keyword()) :: + @spec request(method(), Request.url(), Tesla.Env.body(), Request.headers(), keyword()) :: {:ok, Env.t()} | {:error, any()} def request(method, url, body, headers, options) when is_binary(url) do uri = URI.parse(url) @@ -68,7 +68,9 @@ defmodule Pleroma.HTTP do adapter = Application.get_env(:tesla, :adapter) - client = Tesla.client(adapter_middlewares(adapter), adapter) + extra_middleware = options[:tesla_middleware] || [] + + client = Tesla.client(adapter_middlewares(adapter, extra_middleware), adapter) maybe_limit( fn -> @@ -102,20 +104,21 @@ defmodule Pleroma.HTTP do fun.() end - defp adapter_middlewares(Tesla.Adapter.Gun) do - [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] + defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do + [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] ++ + extra_middleware end - defp adapter_middlewares({Tesla.Adapter.Finch, _}) do - [Tesla.Middleware.FollowRedirects] + defp adapter_middlewares({Tesla.Adapter.Finch, _}, extra_middleware) do + [Tesla.Middleware.FollowRedirects] ++ extra_middleware end - defp adapter_middlewares(_) do + defp adapter_middlewares(_, extra_middleware) do if Pleroma.Config.get(:env) == :test do # Emulate redirects in test env, which are handled by adapters in other environments [Tesla.Middleware.FollowRedirects] else - [] + extra_middleware end end end diff --git a/lib/pleroma/http/adapter_helper/gun.ex b/lib/pleroma/http/adapter_helper/gun.ex index 74ab9851e..1fe8dd4b2 100644 --- a/lib/pleroma/http/adapter_helper/gun.ex +++ b/lib/pleroma/http/adapter_helper/gun.ex @@ -15,7 +15,7 @@ defmodule Pleroma.HTTP.AdapterHelper.Gun do retry_timeout: 1_000 ] - @type pool() :: :federation | :upload | :media | :default + @type pool() :: :federation | :upload | :media | :rich_media | :default @spec options(keyword(), URI.t()) :: keyword() def options(incoming_opts \\ [], %URI{} = uri) do diff --git a/lib/pleroma/http_signatures_api.ex b/lib/pleroma/http_signatures_api.ex new file mode 100644 index 000000000..8e73dc98e --- /dev/null +++ b/lib/pleroma/http_signatures_api.ex @@ -0,0 +1,4 @@ +defmodule Pleroma.HTTPSignaturesAPI do + @callback validate_conn(conn :: Plug.Conn.t()) :: boolean + @callback signature_for_conn(conn :: Plug.Conn.t()) :: map +end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index c497a4fb7..288555146 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Instances.Instance do alias Pleroma.Maps alias Pleroma.Repo alias Pleroma.User - alias Pleroma.Workers.BackgroundWorker + alias Pleroma.Workers.DeleteWorker use Ecto.Schema @@ -297,7 +297,7 @@ defmodule Pleroma.Instances.Instance do all of those users' activities and notifications. """ def delete_users_and_activities(host) when is_binary(host) do - BackgroundWorker.enqueue("delete_instance", %{"host" => host}) + DeleteWorker.enqueue("delete_instance", %{"host" => host}) end def perform(:delete_instance, host) when is_binary(host) do diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index a80279fa6..75f4ba503 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -73,6 +73,7 @@ defmodule Pleroma.Notification do pleroma:report reblog poll + status } def changeset(%Notification{} = notification, attrs) do @@ -280,15 +281,10 @@ defmodule Pleroma.Notification do select: n.id ) - {:ok, %{ids: {_, notification_ids}}} = - Multi.new() - |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) - |> Marker.multi_set_last_read_id(user, "notifications") - |> Repo.transaction() - - for_user_query(user) - |> where([n], n.id in ^notification_ids) - |> Repo.all() + Multi.new() + |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) + |> Marker.multi_set_last_read_id(user, "notifications") + |> Repo.transaction() end @spec read_one(User.t(), String.t()) :: @@ -299,10 +295,6 @@ defmodule Pleroma.Notification do |> Multi.update(:update, changeset(notification, %{seen: true})) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - |> case do - {:ok, %{update: notification}} -> {:ok, notification} - {:error, :update, changeset, _} -> {:error, changeset} - end end end @@ -384,10 +376,15 @@ defmodule Pleroma.Notification do defp do_create_notifications(%Activity{} = activity) do enabled_receivers = get_notified_from_activity(activity) + enabled_subscribers = get_notified_subscribers_from_activity(activity) + notifications = - Enum.map(enabled_receivers, fn user -> - create_notification(activity, user) - end) + (Enum.map(enabled_receivers, fn user -> + create_notification(activity, user) + end) ++ + Enum.map(enabled_subscribers -- enabled_receivers, fn user -> + create_notification(activity, user, type: "status") + end)) |> Enum.reject(&is_nil/1) {:ok, notifications} @@ -492,7 +489,7 @@ defmodule Pleroma.Notification do NOTE: might be called for FAKE Activities, see ActivityPub.Utils.get_notified_from_object/1 """ - @spec get_notified_from_activity(Activity.t(), boolean()) :: {list(User.t()), list(User.t())} + @spec get_notified_from_activity(Activity.t(), boolean()) :: list(User.t()) def get_notified_from_activity(activity, local_only \\ true) def get_notified_from_activity(%Activity{data: %{"type" => type}} = activity, local_only) @@ -520,7 +517,25 @@ defmodule Pleroma.Notification do Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) end - def get_notified_from_activity(_, _local_only), do: {[], []} + def get_notified_from_activity(_, _local_only), do: [] + + def get_notified_subscribers_from_activity(activity, local_only \\ true) + + def get_notified_subscribers_from_activity( + %Activity{data: %{"type" => "Create"}} = activity, + local_only + ) do + notification_enabled_ap_ids = + [] + |> Utils.maybe_notify_subscribers(activity) + + potential_receivers = + User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only) + + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + end + + def get_notified_subscribers_from_activity(_, _), do: [] # For some activities, only notify the author of the object def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}}) @@ -563,7 +578,6 @@ defmodule Pleroma.Notification do [] |> Utils.maybe_notify_to_recipients(activity) |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) |> Utils.maybe_notify_followers(activity) |> Enum.uniq() end @@ -720,7 +734,7 @@ defmodule Pleroma.Notification do def mark_as_read?(activity, target_user) do user = Activity.user_actor(activity) - User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(target_user, activity) + User.mutes_user?(target_user, user) || CommonAPI.thread_muted?(activity, target_user) end def for_user_and_activity(user, activity) do @@ -743,8 +757,9 @@ defmodule Pleroma.Notification do |> Repo.update_all(set: [seen: true]) end - @spec send(list(Notification.t())) :: :ok - def send(notifications) do + @doc "Streams a list of notifications over websockets and web push" + @spec stream(list(Notification.t())) :: :ok + def stream(notifications) do Enum.each(notifications, fn notification -> Streamer.stream(["user", "user:notification"], notification) Push.send(notification) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 55b646b12..eb44b3855 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -99,21 +99,24 @@ defmodule Pleroma.Object do def get_by_id(nil), do: nil def get_by_id(id), do: Repo.get(Object, id) + @spec get_by_id_and_maybe_refetch(integer(), list()) :: Object.t() | nil def get_by_id_and_maybe_refetch(id, opts \\ []) do - %{updated_at: updated_at} = object = get_by_id(id) - - if opts[:interval] && - NaiveDateTime.diff(NaiveDateTime.utc_now(), updated_at) > opts[:interval] do - case Fetcher.refetch_object(object) do - {:ok, %Object{} = object} -> - object - - e -> - Logger.error("Couldn't refresh #{object.data["id"]}:\n#{inspect(e)}") - object + with %Object{updated_at: updated_at} = object <- get_by_id(id) do + if opts[:interval] && + NaiveDateTime.diff(NaiveDateTime.utc_now(), updated_at) > opts[:interval] do + case Fetcher.refetch_object(object) do + {:ok, %Object{} = object} -> + object + + e -> + Logger.error("Couldn't refresh #{object.data["id"]}:\n#{inspect(e)}") + object + end + else + object end else - object + nil -> nil end end diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index af5642af4..c0f671dd4 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -59,6 +59,7 @@ defmodule Pleroma.Object.Fetcher do end # Note: will create a Create activity, which we need internally at the moment. + @spec fetch_object_from_id(String.t(), list()) :: {:ok, Object.t()} | {:error | :reject, any()} def fetch_object_from_id(id, options \\ []) do with {_, nil} <- {:fetch_object, Object.get_cached_by_ap_id(id)}, {_, true} <- {:allowed_depth, Federator.allowed_thread_distance?(options[:depth])}, diff --git a/lib/pleroma/object/updater.ex b/lib/pleroma/object/updater.ex index b1e4870ba..b80bc7faf 100644 --- a/lib/pleroma/object/updater.ex +++ b/lib/pleroma/object/updater.ex @@ -134,7 +134,10 @@ defmodule Pleroma.Object.Updater do else %{updated_object: updated_data} = updated_data - |> maybe_update_history(original_data, updated: updated, use_history_in_new_object?: false) + |> maybe_update_history(original_data, + updated: updated, + use_history_in_new_object?: false + ) updated_data |> Map.put("updated", date) diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 4d13e51fc..8aec4ae58 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -8,7 +8,7 @@ defmodule Pleroma.ReverseProxy do ~w(if-unmodified-since if-none-match) ++ @range_headers @resp_cache_headers ~w(etag date last-modified) @keep_resp_headers @resp_cache_headers ++ - ~w(content-type content-disposition content-encoding) ++ + ~w(content-length content-type content-disposition content-encoding) ++ ~w(content-range accept-ranges vary) @default_cache_control_header "public, max-age=1209600" @valid_resp_codes [200, 206, 304] @@ -180,6 +180,7 @@ defmodule Pleroma.ReverseProxy do result = conn |> put_resp_headers(build_resp_headers(headers, opts)) + |> streaming_compat |> send_chunked(status) |> chunk_reply(client, opts) @@ -417,4 +418,17 @@ defmodule Pleroma.ReverseProxy do @cachex.put(:failed_proxy_url_cache, url, true, ttl: ttl) end + + # When Cowboy handles a chunked response with a content-length header it streams + # over HTTP 1.1 instead of chunking. Bandit cannot stream over HTTP 1.1 so the header + # must be stripped or it breaks RFC compliance for Transfer Encoding: Chunked. RFC9112§6.2 + # + # HTTP2 is always streamed for all adapters. + defp streaming_compat(conn) do + with Phoenix.Endpoint.Cowboy2Adapter <- Pleroma.Web.Endpoint.config(:adapter) do + conn + else + _ -> delete_resp_header(conn, "content-length") + end + end end diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 63c6cb45b..c361d7d89 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -204,7 +204,7 @@ defmodule Pleroma.ScheduledActivity do def job_query(scheduled_activity_id) do from(j in Oban.Job, - where: j.queue == "scheduled_activities", + where: j.queue == "federator_outgoing", where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) ) end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 3b266e59b..b9d2a0188 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -10,8 +10,12 @@ defmodule Pleroma.Search do end def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - + search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.search(options[:for_user], query, options) end + + def healthcheck_endpoints do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + search_module.healthcheck_endpoints() + end end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 31bfc7e33..aef5d1e74 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -28,7 +28,7 @@ defmodule Pleroma.Search.DatabaseSearch do |> Activity.with_preloaded_object() |> Activity.restrict_deactivated_users() |> restrict_public(user) - |> query_with(index_type, search_query, :websearch) + |> query_with(index_type, search_query) |> maybe_restrict_local(user) |> maybe_restrict_author(author) |> maybe_restrict_blocked(user) @@ -48,6 +48,15 @@ defmodule Pleroma.Search.DatabaseSearch do @impl true def remove_from_index(_object), do: :ok + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + + @impl true + def healthcheck_endpoints, do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end @@ -79,25 +88,7 @@ defmodule Pleroma.Search.DatabaseSearch do ) end - defp query_with(q, :gin, search_query, :plain) do - %{rows: [[tsc]]} = - Ecto.Adapters.SQL.query!( - Pleroma.Repo, - "select current_setting('default_text_search_config')::regconfig::oid;" - ) - - from([a, o] in q, - where: - fragment( - "to_tsvector(?::oid::regconfig, ?->>'content') @@ plainto_tsquery(?)", - ^tsc, - o.data, - ^search_query - ) - ) - end - - defp query_with(q, :gin, search_query, :websearch) do + defp query_with(q, :gin, search_query) do %{rows: [[tsc]]} = Ecto.Adapters.SQL.query!( Pleroma.Repo, @@ -115,19 +106,7 @@ defmodule Pleroma.Search.DatabaseSearch do ) end - defp query_with(q, :rum, search_query, :plain) do - from([a, o] in q, - where: - fragment( - "? @@ plainto_tsquery(?)", - o.fts_content, - ^search_query - ), - order_by: [fragment("? <=> now()::date", o.inserted_at)] - ) - end - - defp query_with(q, :rum, search_query, :websearch) do + defp query_with(q, :rum, search_query) do from([a, o] in q, where: fragment( diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex new file mode 100644 index 000000000..e562c8478 --- /dev/null +++ b/lib/pleroma/search/healthcheck.ex @@ -0,0 +1,86 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.Search.Healthcheck do + @doc """ + Monitors health of search backend to control processing of events based on health and availability. + """ + use GenServer + require Logger + + @queue :search_indexing + @tick :timer.seconds(5) + @timeout :timer.seconds(2) + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_) do + state = %{healthy: false} + {:ok, state, {:continue, :start}} + end + + @impl true + def handle_continue(:start, state) do + tick() + {:noreply, state} + end + + @impl true + def handle_info(:check, state) do + urls = Pleroma.Search.healthcheck_endpoints() + + new_state = + if check(urls) do + Oban.resume_queue(queue: @queue) + Map.put(state, :healthy, true) + else + Oban.pause_queue(queue: @queue) + Map.put(state, :healthy, false) + end + + maybe_log_state_change(state, new_state) + + tick() + {:noreply, new_state} + end + + @impl true + def handle_call(:state, _from, state) do + {:reply, state, state, :hibernate} + end + + def state, do: GenServer.call(__MODULE__, :state) + + def check([]), do: true + + def check(urls) when is_list(urls) do + Enum.all?( + urls, + fn url -> + case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do + {:ok, %{status: 200}} -> true + _ -> false + end + end + ) + end + + def check(_), do: true + + defp tick do + Process.send_after(self(), :check, @tick) + end + + defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do + Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure") + end + + defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do + Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass") + end + + defp maybe_log_state_change(_, _), do: :ok +end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2bff663e8..9bba5b30f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do @behaviour Pleroma.Search.SearchBackend + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + defp meili_headers do private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) @@ -178,4 +184,15 @@ defmodule Pleroma.Search.Meilisearch do def remove_from_index(object) do meili_delete("/indexes/objects/documents/#{object.id}") end + + @impl true + def healthcheck_endpoints do + endpoint = + Config.get([Pleroma.Search.Meilisearch, :url]) + |> URI.parse() + |> Map.put(:path, "/health") + |> URI.to_string() + + [endpoint] + end end diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex new file mode 100644 index 000000000..b659bb682 --- /dev/null +++ b/lib/pleroma/search/qdrant_search.ex @@ -0,0 +1,182 @@ +defmodule Pleroma.Search.QdrantSearch do + @behaviour Pleroma.Search.SearchBackend + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config + + alias __MODULE__.OpenAIClient + alias __MODULE__.QdrantClient + + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3] + + @impl true + def create_index do + payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) + + with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do + :ok + else + e -> {:error, e} + end + end + + @impl true + def drop_index do + with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do + :ok + else + e -> {:error, e} + end + end + + def get_embedding(text) do + with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <- + OpenAIClient.post("/v1/embeddings", %{ + input: text, + model: Config.get([Pleroma.Search.QdrantSearch, :openai_model]) + }) do + {:ok, embedding} + else + _ -> + {:error, "Failed to get embedding"} + end + end + + defp actor_from_activity(%{data: %{"actor" => actor}}) do + actor + end + + defp actor_from_activity(_), do: nil + + defp build_index_payload(activity, embedding) do + actor = actor_from_activity(activity) + published_at = activity.data["published"] + + %{ + points: [ + %{ + id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(), + vector: embedding, + payload: %{actor: actor, published_at: published_at} + } + ] + } + end + + defp build_search_payload(embedding, options) do + base = %{ + vector: embedding, + limit: options[:limit] || 20, + offset: options[:offset] || 0 + } + + if author = options[:author] do + Map.put(base, :filter, %{ + must: [%{key: "actor", match: %{value: author.ap_id}}] + }) + else + base + end + end + + @impl true + def add_to_index(activity) do + # This will only index public or unlisted notes + maybe_search_data = object_to_search_data(activity.object) + + if activity.data["type"] == "Create" and maybe_search_data do + with {:ok, embedding} <- get_embedding(maybe_search_data.content), + {:ok, %{status: 200}} <- + QdrantClient.put( + "/collections/posts/points", + build_index_payload(activity, embedding) + ) do + :ok + else + e -> {:error, e} + end + else + :ok + end + end + + @impl true + def remove_from_index(object) do + activity = Activity.get_by_object_ap_id_with_object(object.data["id"]) + id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!() + + with {:ok, %{status: 200}} <- + QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do + :ok + else + e -> {:error, e} + end + end + + @impl true + def search(user, original_query, options) do + query = "Represent this sentence for searching relevant passages: #{original_query}" + + with {:ok, embedding} <- get_embedding(query), + {:ok, %{body: %{"result" => result}}} <- + QdrantClient.post( + "/collections/posts/points/search", + build_search_payload(embedding, options) + ) do + ids = + Enum.map(result, fn %{"id" => id} -> + Ecto.UUID.dump!(id) + end) + + from(a in Activity, where: a.id in ^ids) + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id)) + |> Pleroma.Repo.all() + |> maybe_fetch(user, original_query) + else + _ -> + [] + end + end + + @impl true + def healthcheck_endpoints do + qdrant_health = + Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]) + |> URI.parse() + |> Map.put(:path, "/healthz") + |> URI.to_string() + + openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url]) + + [qdrant_health, openai_health] |> Enum.filter(& &1) + end +end + +defmodule Pleroma.Search.QdrantSearch.OpenAIClient do + use Tesla + alias Pleroma.Config.Getting, as: Config + + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])) + plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"Authorization", + "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"} + ]) +end + +defmodule Pleroma.Search.QdrantSearch.QdrantClient do + use Tesla + alias Pleroma.Config.Getting, as: Config + + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) + plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])} + ]) +end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 68bc48cec..f4ed13c36 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -21,4 +21,22 @@ defmodule Pleroma.Search.SearchBackend do from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} + + @doc """ + Create the index + """ + @callback create_index() :: :ok | {:error, any()} + + @doc """ + Drop the index + """ + @callback drop_index() :: :ok | {:error, any()} + + @doc """ + Healthcheck endpoints of search backend infrastructure to monitor for controlling + processing of jobs in the Oban queue. + + It is expected a 200 response is healthy and other responses are unhealthy. + """ + @callback healthcheck_endpoints :: list() | nil end diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 8fd422a6e..195513478 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -10,6 +10,14 @@ defmodule Pleroma.Signature do alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub + import Plug.Conn, only: [put_req_header: 3] + + @http_signatures_impl Application.compile_env( + :pleroma, + [__MODULE__, :http_signatures_impl], + HTTPSignatures + ) + @known_suffixes ["/publickey", "/main-key"] def key_id_to_actor_id(key_id) do @@ -44,8 +52,7 @@ defmodule Pleroma.Signature do defp remove_suffix(uri, []), do: uri def fetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} else @@ -55,8 +62,7 @@ defmodule Pleroma.Signature do end def refetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} @@ -66,6 +72,16 @@ defmodule Pleroma.Signature do end end + def get_actor_id(conn) do + with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), + {:ok, actor_id} <- key_id_to_actor_id(kid) do + {:ok, actor_id} + else + e -> + {:error, e} + end + end + def sign(%User{keys: keys} = user, headers) do with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers) @@ -77,4 +93,48 @@ defmodule Pleroma.Signature do def signed_date(%NaiveDateTime{} = date) do Timex.format!(date, "{WDshort}, {0D} {Mshort} {YYYY} {h24}:{m}:{s} GMT") end + + @spec validate_signature(Plug.Conn.t(), String.t()) :: boolean() + def validate_signature(%Plug.Conn{} = conn, request_target) do + # Newer drafts for HTTP signatures now use @request-target instead of the + # old (request-target). We'll now support both for incoming signatures. + conn = + conn + |> put_req_header("(request-target)", request_target) + |> put_req_header("@request-target", request_target) + + @http_signatures_impl.validate_conn(conn) + end + + @spec validate_signature(Plug.Conn.t()) :: boolean() + def validate_signature(%Plug.Conn{} = conn) do + # This (request-target) is non-standard, but many implementations do it + # this way due to a misinterpretation of + # https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-06 + # "path" was interpreted as not having the query, though later examples + # show that it must be the absolute path + query. This behavior is kept to + # make sure most software (Pleroma itself, Mastodon, and probably others) + # do not break. + request_target = Enum.join([String.downcase(conn.method), conn.request_path], " ") + + # This is the proper way to build the @request-target, as expected by + # many HTTP signature libraries, clarified in the following draft: + # https://www.ietf.org/archive/id/draft-ietf-httpbis-message-signatures-11.html#section-2.2.6 + # It is the same as before, but containing the query part as well. + proper_target = Enum.join([request_target, "?", conn.query_string], "") + + cond do + # Normal, non-standard behavior but expected by Pleroma and more. + validate_signature(conn, request_target) -> + true + + # Has query string and the previous one failed: let's try the standard. + conn.query_string != "" -> + validate_signature(conn, proper_target) + + # If there's no query string and signature fails, it's rotten. + true -> + false + end + end end diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 9998d8185..31ce3cc20 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -39,7 +39,7 @@ defmodule Pleroma.Telemetry.Logger do _, _ ) do - Logger.error(fn -> + Logger.debug(fn -> "Connection pool failed to reclaim any connections due to all of them being in use. It will have to drop requests for opening connections to new hosts" end) end @@ -70,7 +70,7 @@ defmodule Pleroma.Telemetry.Logger do %{key: key}, _ ) do - Logger.warning(fn -> + Logger.debug(fn -> "Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{inspect(reason)}" end) end diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index e6c484548..b0aef2592 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -239,20 +239,26 @@ defmodule Pleroma.Upload do "" end - [base_url, path] - |> Path.join() + if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do + String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path) + else + [base_url, path] + |> Path.join() + end end defp url_from_spec(_upload, _base_url, {:url, url}), do: url + @spec base_url() :: binary def base_url do uploader = @config_impl.get([Pleroma.Upload, :uploader]) - upload_base_url = @config_impl.get([Pleroma.Upload, :base_url]) + upload_fallback_url = Pleroma.Web.Endpoint.url() <> "/media/" + upload_base_url = @config_impl.get([Pleroma.Upload, :base_url]) || upload_fallback_url public_endpoint = @config_impl.get([uploader, :public_endpoint]) case uploader do Pleroma.Uploaders.Local -> - upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" + upload_base_url Pleroma.Uploaders.S3 -> bucket = @config_impl.get([Pleroma.Uploaders.S3, :bucket]) @@ -264,11 +270,14 @@ defmodule Pleroma.Upload do !is_nil(truncated_namespace) -> truncated_namespace - !is_nil(namespace) -> + !is_nil(namespace) and !is_nil(bucket) -> namespace <> ":" <> bucket - true -> + !is_nil(bucket) -> bucket + + true -> + "" end if public_endpoint do @@ -277,8 +286,11 @@ defmodule Pleroma.Upload do Path.join([upload_base_url, bucket_with_namespace]) end + Pleroma.Uploaders.IPFS -> + @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + _ -> - public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" + public_endpoint || upload_base_url end end end diff --git a/lib/pleroma/upload/filter/exiftool/strip_location.ex b/lib/pleroma/upload/filter/exiftool/strip_location.ex index f2bcc4622..1744a286d 100644 --- a/lib/pleroma/upload/filter/exiftool/strip_location.ex +++ b/lib/pleroma/upload/filter/exiftool/strip_location.ex @@ -9,8 +9,6 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do """ @behaviour Pleroma.Upload.Filter - @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()} - # Formats not compatible with exiftool at this time def filter(%Pleroma.Upload{content_type: "image/heic"}), do: {:ok, :noop} def filter(%Pleroma.Upload{content_type: "image/webp"}), do: {:ok, :noop} @@ -18,7 +16,9 @@ defmodule Pleroma.Upload.Filter.Exiftool.StripLocation do def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do try do - case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", file], parallelism: true) do + case System.cmd("exiftool", ["-overwrite_original", "-gps:all=", "-png:all=", file], + parallelism: true + ) do {_response, 0} -> {:ok, :filtered} {error, 1} -> {:error, error} end diff --git a/lib/pleroma/upload/filter/mogrifun.ex b/lib/pleroma/upload/filter/mogrifun.ex index a0f247b70..9716580a8 100644 --- a/lib/pleroma/upload/filter/mogrifun.ex +++ b/lib/pleroma/upload/filter/mogrifun.ex @@ -38,7 +38,6 @@ defmodule Pleroma.Upload.Filter.Mogrifun do [{"fill", "yellow"}, {"tint", "40"}] ] - @spec filter(Pleroma.Upload.t()) :: {:ok, atom()} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do try do Filter.Mogrify.do_filter(file, [Enum.random(@filters)]) diff --git a/lib/pleroma/upload/filter/mogrify.ex b/lib/pleroma/upload/filter/mogrify.ex index 06efbf321..d1e166022 100644 --- a/lib/pleroma/upload/filter/mogrify.ex +++ b/lib/pleroma/upload/filter/mogrify.ex @@ -8,7 +8,6 @@ defmodule Pleroma.Upload.Filter.Mogrify do @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} @type conversions :: conversion() | [conversion()] - @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _}) do try do do_filter(file, Pleroma.Config.get!([__MODULE__, :args])) diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex new file mode 100644 index 000000000..5930a129e --- /dev/null +++ b/lib/pleroma/uploaders/ipfs.ex @@ -0,0 +1,72 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFS do + @behaviour Pleroma.Uploaders.Uploader + require Logger + + alias Tesla.Multipart + + @api_add "/api/v0/add" + @api_delete "/api/v0/files/rm" + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + + @placeholder "{CID}" + def placeholder, do: @placeholder + + @impl true + def get_file(file) do + b_url = Pleroma.Upload.base_url() + + if String.contains?(b_url, @placeholder) do + {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}} + else + {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} + end + end + + @impl true + def put_file(%Pleroma.Upload{tempfile: tempfile}) do + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(tempfile) + + endpoint = ipfs_endpoint(@api_add) + + with {:ok, %{body: body}} when is_binary(body) <- + Pleroma.HTTP.post(endpoint, mp, [], params: ["cid-version": "1"], pool: :upload), + {_, {:ok, decoded}} <- {:json, Jason.decode(body)}, + {_, true} <- {:hash, Map.has_key?(decoded, "Hash")} do + {:ok, {:file, decoded["Hash"]}} + else + {:hash, false} -> + {:error, "JSON doesn't contain Hash key"} + + {:json, error} -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "JSON decode failed"} + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "IPFS Gateway upload failed"} + end + end + + @impl true + def delete_file(file) do + endpoint = ipfs_endpoint(@api_delete) + + case Pleroma.HTTP.post(endpoint, "", [], params: [arg: file]) do + {:ok, %{status: 204}} -> :ok + error -> {:error, inspect(error)} + end + end + + defp ipfs_endpoint(path) do + URI.parse(@config_impl.get([__MODULE__, :post_gateway_url])) + |> Map.put(:path, path) + |> URI.to_string() + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 778e20526..e28d76a7c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -38,6 +38,8 @@ defmodule Pleroma.User do alias Pleroma.Web.OAuth alias Pleroma.Web.RelMe alias Pleroma.Workers.BackgroundWorker + alias Pleroma.Workers.DeleteWorker + alias Pleroma.Workers.UserRefreshWorker require Logger require Pleroma.Constants @@ -1404,6 +1406,40 @@ defmodule Pleroma.User do |> Repo.all() end + @spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do + friends = + get_friends_query(current_user) + |> where([u], not u.hide_follows) + |> select([u], u.id) + + User.Query.build(%{is_active: true}) + |> where([u], u.id not in ^[user.id, current_user.id]) + |> join(:inner, [u], r in FollowingRelationship, + as: :followers_relationships, + on: r.following_id == ^user.id and r.follower_id == u.id + ) + |> where([followers_relationships: r], r.state == ^:follow_accept) + |> where([followers_relationships: r], r.follower_id in subquery(friends)) + end + + def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do + user + |> get_familiar_followers_query(current_user, nil) + |> User.Query.paginate(page, 20) + end + + @spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user), + do: get_familiar_followers_query(user, current_user, nil) + + @spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())} + def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do + user + |> get_familiar_followers_query(current_user, page) + |> Repo.all() + end + def increase_note_count(%User{} = user) do User |> where(id: ^user.id) @@ -1947,7 +1983,7 @@ defmodule Pleroma.User do def delete(%User{} = user) do # Purge the user immediately purge(user) - BackgroundWorker.enqueue("delete_user", %{"user_id" => user.id}) + DeleteWorker.enqueue("delete_user", %{"user_id" => user.id}) end # *Actually* delete the user from the DB @@ -2019,7 +2055,8 @@ defmodule Pleroma.User do %{scheme: scheme, userinfo: nil, host: host} when not_empty_string(host) and scheme in ["http", "https"] <- URI.parse(value), - {:not_idn, true} <- {:not_idn, to_string(:idna.encode(host)) == host}, + {:not_idn, true} <- + {:not_idn, match?(^host, to_string(:idna.encode(to_charlist(host))))}, "me" <- Pleroma.Web.RelMe.maybe_put_rel_me(value, profile_urls) do CommonUtils.to_masto_date(NaiveDateTime.utc_now()) else @@ -2119,20 +2156,20 @@ defmodule Pleroma.User do def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) + @spec get_or_fetch_by_ap_id(String.t()) :: {:ok, User.t()} | {:error, any()} def get_or_fetch_by_ap_id(ap_id) do - cached_user = get_cached_by_ap_id(ap_id) - - maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id) - - case {cached_user, maybe_fetched_user} do - {_, {:ok, %User{} = user}} -> - {:ok, user} - - {%User{} = user, _} -> - {:ok, user} + with cached_user = %User{} <- get_cached_by_ap_id(ap_id), + _ <- maybe_refresh(cached_user) do + {:ok, cached_user} + else + _ -> fetch_by_ap_id(ap_id) + end + end - _ -> - {:error, :not_found} + defp maybe_refresh(user) do + if needs_update?(user) do + UserRefreshWorker.new(%{"ap_id" => user.ap_id}) + |> Oban.insert() end end @@ -2693,7 +2730,7 @@ defmodule Pleroma.User do end end - @spec add_to_block(User.t(), User.t()) :: + @spec remove_from_block(User.t(), User.t()) :: {:ok, UserRelationship.t()} | {:ok, nil} | {:error, Ecto.Changeset.t()} defp remove_from_block(%User{} = user, %User{} = blocked) do with {:ok, relationship} <- UserRelationship.delete_block(user, blocked) do diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 65e0baccd..1821de667 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -197,12 +197,12 @@ defmodule Pleroma.User.Backup do end @files [ - 'actor.json', - 'outbox.json', - 'likes.json', - 'bookmarks.json', - 'followers.json', - 'following.json' + ~c"actor.json", + ~c"outbox.json", + ~c"likes.json", + ~c"bookmarks.json", + ~c"followers.json", + ~c"following.json" ] @spec export(Pleroma.User.Backup.t(), pid()) :: {:ok, String.t()} | :error def export(%__MODULE__{} = backup, caller_pid) do diff --git a/lib/pleroma/user/import.ex b/lib/pleroma/user/import.ex index 4baa7e3a4..53ffd1ab3 100644 --- a/lib/pleroma/user/import.ex +++ b/lib/pleroma/user/import.ex @@ -31,7 +31,7 @@ defmodule Pleroma.User.Import do identifiers, fn identifier -> with {:ok, %User{} = blocked} <- User.get_or_fetch(identifier), - {:ok, _block} <- CommonAPI.block(blocker, blocked) do + {:ok, _block} <- CommonAPI.block(blocked, blocker) do blocked else error -> handle_error(:blocks_import, identifier, error) @@ -46,7 +46,7 @@ defmodule Pleroma.User.Import do fn identifier -> with {:ok, %User{} = followed} <- User.get_or_fetch(identifier), {:ok, follower, followed} <- User.maybe_direct_follow(follower, followed), - {:ok, _, _, _} <- CommonAPI.follow(follower, followed) do + {:ok, _, _, _} <- CommonAPI.follow(followed, follower) do followed else error -> handle_error(:follow_import, identifier, error) diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index 7a8b176cd..e7e7e96f9 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -163,7 +163,7 @@ defmodule Pleroma.Web do """ def safe_render_many(collection, view, template, assigns \\ %{}) do Enum.map(collection, fn resource -> - as = Map.get(assigns, :as) || view.__resource__ + as = Map.get(assigns, :as) || view.__resource__() assigns = Map.put(assigns, as, resource) safe_render(view, template, assigns) end) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 643877268..b30b0cabe 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -201,7 +201,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def notify_and_stream(activity) do {:ok, notifications} = Notification.create_notifications(activity) - Notification.send(notifications) + Notification.stream(notifications) original_activity = case activity do @@ -979,8 +979,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_replies(query, %{exclude_replies: true}) do from( - [_activity, object] in query, - where: fragment("?->>'inReplyTo' is null", object.data) + [activity, object] in query, + where: + fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data) ) end @@ -1660,7 +1661,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do }} else {:error, _} = e -> e - e -> {:error, e} end end @@ -1793,24 +1793,25 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def pinned_fetch_task(nil), do: nil - - def pinned_fetch_task(%{pinned_objects: pins}) do - if Enum.all?(pins, fn {ap_id, _} -> - Object.get_cached_by_ap_id(ap_id) || - match?({:ok, _object}, Fetcher.fetch_object_from_id(ap_id)) - end) do - :ok - else - :error - end + def enqueue_pin_fetches(%{pinned_objects: pins}) do + # enqueue a task to fetch all pinned objects + Enum.each(pins, fn {ap_id, _} -> + if is_nil(Object.get_cached_by_ap_id(ap_id)) do + Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ + "id" => ap_id, + "depth" => 1 + }) + end + end) end + def enqueue_pin_fetches(_), do: nil + def make_user_from_ap_id(ap_id, additional \\ []) do user = User.get_cached_by_ap_id(ap_id) with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do - {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) + enqueue_pin_fetches(data) if user do user diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index e38a94966..cdd054e1a 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do when action in [:activity, :object] ) + plug(:log_inbox_metadata when action in [:inbox]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -292,8 +293,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do json(conn, "ok") end - def inbox(%{assigns: %{valid_signature: false}, req_headers: req_headers} = conn, params) do - Federator.incoming_ap_doc(%{req_headers: req_headers, params: params}) + def inbox(%{assigns: %{valid_signature: false}} = conn, params) do + Federator.incoming_ap_doc(%{ + method: conn.method, + req_headers: conn.req_headers, + request_path: conn.request_path, + params: params, + query_string: conn.query_string + }) + json(conn, "ok") end @@ -521,6 +529,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do conn end + defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do + Logger.metadata(actor: actor, type: type) + conn + end + + defp log_inbox_metadata(conn, _), do: conn + def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do with {:ok, object} <- ActivityPub.upload( diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index 1071f8e6e..bc418d908 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -204,7 +204,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do if function_exported?(policy, :config_description, 0) do description = @default_description - |> Map.merge(policy.config_description) + |> Map.merge(policy.config_description()) |> Map.put(:group, :pleroma) |> Map.put(:tab, :mrf) |> Map.put(:type, :group) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex new file mode 100644 index 000000000..531e75ce8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + alias Pleroma.Config + alias Pleroma.User + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + defp user_has_posted?(%User{} = u), do: u.note_count > 0 + + defp user_has_age?(%User{} = u) do + user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000) + diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond) + diff >= user_age_limit + end + + defp good_reputation?(%User{} = u) do + user_has_age?(u) and user_has_posted?(u) + end + + # copied from HellthreadPolicy + defp get_recipient_count(message) do + recipients = (message["to"] || []) ++ (message["cc"] || []) + + follower_collection = + User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address + + if Enum.member?(recipients, Pleroma.Constants.as_public()) do + recipients = + recipients + |> List.delete(Pleroma.Constants.as_public()) + |> List.delete(follower_collection) + + {:public, length(recipients)} + else + recipients = + recipients + |> List.delete(follower_collection) + + {:not_public, length(recipients)} + end + end + + defp object_has_recipients?(%{"object" => object} = activity) do + {_, object_count} = get_recipient_count(object) + {_, activity_count} = get_recipient_count(activity) + object_count + activity_count > 0 + end + + defp object_has_recipients?(object) do + {_, count} = get_recipient_count(object) + count > 0 + end + + @impl true + def filter(%{"type" => "Create", "actor" => actor} = activity) do + with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), + {:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)}, + {:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do + {:ok, activity} + else + {:ok, %User{local: true}} -> + {:ok, activity} + + {:has_mentions, false} -> + {:ok, activity} + + {:good_reputation, false} -> + {:reject, "[AntiMentionSpamPolicy] User rejected"} + + {:error, _} -> + {:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"} + + e -> + {:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"} + end + end + + # in all other cases, pass through + def filter(message), do: {:ok, message} + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex new file mode 100644 index 000000000..7c6bb888f --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex @@ -0,0 +1,146 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do + @moduledoc """ + Dynamic activity filtering based on an RBL database + + This MRF makes queries to a custom DNS server which will + respond with values indicating the classification of the domain + the activity originated from. This method has been widely used + in the email anti-spam industry for very fast reputation checks. + + e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK + Other values such as 127.0.0.2 may be used for specific classifications. + + Information for why the host is blocked can be stored in a corresponding TXT record. + + This method is fail-open so if the queries fail the activites are accepted. + + An example of software meant for this purpsoe is rbldnsd which can be found + at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at + https://git.pleroma.social/feld/rbldnsd + + It is highly recommended that you run your own copy of rbldnsd and use an + external mechanism to sync/share the contents of the zone file. This is + important to keep the latency on the queries as low as possible and prevent + your DNS server from being attacked so it fails and content is permitted. + """ + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Config + + require Logger + + @query_retries 1 + @query_timeout 500 + + @impl true + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) + + with {:ok, object} <- check_rbl(actor_info, object) do + {:ok, object} + else + _ -> {:reject, "[DNSRBLPolicy]"} + end + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe do + mrf_dnsrbl = + Config.get(:mrf_dnsrbl) + |> Enum.into(%{}) + + {:ok, %{mrf_dnsrbl: mrf_dnsrbl}} + end + + @impl true + def config_description do + %{ + key: :mrf_dnsrbl, + related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy", + label: "MRF DNSRBL", + description: "DNS RealTime Blackhole Policy", + children: [ + %{ + key: :nameserver, + type: {:string}, + description: "DNSRBL Nameserver to Query (IP or hostame)", + suggestions: ["127.0.0.1"] + }, + %{ + key: :port, + type: {:string}, + description: "Nameserver port", + suggestions: ["53"] + }, + %{ + key: :zone, + type: {:string}, + description: "Root zone for querying", + suggestions: ["bl.pleroma.com"] + } + ] + } + end + + defp check_rbl(%{host: actor_host}, object) do + with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()), + zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do + query = + Enum.join([actor_host, zone], ".") + |> String.to_charlist() + + rbl_response = rblquery(query) + + if Enum.empty?(rbl_response) do + {:ok, object} + else + Task.start(fn -> + reason = + case rblquery(query, :txt) do + [[result]] -> result + _ -> "undefined" + end + + Logger.warning( + "DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}" + ) + end) + + :error + end + else + _ -> {:ok, object} + end + end + + defp get_rblhost_ip(rblhost) do + case rblhost |> String.to_charlist() |> :inet_parse.address() do + {:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address() + _ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()} + end + end + + defp rblquery(query, type \\ :a) do + config = Config.get([:mrf_dnsrbl]) + + case get_rblhost_ip(config[:nameserver]) do + {:ok, rblnsip} -> + :inet_res.lookup(query, :in, type, + nameservers: [{rblnsip, config[:port]}], + timeout: @query_timeout, + retry: @query_retries + ) + + _ -> + [] + end + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 5a4a97626..55ea2683c 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do "#{__MODULE__}: Follow request from #{follower.nickname} to #{user.nickname}" ) - CommonAPI.follow(follower, user) + CommonAPI.follow(user, follower) end end) diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index c95d35bb9..0c5b53def 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @adapter_options [ - pool: :media, - recv_timeout: 10_000 - ] - @impl true def history_awareness, do: :auto @@ -27,17 +22,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") - if Pleroma.Config.get(:env) == :test do - fetch(prefetch_url) - else - ConcurrentLimiter.limit(__MODULE__, fn -> - Task.start(fn -> fetch(prefetch_url) end) - end) - end + fetch(prefetch_url) end end - defp fetch(url), do: HTTP.get(url, [], @adapter_options) + defp fetch(url) do + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) + HTTP.get(url, [], http_client_opts) + end defp preload(%{"object" => %{"attachment" => attachments}} = _message) do Enum.each(attachments, fn diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex new file mode 100644 index 000000000..451a212d4 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -0,0 +1,264 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do + @moduledoc """ + Hide, delete, or mark sensitive NSFW content with artificial intelligence. + + Requires a NSFW API server, configured like so: + + config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF, + url: "http://127.0.0.1:5000/", + threshold: 0.7, + mark_sensitive: true, + unlist: false, + reject: false + + The NSFW API server must implement an HTTP endpoint like this: + + curl http://localhost:5000/?url=https://fedi.com/images/001.jpg + + Returning a response like this: + + {"score", 0.314} + + Where a score is 0-1, with `1` being definitely NSFW. + + A good API server is here: https://github.com/EugenCepoi/nsfw_api + You can run it with Docker with a one-liner: + + docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest + + Options: + + - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/" + - `threshold`: Lowest score to take action on. Default: `0.7` + - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true` + - `unlist`: Unlist all detected NSFW content? Default: `false` + - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` + """ + alias Pleroma.Config + alias Pleroma.Constants + alias Pleroma.HTTP + alias Pleroma.User + + require Logger + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @policy :mrf_nsfw_api + + def build_request_url(url) do + Config.get([@policy, :url]) + |> URI.parse() + |> fix_path() + |> Map.put(:query, "url=#{url}") + |> URI.to_string() + end + + def parse_url(url) do + request = build_request_url(url) + + with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do + Jason.decode(body) + else + error -> + Logger.warning(""" + [NsfwApiPolicy]: The API server failed. Skipping. + #{inspect(error)} + """) + + error + end + end + + def check_url_nsfw(url) when is_binary(url) do + threshold = Config.get([@policy, :threshold]) + + case parse_url(url) do + {:ok, %{"score" => score}} when score >= threshold -> + {:nsfw, %{url: url, score: score, threshold: threshold}} + + {:ok, %{"score" => score}} -> + {:sfw, %{url: url, score: score, threshold: threshold}} + + _ -> + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + end + + def check_url_nsfw(%{"href" => url}) when is_binary(url) do + check_url_nsfw(url) + end + + def check_url_nsfw(url) do + threshold = Config.get([@policy, :threshold]) + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + + def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do + if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do + {:sfw, attachment} + else + {:nsfw, attachment} + end + end + + def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do + case check_url_nsfw(url) do + {:sfw, _} -> {:sfw, attachment} + {:nsfw, _} -> {:nsfw, attachment} + end + end + + def check_attachment_nsfw(attachment), do: {:sfw, attachment} + + def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do + if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do + {:sfw, object} + else + {:nsfw, object} + end + end + + def check_object_nsfw(%{"object" => %{} = child_object} = object) do + case check_object_nsfw(child_object) do + {:sfw, _} -> {:sfw, object} + {:nsfw, _} -> {:nsfw, object} + end + end + + def check_object_nsfw(object), do: {:sfw, object} + + @impl true + def filter(object) do + with {:sfw, object} <- check_object_nsfw(object) do + {:ok, object} + else + {:nsfw, _data} -> handle_nsfw(object) + end + end + + defp handle_nsfw(object) do + if Config.get([@policy, :reject]) do + {:reject, object} + else + {:ok, + object + |> maybe_unlist() + |> maybe_mark_sensitive()} + end + end + + defp maybe_unlist(object) do + if Config.get([@policy, :unlist]) do + unlist(object) + else + object + end + end + + defp maybe_mark_sensitive(object) do + if Config.get([@policy, :mark_sensitive]) do + mark_sensitive(object) + else + object + end + end + + def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + with %User{} = user <- User.get_cached_by_ap_id(actor) do + to = + [user.follower_address | to] + |> List.delete(Constants.as_public()) + |> Enum.uniq() + + cc = + [Constants.as_public() | cc] + |> List.delete(user.follower_address) + |> Enum.uniq() + + object + |> Map.put("to", to) + |> Map.put("cc", cc) + else + _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}" + end + end + + def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do + Map.put(object, "object", mark_sensitive(child_object)) + end + + def mark_sensitive(object) when is_map(object) do + tags = (object["tag"] || []) ++ ["nsfw"] + + object + |> Map.put("tag", tags) + |> Map.put("sensitive", true) + end + + # Hackney needs a trailing slash + defp fix_path(%URI{path: path} = uri) when is_binary(path) do + path = String.trim_trailing(path, "/") <> "/" + Map.put(uri, :path, path) + end + + defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/") + + @impl true + def describe do + options = %{ + threshold: Config.get([@policy, :threshold]), + mark_sensitive: Config.get([@policy, :mark_sensitive]), + unlist: Config.get([@policy, :unlist]), + reject: Config.get([@policy, :reject]) + } + + {:ok, %{@policy => options}} + end + + @impl true + def config_description do + %{ + key: @policy, + related_policy: to_string(__MODULE__), + label: "NSFW API Policy", + description: + "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.", + children: [ + %{ + key: :url, + type: :string, + description: "Base URL of the API server.", + suggestions: ["http://127.0.0.1:5000/"] + }, + %{ + key: :threshold, + type: :float, + description: "Lowest score to take action on. Between 0 and 1.", + suggestions: [0.7] + }, + %{ + key: :mark_sensitive, + type: :boolean, + description: "Mark sensitive all detected NSFW content?", + suggestions: [true] + }, + %{ + key: :unlist, + type: :boolean, + description: "Unlist sensitive all detected NSFW content?", + suggestions: [false] + }, + %{ + key: :reject, + type: :boolean, + description: "Reject sensitive all detected NSFW content (takes precedence)?", + suggestions: [false] + } + ] + } + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 829ddeaea..d708c99eb 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -220,9 +220,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_object(object) do {:ok, object} else - {:reject, nil} -> {:reject, "[SimplePolicy]"} {:reject, _} = e -> e - _ -> {:reject, "[SimplePolicy]"} end end @@ -236,9 +234,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_banner_removal(actor_info, object) do {:ok, object} else - {:reject, nil} -> {:reject, "[SimplePolicy]"} {:reject, _} = e -> e - _ -> {:reject, "[SimplePolicy]"} end end @@ -249,9 +245,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_reject(uri, object) do {:ok, object} else - {:reject, nil} -> {:reject, "[SimplePolicy]"} {:reject, _} = e -> e - _ -> {:reject, "[SimplePolicy]"} end end diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index d9deff35f..1c114558e 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -31,7 +31,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do {:reject, _} = e -> e {:accepted, _} -> {:reject, "[VocabularyPolicy] #{message_type} not in accept list"} {:rejected, _} -> {:reject, "[VocabularyPolicy] #{message_type} in reject list"} - _ -> {:reject, "[VocabularyPolicy]"} end end diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 72975f348..5ee9e7549 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string, default: "Link") field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:name, :string) + field(:summary, :string) field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do @@ -44,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do |> fix_url() struct - |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash]) |> cast_embed(:url, with: &url_changeset/2, required: true) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType]) diff --git a/lib/pleroma/web/activity_pub/pipeline.ex b/lib/pleroma/web/activity_pub/pipeline.ex index 40184bd97..7f11a4d67 100644 --- a/lib/pleroma/web/activity_pub/pipeline.ex +++ b/lib/pleroma/web/activity_pub/pipeline.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.ActivityPub.Pipeline do defp config, do: Config.get([:pipeline, :config], Config) @spec common_pipeline(map(), keyword()) :: - {:ok, Activity.t() | Object.t(), keyword()} | {:error, any()} + {:ok, Activity.t() | Object.t(), keyword()} | {:error | :reject, any()} def common_pipeline(object, meta) do case Repo.transaction(fn -> do_common_pipeline(object, meta) end, Utils.query_timeout()) do {:ok, {:ok, activity, meta}} -> diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index a42b4844e..c8bdf2250 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -123,9 +123,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do Logger.error("Publisher failed to inbox #{inbox} with status #{code}") case response do - %{status: 403} -> {:discard, :forbidden} - %{status: 404} -> {:discard, :not_found} - %{status: 410} -> {:discard, :not_found} + %{status: 400} -> {:cancel, :bad_request} + %{status: 403} -> {:cancel, :forbidden} + %{status: 404} -> {:cancel, :not_found} + %{status: 410} -> {:cancel, :not_found} _ -> {:error, e} end diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index 91a647f29..cff234940 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do def follow(target_instance) do with %User{} = local_user <- get_actor(), {:ok, %User{} = target_user} <- User.get_or_fetch_by_ap_id(target_instance), - {:ok, _, _, activity} <- CommonAPI.follow(local_user, target_user) do + {:ok, _, _, activity} <- CommonAPI.follow(target_user, local_user) do Logger.info("relay: followed instance: #{target_instance}; id=#{activity.data["id"]}") {:ok, activity} else diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 60b4d5f1b..cc1c7a0af 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -453,7 +453,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do ) do orig_object_ap_id = updated_object["id"] orig_object = Object.get_by_ap_id(orig_object_ap_id) - orig_object_data = orig_object.data + orig_object_data = Map.get(orig_object, :data) updated_object = if meta[:local] do @@ -592,9 +592,9 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do with {:ok, _} <- Repo.delete(object), do: :ok end - defp send_notifications(meta) do + defp stream_notifications(meta) do Keyword.get(meta, :notifications, []) - |> Notification.send() + |> Notification.stream() meta end @@ -625,7 +625,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do @impl true def handle_after_transaction(meta) do meta - |> send_notifications() + |> stream_notifications() |> send_streamables() end end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index b3a3777a2..a6f711733 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -534,6 +534,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do else _ -> e end + + e -> + {:error, e} end end @@ -912,9 +915,11 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def add_emoji_tags(object), do: object - defp build_emoji_tag({name, url}) do + def build_emoji_tag({name, url}) do + url = URI.encode(url) + %{ - "icon" => %{"url" => "#{URI.encode(url)}", "type" => "Image"}, + "icon" => %{"url" => "#{url}", "type" => "Image"}, "name" => ":" <> name <> ":", "type" => "Emoji", "updated" => "1970-01-01T00:00:00Z", diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 73db9af56..f30c92abf 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -946,26 +946,14 @@ defmodule Pleroma.Web.ActivityPub.Utils do |> Repo.all() end + @spec maybe_handle_group_posts(Activity.t()) :: :ok + @doc "Automatically repeats posts for local group actor recipients" def maybe_handle_group_posts(activity) do poster = User.get_cached_by_ap_id(activity.actor) - mentions = - activity.data["to"] - |> Enum.filter(&(&1 != activity.actor)) - - mentioned_local_groups = - User.get_all_by_ap_id(mentions) - |> Enum.filter(fn user -> - user.actor_type == "Group" and - user.local and - not User.blocks?(user, poster) - end) - - mentioned_local_groups - |> Enum.each(fn group -> - Pleroma.Web.CommonAPI.repeat(activity.id, group) - end) - - :ok + User.get_recipients_from_activity(activity) + |> Enum.filter(&match?("Group", &1.actor_type)) + |> Enum.reject(&User.blocks?(&1, poster)) + |> Enum.each(&Pleroma.Web.CommonAPI.repeat(activity.id, &1)) end end diff --git a/lib/pleroma/web/admin_api/controllers/invite_controller.ex b/lib/pleroma/web/admin_api/controllers/invite_controller.ex index 7e3020f28..30dbc7e73 100644 --- a/lib/pleroma/web/admin_api/controllers/invite_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/invite_controller.ex @@ -46,7 +46,6 @@ defmodule Pleroma.Web.AdminAPI.InviteController do render(conn, "show.json", invite: updated_invite) else nil -> {:error, :not_found} - error -> error end end diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex index 43b2f209a..5d4427b84 100644 --- a/lib/pleroma/web/admin_api/controllers/rule_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Web.AdminAPI.RuleController do plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index) - action_fallback(AdminAPI.FallbackController) + action_fallback(Pleroma.Web.AdminAPI.FallbackController) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation diff --git a/lib/pleroma/web/api_spec/cast_and_validate.ex b/lib/pleroma/web/api_spec/cast_and_validate.ex index f3e8e093e..672d1c4a1 100644 --- a/lib/pleroma/web/api_spec/cast_and_validate.ex +++ b/lib/pleroma/web/api_spec/cast_and_validate.ex @@ -18,6 +18,8 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do alias OpenApiSpex.Plug.PutApiSpec alias Plug.Conn + require Logger + @impl Plug def init(opts) do opts @@ -51,6 +53,10 @@ defmodule Pleroma.Web.ApiSpec.CastAndValidate do conn {:error, reason} -> + Logger.error( + "Strict ApiSpec: request denied to #{conn.request_path} with params #{inspect(conn.params)}" + ) + opts = render_error.init(reason) conn diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 36025e47a..85f02166f 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do alias Pleroma.Web.ApiSpec.Schemas.ActorType alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.List alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -513,6 +514,48 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end + def familiar_followers_operation do + %Operation{ + tags: ["Retrieve account information"], + summary: "Followers that you follow", + operationId: "AccountController.familiar_followers", + description: + "Obtain a list of all accounts that follow a given account, filtered for accounts you follow.", + security: [%{"oAuth" => ["read:follows"]}], + parameters: [ + Operation.parameter( + :id, + :query, + %Schema{ + oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}] + }, + "Account IDs", + example: "123" + ) + ], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: %Schema{ + title: "Account", + type: :object, + properties: %{ + id: FlakeID, + accounts: %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: Account, + example: [Account.schema().example] + } + } + } + }) + } + } + end + defp create_request do %Schema{ title: "AccountCreateRequest", diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 757429d12..2dc0f66df 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -202,7 +202,11 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "pleroma:report", "move", "follow_request", - "poll" + "poll", + "status", + "update", + "admin.sign_up", + "admin.report" ], description: """ The type of event that resulted in the notification. @@ -216,6 +220,10 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do - `pleroma:emoji_reaction` - Someone reacted with emoji to your status - `pleroma:chat_mention` - Someone mentioned you in a chat message - `pleroma:report` - Someone was reported + - `status` - Someone you are subscribed to created a status + - `update` - A status you boosted has been edited + - `admin.sign_up` - Someone signed up (optionally sent to admins) + - `admin.report` - A new report has been filed """ } end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex index a994345db..0e2865191 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema - alias Pleroma.Web.ApiSpec.NotificationOperation alias Pleroma.Web.ApiSpec.Schemas.ApiError import Pleroma.Web.ApiSpec.Helpers @@ -35,12 +34,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do Operation.response( "A Notification or array of Notifications", "application/json", - %Schema{ - anyOf: [ - %Schema{type: :array, items: NotificationOperation.notification()}, - NotificationOperation.notification() - ] - } + %Schema{type: :string} ), 400 => Operation.response("Bad Request", "application/json", ApiError) } diff --git a/lib/pleroma/web/api_spec/operations/search_operation.ex b/lib/pleroma/web/api_spec/operations/search_operation.ex index 539743ba3..c2afe8e18 100644 --- a/lib/pleroma/web/api_spec/operations/search_operation.ex +++ b/lib/pleroma/web/api_spec/operations/search_operation.ex @@ -79,7 +79,9 @@ defmodule Pleroma.Web.ApiSpec.SearchOperation do %Schema{type: :string, enum: ["accounts", "hashtags", "statuses"]}, "Search type" ), - Operation.parameter(:q, :query, %Schema{type: :string}, "The search query", required: true), + Operation.parameter(:q, :query, %Schema{type: :string}, "The search query", + required: true + ), Operation.parameter( :resolve, :query, diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex index b580bc2f0..47bce07b3 100644 --- a/lib/pleroma/web/api_spec/operations/streaming_operation.ex +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -139,7 +139,7 @@ defmodule Pleroma.Web.ApiSpec.StreamingOperation do end defp get_schema(%Schema{} = schema), do: schema - defp get_schema(schema), do: schema.schema + defp get_schema(schema), do: schema.schema() defp server_sent_event_helper(name, description, type, payload, opts \\ []) do payload_type = Keyword.get(opts, :payload_type, :json) diff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex index 2871b5f99..4104ed25c 100644 --- a/lib/pleroma/web/api_spec/schemas/attachment.ex +++ b/lib/pleroma/web/api_spec/schemas/attachment.ex @@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do pleroma: %Schema{ type: :object, properties: %{ - mime_type: %Schema{type: :string, description: "mime type of the attachment"} + mime_type: %Schema{type: :string, description: "mime type of the attachment"}, + name: %Schema{ + type: :string, + description: "Name of the attachment, typically the filename" + } } } }, diff --git a/lib/pleroma/web/api_spec/schemas/chat.ex b/lib/pleroma/web/api_spec/schemas/chat.ex index a07d12865..affa25a95 100644 --- a/lib/pleroma/web/api_spec/schemas/chat.ex +++ b/lib/pleroma/web/api_spec/schemas/chat.ex @@ -68,7 +68,7 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Chat do }, "id" => "1", "unread" => 2, - "last_message" => ChatMessage.schema().example(), + "last_message" => ChatMessage.schema().example, "updated_at" => "2020-04-21T15:06:45.000Z" } }) diff --git a/lib/pleroma/web/auth/ldap_authenticator.ex b/lib/pleroma/web/auth/ldap_authenticator.ex index e8cd4491c..ea5620cf6 100644 --- a/lib/pleroma/web/auth/ldap_authenticator.ex +++ b/lib/pleroma/web/auth/ldap_authenticator.ex @@ -91,7 +91,8 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do end error -> - error + Logger.error("Could not bind LDAP user #{name}: #{inspect(error)}") + {:error, {:ldap_bind_error, error}} end end @@ -102,28 +103,38 @@ defmodule Pleroma.Web.Auth.LDAPAuthenticator do {:scope, :eldap.wholeSubtree()}, {:timeout, @search_timeout} ]) do - {:ok, {:eldap_search_result, [{:eldap_entry, _, attributes}], _}} -> - params = %{ - name: name, - nickname: name, - password: nil - } - - params = - case List.keyfind(attributes, 'mail', 0) do - {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) - _ -> params - end - - changeset = User.register_changeset_ldap(%User{}, params) + # The :eldap_search_result record structure changed in OTP 24.3 and added a controls field + # https://github.com/erlang/otp/pull/5538 + {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals}} -> + try_register(name, attributes) - case User.register(changeset) do - {:ok, user} -> user - error -> error - end + {:ok, {:eldap_search_result, [{:eldap_entry, _object, attributes}], _referrals, _controls}} -> + try_register(name, attributes) error -> - error + Logger.error("Couldn't register user because LDAP search failed: #{inspect(error)}") + {:error, {:ldap_search_error, error}} + end + end + + defp try_register(name, attributes) do + params = %{ + name: name, + nickname: name, + password: nil + } + + params = + case List.keyfind(attributes, ~c"mail", 0) do + {_, [mail]} -> Map.put_new(params, :email, :erlang.list_to_binary(mail)) + _ -> params + end + + changeset = User.register_changeset_ldap(%User{}, params) + + case User.register(changeset) do + {:ok, user} -> user + error -> error end end end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 34e480d73..06faf845e 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -19,19 +19,23 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI.ActivityDraft + import Ecto.Query, only: [where: 3] import Pleroma.Web.Gettext import Pleroma.Web.CommonAPI.Utils require Pleroma.Constants require Logger - def block(blocker, blocked) do + @spec block(User.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + def block(blocked, blocker) do with {:ok, block_data, _} <- Builder.block(blocker, blocked), {:ok, block, _} <- Pipeline.common_pipeline(block_data, local: true) do {:ok, block} end end + @spec post_chat_message(User.t(), User.t(), String.t(), list()) :: + {:ok, Activity.t()} | {:error, any()} def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]), :ok <- validate_chat_attachment_attribution(maybe_attachment, user), @@ -95,7 +99,8 @@ defmodule Pleroma.Web.CommonAPI do end end - def unblock(blocker, blocked) do + @spec unblock(User.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + def unblock(blocked, blocker) do with {_, %Activity{} = block} <- {:fetch_block, Utils.fetch_latest_block(blocker, blocked)}, {:ok, unblock_data, _} <- Builder.undo(blocker, block), {:ok, unblock, _} <- Pipeline.common_pipeline(unblock_data, local: true) do @@ -114,7 +119,9 @@ defmodule Pleroma.Web.CommonAPI do end end - def follow(follower, followed) do + @spec follow(User.t(), User.t()) :: + {:ok, User.t(), User.t(), Activity.t() | Object.t()} | {:error, :rejected} + def follow(followed, follower) do timeout = Pleroma.Config.get([:activitypub, :follow_handshake_timeout]) with {:ok, follow_data, _} <- Builder.follow(follower, followed), @@ -128,7 +135,8 @@ defmodule Pleroma.Web.CommonAPI do end end - def unfollow(follower, unfollowed) do + @spec unfollow(User.t(), User.t()) :: {:ok, User.t()} | {:error, any()} + def unfollow(unfollowed, follower) do with {:ok, follower, _follow_activity} <- User.unfollow(follower, unfollowed), {:ok, _activity} <- ActivityPub.unfollow(follower, unfollowed), {:ok, _subscription} <- User.unsubscribe(follower, unfollowed), @@ -137,6 +145,7 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec accept_follow_request(User.t(), User.t()) :: {:ok, User.t()} | {:error, any()} def accept_follow_request(follower, followed) do with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), {:ok, accept_data, _} <- Builder.accept(followed, follow_activity), @@ -145,6 +154,7 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec reject_follow_request(User.t(), User.t()) :: {:ok, User.t()} | {:error, any()} | nil def reject_follow_request(follower, followed) do with %Activity{} = follow_activity <- Utils.fetch_latest_follow(follower, followed), {:ok, reject_data, _} <- Builder.reject(followed, follow_activity), @@ -153,9 +163,11 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec delete(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} def delete(activity_id, user) do with {_, %Activity{data: %{"object" => _, "type" => "Create"}} = activity} <- {:find_activity, Activity.get_by_id(activity_id, filter: [])}, + {_, {:ok, _}} <- {:cancel_jobs, maybe_cancel_jobs(activity)}, {_, %Object{} = object, _} <- {:find_object, Object.normalize(activity, fetch: false), activity}, true <- User.privileged?(user, :messages_delete) || user.ap_id == object.data["actor"], @@ -201,6 +213,7 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec repeat(String.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} def repeat(id, user, params \\ %{}) do with %Activity{data: %{"type" => "Create"}} = activity <- Activity.get_by_id(id), object = %Object{} <- Object.normalize(activity, fetch: false), @@ -218,11 +231,13 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec unrepeat(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} def unrepeat(id, user) do with {_, %Activity{data: %{"type" => "Create"}} = activity} <- {:find_activity, Activity.get_by_id(id)}, %Object{} = note <- Object.normalize(activity, fetch: false), %Activity{} = announce <- Utils.get_existing_announce(user.ap_id, note), + {_, {:ok, _}} <- {:cancel_jobs, maybe_cancel_jobs(announce)}, {:ok, undo, _} <- Builder.undo(user, announce), {:ok, activity, _} <- Pipeline.common_pipeline(undo, local: true) do {:ok, activity} @@ -232,8 +247,8 @@ defmodule Pleroma.Web.CommonAPI do end end - @spec favorite(User.t(), binary()) :: {:ok, Activity.t() | :already_liked} | {:error, any()} - def favorite(%User{} = user, id) do + @spec favorite(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + def favorite(id, %User{} = user) do case favorite_helper(user, id) do {:ok, _} = res -> res @@ -247,7 +262,7 @@ defmodule Pleroma.Web.CommonAPI do end end - def favorite_helper(user, id) do + defp favorite_helper(user, id) do with {_, %Activity{object: object}} <- {:find_object, Activity.get_by_id_with_object(id)}, {_, {:ok, like_object, meta}} <- {:build_object, Builder.like(user, object)}, {_, {:ok, %Activity{} = activity, _meta}} <- @@ -270,11 +285,13 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec unfavorite(String.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} def unfavorite(id, user) do with {_, %Activity{data: %{"type" => "Create"}} = activity} <- {:find_activity, Activity.get_by_id(id)}, %Object{} = note <- Object.normalize(activity, fetch: false), %Activity{} = like <- Utils.get_existing_like(user.ap_id, note), + {_, {:ok, _}} <- {:cancel_jobs, maybe_cancel_jobs(like)}, {:ok, undo, _} <- Builder.undo(user, like), {:ok, activity, _} <- Pipeline.common_pipeline(undo, local: true) do {:ok, activity} @@ -284,6 +301,8 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec react_with_emoji(String.t(), User.t(), String.t()) :: + {:ok, Activity.t()} | {:error, any()} def react_with_emoji(id, user, emoji) do with %Activity{} = activity <- Activity.get_by_id(id), object <- Object.normalize(activity, fetch: false), @@ -296,8 +315,11 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec unreact_with_emoji(String.t(), User.t(), String.t()) :: + {:ok, Activity.t()} | {:error, any()} def unreact_with_emoji(id, user, emoji) do with %Activity{} = reaction_activity <- Utils.get_latest_reaction(id, user, emoji), + {_, {:ok, _}} <- {:cancel_jobs, maybe_cancel_jobs(reaction_activity)}, {:ok, undo, _} <- Builder.undo(user, reaction_activity), {:ok, activity, _} <- Pipeline.common_pipeline(undo, local: true) do {:ok, activity} @@ -307,7 +329,8 @@ defmodule Pleroma.Web.CommonAPI do end end - def vote(user, %{data: %{"type" => "Question"}} = object, choices) do + @spec vote(Object.t(), User.t(), list()) :: {:ok, list(), Object.t()} | {:error, any()} + def vote(%Object{data: %{"type" => "Question"}} = object, %User{} = user, choices) do with :ok <- validate_not_author(object, user), :ok <- validate_existing_votes(user, object), {:ok, options, choices} <- normalize_and_validate_choices(choices, object) do @@ -368,14 +391,16 @@ defmodule Pleroma.Web.CommonAPI do end end - def public_announce?(_, %{visibility: visibility}) - when visibility in ~w{public unlisted private direct}, - do: visibility in ~w(public unlisted) + defp public_announce?(_, %{visibility: visibility}) + when visibility in ~w{public unlisted private direct}, + do: visibility in ~w(public unlisted) - def public_announce?(object, _) do + defp public_announce?(object, _) do Visibility.public?(object) end + @spec get_visibility(map(), map() | nil, Participation.t() | nil) :: + {String.t() | nil, String.t() | nil} def get_visibility(_, _, %Participation{}), do: {"direct", "direct"} def get_visibility(%{visibility: visibility}, in_reply_to, _) @@ -394,6 +419,7 @@ defmodule Pleroma.Web.CommonAPI do def get_visibility(_, in_reply_to, _), do: {"public", get_replied_to_visibility(in_reply_to)} + @spec get_replied_to_visibility(Activity.t() | nil) :: String.t() | nil def get_replied_to_visibility(nil), do: nil def get_replied_to_visibility(activity) do @@ -402,6 +428,8 @@ defmodule Pleroma.Web.CommonAPI do end end + @spec check_expiry_date({:ok, nil | integer()} | String.t()) :: + {:ok, boolean() | nil} | {:error, String.t()} def check_expiry_date({:ok, nil} = res), do: res def check_expiry_date({:ok, in_seconds}) do @@ -419,19 +447,22 @@ defmodule Pleroma.Web.CommonAPI do |> check_expiry_date() end + @spec listen(User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} def listen(user, data) do with {:ok, draft} <- ActivityDraft.listen(user, data) do ActivityPub.listen(draft.changes) end end + @spec post(User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} def post(user, %{status: _} = data) do with {:ok, draft} <- ActivityDraft.create(user, data) do ActivityPub.create(draft.changes, draft.preview?) end end - def update(user, orig_activity, changes) do + @spec update(Activity.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} + def update(orig_activity, %User{} = user, changes) do with orig_object <- Object.normalize(orig_activity), {:ok, new_object} <- make_update_data(user, orig_object, changes), {:ok, update_data, _} <- Builder.update(user, new_object), @@ -521,7 +552,8 @@ defmodule Pleroma.Web.CommonAPI do end end - def add_mute(user, activity, params \\ %{}) do + @spec add_mute(Activity.t(), User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} + def add_mute(activity, user, params \\ %{}) do expires_in = Map.get(params, :expires_in, 0) with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]), @@ -540,15 +572,17 @@ defmodule Pleroma.Web.CommonAPI do end end - def remove_mute(%User{} = user, %Activity{} = activity) do + @spec remove_mute(Activity.t(), User.t()) :: {:ok, Activity.t()} | {:error, any()} + def remove_mute(%Activity{} = activity, %User{} = user) do ThreadMute.remove_mute(user.id, activity.data["context"]) {:ok, activity} end - def remove_mute(user_id, activity_id) do + @spec remove_mute(String.t(), String.t()) :: {:ok, Activity.t()} | {:error, any()} + def remove_mute(activity_id, user_id) do with {:user, %User{} = user} <- {:user, User.get_by_id(user_id)}, {:activity, %Activity{} = activity} <- {:activity, Activity.get_by_id(activity_id)} do - remove_mute(user, activity) + remove_mute(activity, user) else {what, result} = error -> Logger.warning( @@ -559,13 +593,15 @@ defmodule Pleroma.Web.CommonAPI do end end - def thread_muted?(%User{id: user_id}, %{data: %{"context" => context}}) + @spec thread_muted?(Activity.t(), User.t()) :: boolean() + def thread_muted?(%{data: %{"context" => context}}, %User{id: user_id}) when is_binary(context) do ThreadMute.exists?(user_id, context) end def thread_muted?(_, _), do: false + @spec report(User.t(), map()) :: {:ok, Activity.t()} | {:error, any()} def report(user, data) do with {:ok, account} <- get_reported_account(data.account_id), {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]), @@ -599,6 +635,8 @@ defmodule Pleroma.Web.CommonAPI do |> Enum.filter(&Rule.exists?/1) end + @spec update_report_state(String.t() | [String.t()], String.t()) :: + {:ok, any()} | {:error, any()} def update_report_state(activity_ids, state) when is_list(activity_ids) do case Utils.update_report_state(activity_ids, state) do :ok -> {:ok, activity_ids} @@ -611,17 +649,16 @@ defmodule Pleroma.Web.CommonAPI do Utils.update_report_state(activity, state) else nil -> {:error, :not_found} - _ -> {:error, dgettext("errors", "Could not update state")} end end + @spec update_activity_scope(String.t(), map()) :: {:ok, any()} | {:error, any()} def update_activity_scope(activity_id, opts \\ %{}) do with %Activity{} = activity <- Activity.get_by_id_with_object(activity_id), {:ok, activity} <- toggle_sensitive(activity, opts) do set_visibility(activity, opts) else nil -> {:error, :not_found} - {:error, reason} -> {:error, reason} end end @@ -649,14 +686,17 @@ defmodule Pleroma.Web.CommonAPI do defp set_visibility(activity, _), do: {:ok, activity} - def hide_reblogs(%User{} = user, %User{} = target) do + @spec hide_reblogs(User.t(), User.t()) :: {:ok, any()} | {:error, any()} + def hide_reblogs(%User{} = target, %User{} = user) do UserRelationship.create_reblog_mute(user, target) end - def show_reblogs(%User{} = user, %User{} = target) do + @spec show_reblogs(User.t(), User.t()) :: {:ok, any()} | {:error, any()} + def show_reblogs(%User{} = target, %User{} = user) do UserRelationship.delete_reblog_mute(user, target) end + @spec get_user(String.t(), boolean()) :: User.t() | nil def get_user(ap_id, fake_record_fallback \\ true) do cond do user = User.get_cached_by_ap_id(ap_id) -> @@ -673,4 +713,14 @@ defmodule Pleroma.Web.CommonAPI do nil end end + + defp maybe_cancel_jobs(%Activity{data: %{"id" => ap_id}}) do + Oban.Job + |> where([j], j.worker == "Pleroma.Workers.PublisherWorker") + |> where([j], j.args["op"] == "publish_one") + |> where([j], j.args["params"]["id"] == ^ap_id) + |> Oban.cancel_all_jobs() + end + + defp maybe_cancel_jobs(_), do: {:ok, 0} end diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 4b7d28f5c..4220757df 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -134,8 +134,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft - defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do - %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)} + defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do + add_error(draft, dgettext("errors", "Cannot reply to a deleted status")) + end + + defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do + activity = Activity.get_by_id(id) + + params = + if is_nil(activity) do + # Deleted activities are returned as nil + Map.put(params, :in_reply_to_status_id, :deleted) + else + Map.put(params, :in_reply_to_status_id, activity) + end + + in_reply_to(%{draft | params: params}) end defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 2e2104904..fef907ace 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) + plug(Pleroma.Web.Plugs.LoggerMetadataPath) + plug(Pleroma.Web.Plugs.SetLocalePlug) plug(CORSPlug) plug(Pleroma.Web.Plugs.HTTPSecurityPlug) diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 1f2c3835a..3d3101d61 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -35,16 +35,18 @@ defmodule Pleroma.Web.Federator do end # Client API - def incoming_ap_doc(%{params: params, req_headers: req_headers}) do + def incoming_ap_doc(%{params: _params, req_headers: _req_headers} = args) do + job_args = Enum.into(args, %{}, fn {k, v} -> {Atom.to_string(k), v} end) + ReceiverWorker.enqueue( "incoming_ap_doc", - %{"req_headers" => req_headers, "params" => params, "timeout" => :timer.seconds(20)}, + Map.put(job_args, "timeout", :timer.seconds(20)), priority: 2 ) end def incoming_ap_doc(%{"type" => "Delete"} = params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow) end def incoming_ap_doc(params) do diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 9226a2deb..80ab95a57 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do %{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock] ) - plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships) + plug( + OAuthScopesPlug, + %{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers] + ) plug( OAuthScopesPlug, @@ -457,7 +460,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do end def unfollow(%{assigns: %{user: follower, account: followed}} = conn, _params) do - with {:ok, follower} <- CommonAPI.unfollow(follower, followed) do + with {:ok, follower} <- CommonAPI.unfollow(followed, follower) do render(conn, "relationship.json", user: follower, target: followed) end end @@ -492,7 +495,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do @doc "POST /api/v1/accounts/:id/block" def block(%{assigns: %{user: blocker, account: blocked}} = conn, _params) do - with {:ok, _activity} <- CommonAPI.block(blocker, blocked) do + with {:ok, _activity} <- CommonAPI.block(blocked, blocker) do render(conn, "relationship.json", user: blocker, target: blocked) else {:error, message} -> json_response(conn, :forbidden, %{error: message}) @@ -501,7 +504,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do @doc "POST /api/v1/accounts/:id/unblock" def unblock(%{assigns: %{user: blocker, account: blocked}} = conn, _params) do - with {:ok, _activity} <- CommonAPI.unblock(blocker, blocked) do + with {:ok, _activity} <- CommonAPI.unblock(blocked, blocker) do render(conn, "relationship.json", user: blocker, target: blocked) else {:error, message} -> json_response(conn, :forbidden, %{error: message}) @@ -629,6 +632,35 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do ) end + @doc "GET /api/v1/accounts/familiar_followers" + def familiar_followers( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, + _id + ) do + users = + User.get_all_by_ids(List.wrap(id)) + |> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)}) + + conn + |> render("familiar_followers.json", + for: user, + users: users, + as: :user + ) + end + + defp get_familiar_followers(%{id: id} = user, %{id: id}) do + User.get_familiar_followers(user, user) + end + + defp get_familiar_followers(%{hide_followers: true}, _current_user) do + [] + end + + defp get_familiar_followers(user, current_user) do + User.get_familiar_followers(user, current_user) + end + @doc "GET /api/v1/identity_proofs" def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params) end diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index e305aea94..afd83b785 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -34,6 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do pleroma:emoji_reaction poll update + status } # GET /api/v1/notifications diff --git a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex index b074ee405..a2af8148c 100644 --- a/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/poll_controller.ex @@ -51,7 +51,7 @@ defmodule Pleroma.Web.MastodonAPI.PollController do with %Object{data: %{"type" => "Question"}} = object <- Object.get_by_id(id), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), true <- Visibility.visible_for_user?(activity, user), - {:ok, _activities, object} <- get_cached_vote_or_vote(user, object, choices) do + {:ok, _activities, object} <- get_cached_vote_or_vote(object, user, choices) do try_render(conn, "show.json", %{object: object, for: user}) else nil -> render_error(conn, :not_found, "Record not found") @@ -60,11 +60,11 @@ defmodule Pleroma.Web.MastodonAPI.PollController do end end - defp get_cached_vote_or_vote(user, object, choices) do + defp get_cached_vote_or_vote(object, user, choices) do idempotency_key = "polls:#{user.id}:#{object.data["id"]}" @cachex.fetch!(:idempotency_cache, idempotency_key, fn _ -> - case CommonAPI.vote(user, object, choices) do + case CommonAPI.vote(object, user, choices) do {:error, _message} = res -> {:ignore, res} res -> {:commit, res} end diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 83e1bee54..b9b236920 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -276,7 +276,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do actor <- Activity.user_actor(activity), {_, true} <- {:own_status, actor.id == user.id}, changes <- body_params |> put_application(conn), - {_, {:ok, _update_activity}} <- {:pipeline, CommonAPI.update(user, activity, changes)}, + {_, {:ok, _update_activity}} <- {:pipeline, CommonAPI.update(activity, user, changes)}, {_, %Activity{}} = {_, activity} <- {:refetched, Activity.get_by_id_with_object(id)} do try_render(conn, "show.json", activity: activity, @@ -357,7 +357,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do conn, _ ) do - with {:ok, _fav} <- CommonAPI.favorite(user, activity_id), + with {:ok, _fav} <- CommonAPI.favorite(activity_id, user), %Activity{} = activity <- Activity.get_by_id(activity_id) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) end @@ -453,7 +453,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do _ ) do with %Activity{} = activity <- Activity.get_by_id(id), - {:ok, activity} <- CommonAPI.add_mute(user, activity, params) do + {:ok, activity} <- CommonAPI.add_mute(activity, user, params) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) end end @@ -467,7 +467,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do _ ) do with %Activity{} = activity <- Activity.get_by_id(id), - {:ok, activity} <- CommonAPI.remove_mute(user, activity) do + {:ok, activity} <- CommonAPI.remove_mute(activity, user) do try_render(conn, "show.json", activity: activity, for: user, as: :activity) end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 467dc2fac..6dcbfb097 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do def follow(follower, followed, params \\ %{}) do result = if not User.following?(follower, followed) do - CommonAPI.follow(follower, followed) + CommonAPI.follow(followed, follower) else {:ok, follower, followed, nil} end @@ -30,11 +30,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end defp set_reblogs_visibility(false, {:ok, follower, followed, _}) do - CommonAPI.hide_reblogs(follower, followed) + CommonAPI.hide_reblogs(followed, follower) end defp set_reblogs_visibility(_, {:ok, follower, followed, _}) do - CommonAPI.show_reblogs(follower, followed) + CommonAPI.show_reblogs(followed, follower) end defp set_subscription(true, {:ok, follower, followed, _}) do diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 267c3e3ed..6976ca6e5 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -193,6 +193,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render_many(targets, AccountView, "relationship.json", render_opts) end + def render("familiar_followers.json", %{users: users} = opts) do + opts = + opts + |> Map.merge(%{as: :user}) + |> Map.delete(:users) + + users + |> render_many(AccountView, "familiar_followers.json", opts) + end + + def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do + accounts = + accounts + |> render_many(AccountView, "show.json", opts) + |> Enum.filter(&Enum.any?/1) + + %{id: id, accounts: accounts} + end + defp do_render("show.json", %{user: user} = opts) do self = opts[:for] == user diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 99fc6d0c3..913684928 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -152,6 +152,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def federation do quarantined = Config.get([:instance, :quarantined_instances], []) + rejected = Config.get([:instance, :rejected_instances], []) if Config.get([:mrf, :transparency]) do {:ok, data} = MRF.describe() @@ -171,6 +172,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) |> Map.new() }) + |> Map.put( + :rejected_instances, + rejected + |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) + |> Map.new() + ) else %{} end diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 2a51f3755..3f2478719 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -108,6 +108,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do "mention" -> put_status(response, activity, reading_user, status_render_opts) + "status" -> + put_status(response, activity, reading_user, status_render_opts) + "favourite" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 5c5df79f2..a739b5d1a 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -30,7 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # pagination is restricted to 40 activities at a time defp fetch_rich_media_for_activities(activities) do Enum.each(activities, fn activity -> - spawn(fn -> Card.get_by_activity(activity) end) + Card.get_by_activity(activity) end) end @@ -293,7 +293,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do cond do is_nil(opts[:for]) -> false is_boolean(activity.thread_muted?) -> activity.thread_muted? - true -> CommonAPI.thread_muted?(opts[:for], activity) + true -> CommonAPI.thread_muted?(activity, opts[:for]) end attachment_data = object.data["attachment"] || [] @@ -624,6 +624,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do to_string(attachment["id"] || hash_id) end + description = + if attachment["summary"] do + HTML.strip_tags(attachment["summary"]) + else + attachment["name"] + end + + name = if attachment["summary"], do: attachment["name"] + + pleroma = + %{mime_type: media_type} + |> Maps.put_if_present(:name, name) + %{ id: attachment_id, url: href, @@ -631,8 +644,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do preview_url: href_preview, text_url: href, type: type, - description: attachment["name"], - pleroma: %{mime_type: media_type}, + description: description, + pleroma: pleroma, blurhash: attachment["blurhash"] } |> Maps.put_if_present(:meta, meta) diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index bb27d806d..730295a4c 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -104,7 +104,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do end def handle_info(:close, state) do - {:stop, {:closed, 'connection closed by server'}, state} + {:stop, {:closed, ~c"connection closed by server"}, state} end def handle_info(msg, state) do diff --git a/lib/pleroma/web/media_proxy.ex b/lib/pleroma/web/media_proxy.ex index d64760fc2..29882542c 100644 --- a/lib/pleroma/web/media_proxy.ex +++ b/lib/pleroma/web/media_proxy.ex @@ -75,8 +75,7 @@ defmodule Pleroma.Web.MediaProxy do %{host: domain} = URI.parse(url) mediaproxy_whitelist_domains = - [:media_proxy, :whitelist] - |> Config.get() + Config.get([:media_proxy, :whitelist], []) |> Kernel.++(["#{Upload.base_url()}"]) |> Enum.map(&maybe_get_domain_from_url/1) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index c11484ecb..0b446e0a6 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -54,9 +54,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do defp handle_preview(conn, url) do media_proxy_url = MediaProxy.url(url) + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) with {:ok, %{status: status} = head_response} when status in 200..299 <- - Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do + Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do content_type = Tesla.get_header(head_response, "content-type") content_length = Tesla.get_header(head_response, "content-length") content_length = content_length && String.to_integer(content_length) diff --git a/lib/pleroma/web/metadata/utils.ex b/lib/pleroma/web/metadata/utils.ex index 80a8be9a2..8f61ace24 100644 --- a/lib/pleroma/web/metadata/utils.ex +++ b/lib/pleroma/web/metadata/utils.ex @@ -25,11 +25,14 @@ defmodule Pleroma.Web.Metadata.Utils do |> scrub_html_and_truncate_object_field(object) end - def scrub_html_and_truncate(%{data: %{"content" => content}} = object) do + def scrub_html_and_truncate(%{data: %{"content" => content}} = object) + when is_binary(content) and content != "" do content |> scrub_html_and_truncate_object_field(object) end + def scrub_html_and_truncate(%{}), do: "" + def scrub_html_and_truncate(content, max_length \\ 200, omission \\ "...") when is_binary(content) do content diff --git a/lib/pleroma/web/o_auth/app.ex b/lib/pleroma/web/o_auth/app.ex index 0aa655381..d1bf6dd18 100644 --- a/lib/pleroma/web/o_auth/app.ex +++ b/lib/pleroma/web/o_auth/app.ex @@ -62,7 +62,7 @@ defmodule Pleroma.Web.OAuth.App do |> Repo.insert() end - @spec update(pos_integer(), map()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} + @spec update(pos_integer(), map()) :: {:ok, t()} | {:error, Ecto.Changeset.t()} | nil def update(id, params) do with %__MODULE__{} = app <- Repo.get(__MODULE__, id) do app diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index a5ad2e909..9b1198b42 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -96,7 +96,7 @@ defmodule Pleroma.Web.OAuth.Token do |> validate_required([:valid_until]) end - @spec create(App.t(), User.t(), map()) :: {:ok, Token} | {:error, Ecto.Changeset.t()} + @spec create(App.t(), User.t(), map()) :: {:ok, Token.t()} | {:error, Ecto.Changeset.t()} def create(%App{} = app, %User{} = user, attrs \\ %{}) do with {:ok, token} <- do_create(app, user, attrs) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do diff --git a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex index f860eaf7e..435ccfabe 100644 --- a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex @@ -23,8 +23,9 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do } = conn, _ ) do - with {:ok, notification} <- Notification.read_one(user, notification_id) do - render(conn, "show.json", notification: notification, for: user) + with {:ok, _} <- Notification.read_one(user, notification_id) do + conn + |> json("ok") else {:error, message} -> conn @@ -38,11 +39,14 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do conn, _ ) do - notifications = - user - |> Notification.set_read_up_to(max_id) - |> Enum.take(80) - - render(conn, "index.json", notifications: notifications, for: user) + with {:ok, _} <- Notification.set_read_up_to(user, max_id) do + conn + |> json("ok") + else + {:error, message} -> + conn + |> put_status(:bad_request) + |> json(%{"error" => message}) + end end end diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index a27dcd0ab..38f6c511e 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -3,26 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do - alias Pleroma.Config import Plug.Conn require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + def init(opts), do: opts def call(conn, _options) do - if Config.get([:http_security, :enabled]) do + if @config_impl.get([:http_security, :enabled]) do conn |> merge_resp_headers(headers()) - |> maybe_send_sts_header(Config.get([:http_security, :sts])) + |> maybe_send_sts_header(@config_impl.get([:http_security, :sts])) else conn end end def primary_frontend do - with %{"name" => frontend} <- Config.get([:frontends, :primary]), - available <- Config.get([:frontends, :available]), + with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]), + available <- @config_impl.get([:frontends, :available]), %{} = primary_frontend <- Map.get(available, frontend) do {:ok, primary_frontend} end @@ -37,8 +38,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end def headers do - referrer_policy = Config.get([:http_security, :referrer_policy]) - report_uri = Config.get([:http_security, :report_uri]) + referrer_policy = @config_impl.get([:http_security, :referrer_policy]) + report_uri = @config_impl.get([:http_security, :report_uri]) custom_http_frontend_headers = custom_http_frontend_headers() headers = [ @@ -86,10 +87,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do @csp_start [Enum.join(static_csp_rules, ";") <> ";"] defp csp_string do - scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] + scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() - report_uri = Config.get([:http_security, :report_uri]) + report_uri = @config_impl.get([:http_security, :report_uri]) img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" @@ -97,8 +98,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = - if Config.get([:media_proxy, :enabled]) && - !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do + if @config_impl.get([:media_proxy, :enabled]) && + !@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do sources = build_csp_multimedia_source_list() { @@ -115,17 +116,21 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end connect_src = - if Config.get(:env) == :dev do + if @config_impl.get([:env]) == :dev do [connect_src, " http://localhost:3035/"] else connect_src end script_src = - if Config.get(:env) == :dev do - "script-src 'self' 'unsafe-eval'" + if @config_impl.get([:http_security, :allow_unsafe_eval]) do + if @config_impl.get([:env]) == :dev do + "script-src 'self' 'unsafe-eval'" + else + "script-src 'self' 'wasm-unsafe-eval'" + end else - "script-src 'self' 'wasm-unsafe-eval'" + "script-src 'self'" end report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"] @@ -161,11 +166,11 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do defp build_csp_multimedia_source_list do media_proxy_whitelist = [:media_proxy, :whitelist] - |> Config.get() + |> @config_impl.get() |> build_csp_from_whitelist([]) - captcha_method = Config.get([Pleroma.Captcha, :method]) - captcha_endpoint = Config.get([captcha_method, :endpoint]) + captcha_method = @config_impl.get([Pleroma.Captcha, :method]) + captcha_endpoint = @config_impl.get([captcha_method, :endpoint]) base_endpoints = [ @@ -173,7 +178,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do [Pleroma.Upload, :base_url], [Pleroma.Uploaders.S3, :public_endpoint] ] - |> Enum.map(&Config.get/1) + |> Enum.map(&@config_impl.get/1) [captcha_endpoint | base_endpoints] |> Enum.map(&build_csp_param/1) @@ -200,7 +205,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end def warn_if_disabled do - unless Config.get([:http_security, :enabled]) do + unless Pleroma.Config.get([:http_security, :enabled]) do Logger.warning(" .i;;;;i. iYcviii;vXY: @@ -245,8 +250,8 @@ your instance and your users via malicious posts: end defp maybe_send_sts_header(conn, true) do - max_age_sts = Config.get([:http_security, :sts_max_age]) - max_age_ct = Config.get([:http_security, :ct_max_age]) + max_age_sts = @config_impl.get([:http_security, :sts_max_age]) + max_age_ct = @config_impl.get([:http_security, :ct_max_age]) merge_resp_headers(conn, [ {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"}, diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index e814efc2c..67974599a 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -3,10 +3,18 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do + alias Pleroma.Helpers.InetHelper + import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] + + alias Pleroma.Signature + alias Pleroma.Web.ActivityPub.MRF + require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + def init(options) do options end @@ -19,54 +27,14 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do if get_format(conn) in ["json", "activity+json"] do conn |> maybe_assign_valid_signature() + |> maybe_assign_actor_id() |> maybe_require_signature() + |> maybe_filter_requests() else conn end end - defp validate_signature(conn, request_target) do - # Newer drafts for HTTP signatures now use @request-target instead of the - # old (request-target). We'll now support both for incoming signatures. - conn = - conn - |> put_req_header("(request-target)", request_target) - |> put_req_header("@request-target", request_target) - - HTTPSignatures.validate_conn(conn) - end - - defp validate_signature(conn) do - # This (request-target) is non-standard, but many implementations do it - # this way due to a misinterpretation of - # https://datatracker.ietf.org/doc/html/draft-cavage-http-signatures-06 - # "path" was interpreted as not having the query, though later examples - # show that it must be the absolute path + query. This behavior is kept to - # make sure most software (Pleroma itself, Mastodon, and probably others) - # do not break. - request_target = String.downcase("#{conn.method}") <> " #{conn.request_path}" - - # This is the proper way to build the @request-target, as expected by - # many HTTP signature libraries, clarified in the following draft: - # https://www.ietf.org/archive/id/draft-ietf-httpbis-message-signatures-11.html#section-2.2.6 - # It is the same as before, but containing the query part as well. - proper_target = request_target <> "?#{conn.query_string}" - - cond do - # Normal, non-standard behavior but expected by Pleroma and more. - validate_signature(conn, request_target) -> - true - - # Has query string and the previous one failed: let's try the standard. - conn.query_string != "" -> - validate_signature(conn, proper_target) - - # If there's no query string and signature fails, it's rotten. - true -> - false - end - end - defp maybe_assign_valid_signature(conn) do if has_signature_header?(conn) do # we replace the digest header with the one we computed in DigestPlug @@ -76,27 +44,70 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do conn -> conn end - assign(conn, :valid_signature, validate_signature(conn)) + assign(conn, :valid_signature, Signature.validate_signature(conn)) else Logger.debug("No signature header!") conn end end + defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do + adapter = Application.get_env(:http_signatures, :adapter) + + {:ok, actor_id} = adapter.get_actor_id(conn) + + assign(conn, :actor_id, actor_id) + end + + defp maybe_assign_actor_id(conn), do: conn + defp has_signature_header?(conn) do conn |> get_req_header("signature") |> Enum.at(0, false) end defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn - defp maybe_require_signature(conn) do - if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do + exceptions = + @config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], []) + |> Enum.map(&InetHelper.parse_cidr/1) + + if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do + conn + else + conn + |> put_status(:unauthorized) + |> text("Request not signed") + |> halt() + end + else conn - |> put_status(:unauthorized) - |> text("Request not signed") - |> halt() + end + end + + defp maybe_filter_requests(%{halted: true} = conn), do: conn + + defp maybe_filter_requests(conn) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and + conn.assigns[:actor_id] do + %{host: host} = URI.parse(conn.assigns.actor_id) + + if MRF.subdomain_match?(rejected_domains(), host) do + conn + |> put_status(:unauthorized) + |> halt() + else + conn + end else conn end end + + defp rejected_domains do + @config_impl.get([:instance, :rejected_instances]) + |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + end end diff --git a/lib/pleroma/web/plugs/logger_metadata_path.ex b/lib/pleroma/web/plugs/logger_metadata_path.ex new file mode 100644 index 000000000..a5553cfc8 --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_path.ex @@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataPath do + def init(opts), do: opts + + def call(conn, _) do + Logger.metadata(path: conn.request_path) + conn + end +end diff --git a/lib/pleroma/web/plugs/logger_metadata_user.ex b/lib/pleroma/web/plugs/logger_metadata_user.ex new file mode 100644 index 000000000..6a5c0041d --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_user.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataUser do + alias Pleroma.User + + def init(opts), do: opts + + def call(%{assigns: %{user: user = %User{}}} = conn, _) do + Logger.metadata(user: user.nickname) + conn + end + + def call(conn, _) do + conn + end +end diff --git a/lib/pleroma/web/plugs/o_auth_scopes_plug.ex b/lib/pleroma/web/plugs/o_auth_scopes_plug.ex index faf0fd8c6..08c2f61ea 100644 --- a/lib/pleroma/web/plugs/o_auth_scopes_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_scopes_plug.ex @@ -34,7 +34,9 @@ defmodule Pleroma.Web.Plugs.OAuthScopesPlug do permissions = Enum.join(missing_scopes, " #{op} ") error_message = - dgettext("errors", "Insufficient permissions: %{permissions}.", permissions: permissions) + dgettext("errors", "Insufficient permissions: %{permissions}.", + permissions: permissions + ) conn |> put_resp_content_type("application/json") diff --git a/lib/pleroma/web/plugs/remote_ip.ex b/lib/pleroma/web/plugs/remote_ip.ex index 9f733a96f..3a4bffb50 100644 --- a/lib/pleroma/web/plugs/remote_ip.ex +++ b/lib/pleroma/web/plugs/remote_ip.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do """ alias Pleroma.Config + alias Pleroma.Helpers.InetHelper import Plug.Conn @behaviour Plug @@ -30,19 +31,8 @@ defmodule Pleroma.Web.Plugs.RemoteIp do proxies = Config.get([__MODULE__, :proxies], []) |> Enum.concat(reserved) - |> Enum.map(&maybe_add_cidr/1) + |> Enum.map(&InetHelper.parse_cidr/1) {headers, proxies} end - - defp maybe_add_cidr(proxy) when is_binary(proxy) do - proxy = - cond do - "/" in String.codepoints(proxy) -> proxy - InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" - InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" - end - - InetCidr.parse_cidr!(proxy, true) - end end diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex index 0d43f402e..d4693f63e 100644 --- a/lib/pleroma/web/push.ex +++ b/lib/pleroma/web/push.ex @@ -20,17 +20,13 @@ defmodule Pleroma.Web.Push do end def vapid_config do - Application.get_env(:web_push_encryption, :vapid_details, []) + Application.get_env(:web_push_encryption, :vapid_details, nil) end - def enabled do - case vapid_config() do - [] -> false - list when is_list(list) -> true - _ -> false - end - end + def enabled, do: match?([subject: _, public_key: _, private_key: _], vapid_config()) + @spec send(Pleroma.Notification.t()) :: + {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset() | term()} def send(notification) do WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id}) end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 36f44d8e8..d71e134cb 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -16,62 +16,70 @@ defmodule Pleroma.Web.Push.Impl do require Logger import Ecto.Query + @body_chars 140 @types ["Create", "Follow", "Announce", "Like", "Move", "EmojiReact", "Update"] - @doc "Performs sending notifications for user subscriptions" - @spec perform(Notification.t()) :: list(any) | :error | {:error, :unknown_type} - def perform( + @doc "Builds webpush notification payloads for the subscriptions enabled by the receiving user" + @spec build(Notification.t()) :: + list(%{content: map(), subscription: Subscription.t()}) | [] + def build( %{ activity: %{data: %{"type" => activity_type}} = activity, - user: %User{id: user_id} + user_id: user_id } = notification ) when activity_type in @types do - actor = User.get_cached_by_ap_id(notification.activity.data["actor"]) + notification_actor = User.get_cached_by_ap_id(notification.activity.data["actor"]) + avatar_url = User.avatar_url(notification_actor) - mastodon_type = notification.type - gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) - avatar_url = User.avatar_url(actor) object = Object.normalize(activity, fetch: false) user = User.get_cached_by_id(user_id) direct_conversation_id = Activity.direct_conversation_id(activity, user) - for subscription <- fetch_subscriptions(user_id), - Subscription.enabled?(subscription, mastodon_type) do - %{ - access_token: subscription.token.token, - notification_id: notification.id, - notification_type: mastodon_type, - icon: avatar_url, - preferred_locale: "en", - pleroma: %{ - activity_id: notification.activity.id, - direct_conversation_id: direct_conversation_id + subscriptions = fetch_subscriptions(user_id) + + subscriptions + |> Enum.filter(&Subscription.enabled?(&1, notification.type)) + |> Enum.map(fn subscription -> + payload = + %{ + access_token: subscription.token.token, + notification_id: notification.id, + notification_type: notification.type, + icon: avatar_url, + preferred_locale: "en", + pleroma: %{ + activity_id: notification.activity.id, + direct_conversation_id: direct_conversation_id + } } - } - |> Map.merge(build_content(notification, actor, object, mastodon_type)) - |> Jason.encode!() - |> push_message(build_sub(subscription), gcm_api_key, subscription) - end - |> (&{:ok, &1}).() + |> Map.merge(build_content(notification, notification_actor, object)) + |> Jason.encode!() + + %{payload: payload, subscription: subscription} + end) end - def perform(_) do - Logger.warning("Unknown notification type") - {:error, :unknown_type} + def build(notif) do + Logger.warning("WebPush: unknown activity type: #{inspect(notif)}") + [] end - @doc "Push message to web" - def push_message(body, sub, api_key, subscription) do - case WebPushEncryption.send_web_push(body, sub, api_key) do + @doc "Deliver push notification to the provided webpush subscription" + @spec deliver(%{payload: String.t(), subscription: Subscription.t()}) :: :ok | :error + def deliver(%{payload: payload, subscription: subscription}) do + gcm_api_key = Application.get_env(:web_push_encryption, :gcm_api_key) + formatted_subscription = build_sub(subscription) + + case WebPushEncryption.send_web_push(payload, formatted_subscription, gcm_api_key) do + {:ok, %{status: code}} when code in 200..299 -> + :ok + {:ok, %{status: code}} when code in 400..499 -> Logger.debug("Removing subscription record") Repo.delete!(subscription) :ok - {:ok, %{status: code}} when code in 200..299 -> - :ok - {:ok, %{status: code}} -> Logger.error("Web Push Notification failed with code: #{code}") :error @@ -100,106 +108,106 @@ defmodule Pleroma.Web.Push.Impl do } end - def build_content(notification, actor, object, mastodon_type \\ nil) - def build_content( %{ user: %{notification_settings: %{hide_notification_contents: true}} } = notification, - _actor, - _object, - mastodon_type + _user, + _object ) do - %{body: format_title(notification, mastodon_type)} + %{body: format_title(notification)} end - def build_content(notification, actor, object, mastodon_type) do - mastodon_type = mastodon_type || notification.type - + def build_content(notification, user, object) do %{ - title: format_title(notification, mastodon_type), - body: format_body(notification, actor, object, mastodon_type) + title: format_title(notification), + body: format_body(notification, user, object) } end - def format_body(activity, actor, object, mastodon_type \\ nil) - - def format_body(_activity, actor, %{data: %{"type" => "ChatMessage"} = data}, _) do - case data["content"] do - nil -> "@#{actor.nickname}: (Attachment)" - content -> "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}" + @spec format_body(Notification.t(), User.t(), Object.t()) :: String.t() + def format_body(_notification, user, %{data: %{"type" => "ChatMessage"} = object}) do + case object["content"] do + nil -> "@#{user.nickname}: (Attachment)" + content -> "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, @body_chars)}" end end def format_body( + %{type: "poll"} = _notification, + _user, + %{data: %{"content" => content} = data} = _object + ) do + options = Map.get(data, "anyOf") || Map.get(data, "oneOf") + + content_text = content <> "\n" + + options_text = Enum.map_join(options, "\n", fn x -> "○ #{x["name"]}" end) + + [content_text, options_text] + |> Enum.join("\n") + |> Utils.scrub_html_and_truncate(@body_chars) + end + + def format_body( %{activity: %{data: %{"type" => "Create"}}}, - actor, - %{data: %{"content" => content}}, - _mastodon_type + user, + %{data: %{"content" => content}} ) do - "@#{actor.nickname}: #{Utils.scrub_html_and_truncate(content, 80)}" + "@#{user.nickname}: #{Utils.scrub_html_and_truncate(content, @body_chars)}" end def format_body( %{activity: %{data: %{"type" => "Announce"}}}, - actor, - %{data: %{"content" => content}}, - _mastodon_type + user, + %{data: %{"content" => content}} ) do - "@#{actor.nickname} repeated: #{Utils.scrub_html_and_truncate(content, 80)}" + "@#{user.nickname} repeated: #{Utils.scrub_html_and_truncate(content, @body_chars)}" end def format_body( %{activity: %{data: %{"type" => "EmojiReact", "content" => content}}}, - actor, - _object, - _mastodon_type + user, + _object ) do - "@#{actor.nickname} reacted with #{content}" + "@#{user.nickname} reacted with #{content}" end def format_body( %{activity: %{data: %{"type" => type}}} = notification, - actor, - _object, - mastodon_type + user, + _object ) when type in ["Follow", "Like"] do - mastodon_type = mastodon_type || notification.type - - case mastodon_type do - "follow" -> "@#{actor.nickname} has followed you" - "follow_request" -> "@#{actor.nickname} has requested to follow you" - "favourite" -> "@#{actor.nickname} has favorited your post" + case notification.type do + "follow" -> "@#{user.nickname} has followed you" + "follow_request" -> "@#{user.nickname} has requested to follow you" + "favourite" -> "@#{user.nickname} has favorited your post" end end def format_body( %{activity: %{data: %{"type" => "Update"}}}, - actor, - _object, - _mastodon_type + user, + _object ) do - "@#{actor.nickname} edited a status" + "@#{user.nickname} edited a status" end - def format_title(activity, mastodon_type \\ nil) - - def format_title(%{activity: %{data: %{"directMessage" => true}}}, _mastodon_type) do + @spec format_title(Notification.t()) :: String.t() + def format_title(%{activity: %{data: %{"directMessage" => true}}}) do "New Direct Message" end - def format_title(%{type: type}, mastodon_type) do - case mastodon_type || type do - "mention" -> "New Mention" - "follow" -> "New Follower" - "follow_request" -> "New Follow Request" - "reblog" -> "New Repeat" - "favourite" -> "New Favorite" - "update" -> "New Update" - "pleroma:chat_mention" -> "New Chat Message" - "pleroma:emoji_reaction" -> "New Reaction" - type -> "New #{String.capitalize(type || "event")}" - end - end + def format_title(%{type: "mention"}), do: "New Mention" + def format_title(%{type: "status"}), do: "New Status" + def format_title(%{type: "follow"}), do: "New Follower" + def format_title(%{type: "follow_request"}), do: "New Follow Request" + def format_title(%{type: "reblog"}), do: "New Repeat" + def format_title(%{type: "favourite"}), do: "New Favorite" + def format_title(%{type: "update"}), do: "New Update" + def format_title(%{type: "pleroma:chat_mention"}), do: "New Chat Message" + def format_title(%{type: "pleroma:emoji_reaction"}), do: "New Reaction" + def format_title(%{type: "poll"}), do: "Poll Results" + def format_title(%{type: type}), do: "New #{String.capitalize(type || "event")}" end diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index 4ec50e132..1cd90629f 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -4,75 +4,50 @@ defmodule Pleroma.Web.RichMedia.Backfill do alias Pleroma.Web.RichMedia.Card + alias Pleroma.Web.RichMedia.Helpers alias Pleroma.Web.RichMedia.Parser alias Pleroma.Web.RichMedia.Parser.TTL - alias Pleroma.Workers.RichMediaExpirationWorker + alias Pleroma.Workers.RichMediaWorker require Logger - @backfiller Pleroma.Config.get([__MODULE__, :provider], Pleroma.Web.RichMedia.Backfill.Task) @cachex Pleroma.Config.get([:cachex, :provider], Cachex) - @max_attempts 3 - @retry 5_000 + @stream_out_impl Pleroma.Config.get( + [__MODULE__, :stream_out], + Pleroma.Web.ActivityPub.ActivityPub + ) - def start(%{url: url} = args) when is_binary(url) do + @spec run(map()) :: :ok | Parser.parse_errors() | Helpers.get_errors() + def run(%{"url" => url} = args) do url_hash = Card.url_to_hash(url) - args = - args - |> Map.put(:attempt, 1) - |> Map.put(:url_hash, url_hash) - - @backfiller.run(args) - end - - def run(%{url: url, url_hash: url_hash, attempt: attempt} = args) - when attempt <= @max_attempts do case Parser.parse(url) do {:ok, fields} -> {:ok, card} = Card.create(url, fields) maybe_schedule_expiration(url, fields) - if Map.has_key?(args, :activity_id) do + with %{"activity_id" => activity_id} <- args, + false <- is_nil(activity_id) do stream_update(args) end warm_cache(url_hash, card) + :ok - {:error, {:invalid_metadata, fields}} -> - Logger.debug("Rich media incomplete or invalid metadata for #{url}: #{inspect(fields)}") - negative_cache(url_hash) - - {:error, :body_too_large} -> - Logger.error("Rich media error for #{url}: :body_too_large") - negative_cache(url_hash) - - {:error, {:content_type, type}} -> - Logger.debug("Rich media error for #{url}: :content_type is #{type}") + {:error, type} = error + when type in [:invalid_metadata, :body_too_large, :content_type, :validate, :get, :head] -> negative_cache(url_hash) - - e -> - Logger.debug("Rich media error for #{url}: #{inspect(e)}") - - :timer.sleep(@retry * attempt) - - run(%{args | attempt: attempt + 1}) + error end end - def run(%{url: url, url_hash: url_hash}) do - Logger.debug("Rich media failure for #{url}") - - negative_cache(url_hash, :timer.minutes(15)) - end - defp maybe_schedule_expiration(url, fields) do case TTL.process(fields, url) do {:ok, ttl} when is_number(ttl) -> timestamp = DateTime.from_unix!(ttl) - RichMediaExpirationWorker.new(%{"url" => url}, scheduled_at: timestamp) + RichMediaWorker.new(%{"op" => "expire", "url" => url}, scheduled_at: timestamp) |> Oban.insert() _ -> @@ -80,22 +55,14 @@ defmodule Pleroma.Web.RichMedia.Backfill do end end - defp stream_update(%{activity_id: activity_id}) do + defp stream_update(%{"activity_id" => activity_id}) do Pleroma.Activity.get_by_id(activity_id) |> Pleroma.Activity.normalize() - |> Pleroma.Web.ActivityPub.ActivityPub.stream_out() + |> @stream_out_impl.stream_out() end defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val) - defp negative_cache(key, ttl \\ nil), do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl) -end -defmodule Pleroma.Web.RichMedia.Backfill.Task do - alias Pleroma.Web.RichMedia.Backfill - - def run(args) do - Task.Supervisor.start_child(Pleroma.TaskSupervisor, Backfill, :run, [args], - name: {:global, {:rich_media, args.url_hash}} - ) - end + defp negative_cache(key, ttl \\ :timer.minutes(15)), + do: @cachex.put(:rich_media_cache, key, :error, ttl: ttl) end diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex index 36a1ae44a..abad4957e 100644 --- a/lib/pleroma/web/rich_media/card.ex +++ b/lib/pleroma/web/rich_media/card.ex @@ -7,8 +7,8 @@ defmodule Pleroma.Web.RichMedia.Card do alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Repo - alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Parser + alias Pleroma.Workers.RichMediaWorker @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) @@ -75,21 +75,26 @@ defmodule Pleroma.Web.RichMedia.Card do def get_by_url(nil), do: nil - @spec get_or_backfill_by_url(String.t(), map()) :: t() | nil - def get_or_backfill_by_url(url, backfill_opts \\ %{}) do - case get_by_url(url) do - %__MODULE__{} = card -> - card + @spec get_or_backfill_by_url(String.t(), keyword()) :: t() | nil + def get_or_backfill_by_url(url, opts \\ []) do + if @config_impl.get([:rich_media, :enabled]) do + case get_by_url(url) do + %__MODULE__{} = card -> + card - nil -> - backfill_opts = Map.put(backfill_opts, :url, url) + nil -> + activity_id = Keyword.get(opts, :activity_id, nil) - Backfill.start(backfill_opts) + RichMediaWorker.new(%{"op" => "backfill", "url" => url, "activity_id" => activity_id}) + |> Oban.insert() - nil + nil - :error -> - nil + :error -> + nil + end + else + nil end end @@ -104,7 +109,8 @@ defmodule Pleroma.Web.RichMedia.Card do @spec get_by_activity(Activity.t()) :: t() | nil | :error # Fake/Draft activity def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do - with %Object{} = object <- Object.normalize(activity, fetch: false), + with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])}, + %Object{} = object <- Object.normalize(activity, fetch: false), url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do case get_by_url(url) do # Cache hit @@ -132,7 +138,7 @@ defmodule Pleroma.Web.RichMedia.Card do nil else {:cached, url} -> - get_or_backfill_by_url(url, %{activity_id: activity.id}) + get_or_backfill_by_url(url, activity_id: activity.id) _ -> :error diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 119994458..e2889b351 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -5,26 +5,38 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Config + require Logger + + @type get_errors :: {:error, :body_too_large | :content_type | :head | :get} + + @spec rich_media_get(String.t()) :: {:ok, String.t()} | get_errors() def rich_media_get(url) do headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] - head_check = - case Pleroma.HTTP.head(url, headers, http_options()) do - # If the HEAD request didn't reach the server for whatever reason, - # we assume the GET that comes right after won't either - {:error, _} = e -> - e + with {_, {:ok, %Tesla.Env{status: 200, headers: headers}}} <- + {:head, Pleroma.HTTP.head(url, headers, http_options())}, + {_, :ok} <- {:content_type, check_content_type(headers)}, + {_, :ok} <- {:content_length, check_content_length(headers)}, + {_, {:ok, %Tesla.Env{status: 200, body: body}}} <- + {:get, Pleroma.HTTP.get(url, headers, http_options())} do + {:ok, body} + else + {:head, _} -> + Logger.debug("Rich media error for #{url}: HTTP HEAD failed") + {:error, :head} - {:ok, %Tesla.Env{status: 200, headers: headers}} -> - with :ok <- check_content_type(headers), - :ok <- check_content_length(headers), - do: :ok + {:content_type, {_, type}} -> + Logger.debug("Rich media error for #{url}: content-type is #{type}") + {:error, :content_type} - _ -> - :ok - end + {:content_length, {_, length}} -> + Logger.debug("Rich media error for #{url}: content-length is #{length}") + {:error, :body_too_large} - with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, http_options()) + {:get, _} -> + Logger.debug("Rich media error for #{url}: HTTP GET failed") + {:error, :get} + end end defp check_content_type(headers) do @@ -32,7 +44,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do {_, content_type} -> case Plug.Conn.Utils.media_type(content_type) do {:ok, "text", "html", _} -> :ok - _ -> {:error, {:content_type, content_type}} + _ -> {:error, content_type} end _ -> @@ -47,7 +59,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do {_, maybe_content_length} -> case Integer.parse(maybe_content_length) do {content_length, ""} when content_length <= max_body -> :ok - {_, ""} -> {:error, :body_too_large} + {_, ""} -> {:error, maybe_content_length} _ -> :ok end @@ -57,9 +69,12 @@ defmodule Pleroma.Web.RichMedia.Helpers do end defp http_options do + timeout = Config.get!([:rich_media, :timeout]) + [ - pool: :media, - max_body: Config.get([:rich_media, :max_body], 5_000_000) + pool: :rich_media, + max_body: Config.get([:rich_media, :max_body], 5_000_000), + tesla_middleware: [{Tesla.Middleware.Timeout, timeout: timeout}] ] end end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 37cf29029..a3a522d7a 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Parser do + alias Pleroma.Web.RichMedia.Helpers require Logger @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) @@ -11,20 +12,26 @@ defmodule Pleroma.Web.RichMedia.Parser do Pleroma.Config.get([:rich_media, :parsers]) end - def parse(nil), do: nil + @type parse_errors :: {:error, :rich_media_disabled | :validate} - @spec parse(String.t()) :: {:ok, map()} | {:error, any()} - def parse(url) do - with :ok <- validate_page_url(url), - {:ok, data} <- parse_url(url) do + @spec parse(String.t()) :: + {:ok, map()} | parse_errors() | Helpers.get_errors() + def parse(url) when is_binary(url) do + with {_, true} <- {:config, @config_impl.get([:rich_media, :enabled])}, + {_, :ok} <- {:validate, validate_page_url(url)}, + {_, {:ok, data}} <- {:parse, parse_url(url)} do data = Map.put(data, "url", url) {:ok, data} + else + {:config, _} -> {:error, :rich_media_disabled} + {:validate, _} -> {:error, :validate} + {:parse, error} -> error end end defp parse_url(url) do - with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url), - {:ok, html} <- Floki.parse_document(html) do + with {:ok, body} <- Helpers.rich_media_get(url), + {:ok, html} <- Floki.parse_document(body) do html |> maybe_parse() |> clean_parsed_data() @@ -46,8 +53,8 @@ defmodule Pleroma.Web.RichMedia.Parser do {:ok, data} end - defp check_parsed_data(data) do - {:error, {:invalid_metadata, data}} + defp check_parsed_data(_data) do + {:error, :invalid_metadata} end defp clean_parsed_data(data) do diff --git a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex index 948c727e1..1172a120a 100644 --- a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do %URI{host: host, query: query} = URI.parse(image) is_binary(host) and String.contains?(host, "amazonaws.com") and - String.contains?(query, "X-Amz-Expires") + is_binary(query) and String.contains?(query, "X-Amz-Expires") end defp aws_signed_url?(_), do: nil diff --git a/lib/pleroma/web/rich_media/parsers/o_embed.ex b/lib/pleroma/web/rich_media/parsers/o_embed.ex index 0f303176c..35ec14426 100644 --- a/lib/pleroma/web/rich_media/parsers/o_embed.ex +++ b/lib/pleroma/web/rich_media/parsers/o_embed.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.RichMedia.Parsers.OEmbed do end defp get_oembed_data(url) do - with {:ok, %Tesla.Env{body: json}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do + with {:ok, json} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url) do Jason.decode(json) end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index e35a89ce2..fc40a1143 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do pipeline :browser do plug(:accepts, ["html"]) plug(:fetch_session) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :oauth do @@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do plug(:fetch_session) plug(:authenticate) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :no_auth_or_privacy_expectations_api do plug(:base_api) plug(:after_auth) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported) @@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do pipeline :api do plug(:expect_public_instance_or_user_authentication) plug(:no_auth_or_privacy_expectations_api) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :authenticated_api do plug(:expect_user_authentication) plug(:no_auth_or_privacy_expectations_api) plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :admin_api do @@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Web.Plugs.UserIsStaffPlug) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :require_admin do @@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do plug(:browser) plug(:authenticate) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :well_known do @@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do pipeline :pleroma_api do plug(:accepts, ["html", "json"]) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :mailbox_preview do @@ -638,6 +646,7 @@ defmodule Pleroma.Web.Router do patch("/accounts/update_credentials", AccountController, :update_credentials) get("/accounts/relationships", AccountController, :relationships) + get("/accounts/familiar_followers", AccountController, :familiar_followers) get("/accounts/:id/lists", AccountController, :lists) get("/accounts/:id/identity_proofs", AccountController, :identity_proofs) get("/endorsements", AccountController, :endorsements) @@ -979,7 +988,7 @@ defmodule Pleroma.Web.Router do scope "/" do pipe_through([:pleroma_html, :authenticate, :require_admin]) - live_dashboard("/phoenix/live_dashboard") + live_dashboard("/phoenix/live_dashboard", additional_pages: [oban: Oban.LiveDashboard]) end # Test-only routes needed to test action dispatching and plug chain execution diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 7065fdab2..76dc0f42d 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -172,7 +172,13 @@ defmodule Pleroma.Web.Streamer do def stream(topics, items) do if should_env_send?() do for topic <- List.wrap(topics), item <- List.wrap(items) do - spawn(fn -> do_stream(topic, item) end) + fun = fn -> do_stream(topic, item) end + + if Config.get([__MODULE__, :sync_streaming], false) do + fun.() + else + spawn(fun) + end end end end @@ -200,7 +206,7 @@ defmodule Pleroma.Web.Streamer do false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, item_host), false <- Pleroma.Web.ActivityPub.MRF.subdomain_match?(domain_blocks, parent_host), true <- thread_containment(item, user), - false <- CommonAPI.thread_muted?(user, parent) do + false <- CommonAPI.thread_muted?(parent, user) do false else _ -> true diff --git a/lib/pleroma/web/templates/feed/feed/tag.atom.eex b/lib/pleroma/web/templates/feed/feed/tag.atom.eex index 14b0ee594..3449c97ff 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.atom.eex @@ -12,7 +12,7 @@ <subtitle><%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %></subtitle> <logo><%= feed_logo() %></logo> <updated><%= most_recent_update(@activities) %></updated> - <link rel="self" href="<%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.atom' %>" type="application/atom+xml"/> + <link rel="self" href="<%= "#{Routes.tag_feed_url(@conn, :feed, @tag)}.atom" %>" type="application/atom+xml"/> <%= for activity <- @activities do %> <%= render Phoenix.Controller.view_module(@conn), "_tag_activity.atom", Map.merge(assigns, prepare_activity(activity, actor: true)) %> diff --git a/lib/pleroma/web/templates/feed/feed/tag.rss.eex b/lib/pleroma/web/templates/feed/feed/tag.rss.eex index 27dde5627..a87f9bf50 100644 --- a/lib/pleroma/web/templates/feed/feed/tag.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/tag.rss.eex @@ -6,7 +6,7 @@ <title>#<%= @tag %></title> <description><%= Gettext.dpgettext("static_pages", "tag feed description", "These are public toots tagged with #%{tag}. You can interact with them if you have an account anywhere in the fediverse.", tag: @tag) %></description> - <link><%= '#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss' %></link> + <link><%= "#{Routes.tag_feed_url(@conn, :feed, @tag)}.rss" %></link> <webfeeds:logo><%= feed_logo() %></webfeeds:logo> <webfeeds:accentColor>2b90d9</webfeeds:accentColor> <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/feed/feed/user.atom.eex b/lib/pleroma/web/templates/feed/feed/user.atom.eex index e36bfc66c..c2c77cfed 100644 --- a/lib/pleroma/web/templates/feed/feed/user.atom.eex +++ b/lib/pleroma/web/templates/feed/feed/user.atom.eex @@ -11,12 +11,12 @@ <subtitle><%= escape(@user.bio) %></subtitle> <updated><%= most_recent_update(@activities, @user, :atom) %></updated> <logo><%= logo(@user) %></logo> - <link rel="self" href="<%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.atom' %>" type="application/atom+xml"/> + <link rel="self" href="<%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.atom" %>" type="application/atom+xml"/> <%= render Phoenix.Controller.view_module(@conn), "_author.atom", assigns %> <%= if last_activity(@activities) do %> - <link rel="next" href="<%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.atom?max_id=#{last_activity(@activities).id}' %>" type="application/atom+xml"/> + <link rel="next" href="<%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.atom?max_id=#{last_activity(@activities).id}" %>" type="application/atom+xml"/> <% end %> <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/templates/feed/feed/user.rss.eex b/lib/pleroma/web/templates/feed/feed/user.rss.eex index fae3fcf3d..b907a7e57 100644 --- a/lib/pleroma/web/templates/feed/feed/user.rss.eex +++ b/lib/pleroma/web/templates/feed/feed/user.rss.eex @@ -7,20 +7,20 @@ xmlns:poco="http://portablecontacts.net/spec/1.0"> <channel> <title><%= @user.nickname <> "'s timeline" %></title> - <link><%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss' %></link> + <link><%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %></link> <atom:link href="<%= Routes.user_feed_url(@conn, :feed, @user.nickname) <> ".atom" %>" rel="self" type="application/rss+xml" /> <description><%= escape(@user.bio) %></description> <image> <url><%= logo(@user) %></url> <title><%= @user.nickname <> "'s timeline" %></title> - <link><%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss' %></link> + <link><%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss" %></link> </image> <%= render Phoenix.Controller.view_module(@conn), "_author.rss", assigns %> <%= if last_activity(@activities) do %> - <link rel="next"><%= '#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}' %></link> + <link rel="next"><%= "#{Routes.user_feed_url(@conn, :feed, @user.nickname)}.rss?max_id=#{last_activity(@activities).id}" %></link> <% end %> <%= for activity <- @activities do %> diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index 1557d95ab..38ebc8c5d 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -73,7 +73,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do # def do_follow(%{assigns: %{user: %User{} = user}} = conn, %{"user" => %{"id" => id}}) do with {:fetch_user, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, - {:ok, _, _, _} <- CommonAPI.follow(user, followee) do + {:ok, _, _, _} <- CommonAPI.follow(followee, user) do redirect(conn, to: "/users/#{followee.id}") else error -> @@ -90,7 +90,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do with {_, %User{} = followee} <- {:fetch_user, User.get_cached_by_id(id)}, {_, {:ok, user}, _} <- {:auth, WrapperAuthenticator.get_user(conn), followee}, {_, _, _, false} <- {:mfa_required, followee, user, MFA.require?(user)}, - {:ok, _, _, _} <- CommonAPI.follow(user, followee) do + {:ok, _, _, _} <- CommonAPI.follow(followee, user) do redirect(conn, to: "/users/#{followee.id}") else error -> @@ -108,7 +108,7 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do {_, _, {:ok, %{user: user}}} <- {:mfa_token, followee, MFA.Token.validate(token)}, {_, _, _, {:ok, _}} <- {:verify_mfa_code, followee, token, TOTPAuthenticator.verify(code, user)}, - {:ok, _, _, _} <- CommonAPI.follow(user, followee) do + {:ok, _, _, _} <- CommonAPI.follow(followee, user) do redirect(conn, to: "/users/#{followee.id}") else error -> diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 040fa3286..6805233df 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -207,9 +207,6 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do {:error, changeset} -> {_, {error, _}} = Enum.at(changeset.errors, 0) json(conn, %{error: "New password #{error}."}) - - _ -> - json(conn, %{error: "Unable to change password."}) end {:error, msg} -> diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 26fb8af84..e653b3338 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do end end + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def find_lrdd_template(domain) do + @cachex.fetch!(:host_meta_cache, domain, fn _ -> + {:commit, fetch_lrdd_template(domain)} + end) + rescue + e -> {:error, "Cachex error: #{inspect(e)}"} + end + + defp fetch_lrdd_template(domain) do # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 meta_url = "https://#{domain}/.well-known/host-meta" @@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do end end - defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do + defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do case find_lrdd_template(domain) do {:ok, template} -> String.replace(template, "{uri}", encoded_account) @@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do end end + defp get_address_from_domain(domain, account) when is_binary(domain) do + encoded_account = URI.encode("acct:#{account}") + get_address_from_domain(domain, encoded_account) + end + defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain} @spec finger(String.t()) :: {:ok, map()} | {:error, any()} @@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do URI.parse(account).host end - encoded_account = URI.encode("acct:#{account}") - - with address when is_binary(address) <- get_address_from_domain(domain, encoded_account), + with address when is_binary(address) <- get_address_from_domain(domain, account), {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- HTTP.get( address, @@ -216,10 +228,28 @@ defmodule Pleroma.Web.WebFinger do _ -> {:error, {:content_type, nil}} end + |> case do + {:ok, data} -> validate_webfinger(address, data) + error -> error + end else error -> Logger.debug("Couldn't finger #{account}: #{inspect(error)}") error end end + + defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do + with [_name, acct_host] <- String.split(acct, "@"), + {_, url} <- {:address, get_address_from_domain(acct_host, subject)}, + %URI{host: request_host} <- URI.parse(request_url), + %URI{host: acct_host} <- URI.parse(url), + {_, true} <- {:hosts_match, acct_host == request_host} do + {:ok, data} + else + _ -> {:error, {:webfinger_invalid, request_url, data}} + end + end + + defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}} end diff --git a/lib/pleroma/web/xml.ex b/lib/pleroma/web/xml.ex index 64329e4ba..3997e1661 100644 --- a/lib/pleroma/web/xml.ex +++ b/lib/pleroma/web/xml.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.XML do def string_from_xpath(xpath, doc) do try do - {:xmlObj, :string, res} = :xmerl_xpath.string('string(#{xpath})', doc) + {:xmlObj, :string, res} = :xmerl_xpath.string(~c"string(#{xpath})", doc) res = res diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 4c1764053..0b570b70b 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo - use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" + use Pleroma.Workers.WorkerHelper, queue: "slow" @impl Oban.Worker def perform(%Job{ diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index dbf40ee1b..870aef3c6 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -3,7 +3,6 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackgroundWorker do - alias Pleroma.Instances.Instance alias Pleroma.User use Pleroma.Workers.WorkerHelper, queue: "background" @@ -15,11 +14,6 @@ defmodule Pleroma.Workers.BackgroundWorker do User.perform(:set_activation_async, user, status) end - def perform(%Job{args: %{"op" => "delete_user", "user_id" => user_id}}) do - user = User.get_cached_by_id(user_id) - User.perform(:delete, user) - end - def perform(%Job{args: %{"op" => "force_password_reset", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) User.perform(:force_password_reset, user) @@ -45,10 +39,6 @@ defmodule Pleroma.Workers.BackgroundWorker do User.perform(:verify_fields_links, user) end - def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do - Instance.perform(:delete_instance, host) - end - @impl Oban.Worker - def timeout(_job), do: :timer.seconds(900) + def timeout(_job), do: :timer.seconds(15) end diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index a485ddb4b..54ac31a3c 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackupWorker do - use Oban.Worker, queue: :backup, max_attempts: 1 + use Oban.Worker, queue: :slow, max_attempts: 1 alias Oban.Job alias Pleroma.User.Backup diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 0292bbb3b..17e92d10b 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -58,4 +58,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do User.touch_last_digest_emailed_at(user) end + + @impl Oban.Worker + def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 1c3e445aa..1f57aad4a 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(_job) do @@ -60,4 +60,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do :ok end + + @impl Oban.Worker + def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/delete_worker.ex b/lib/pleroma/workers/delete_worker.ex new file mode 100644 index 000000000..97003fb69 --- /dev/null +++ b/lib/pleroma/workers/delete_worker.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.DeleteWorker do + alias Pleroma.Instances.Instance + alias Pleroma.User + + use Pleroma.Workers.WorkerHelper, queue: "slow" + + @impl Oban.Worker + + def perform(%Job{args: %{"op" => "delete_user", "user_id" => user_id}}) do + user = User.get_cached_by_id(user_id) + User.perform(:delete, user) + end + + def perform(%Job{args: %{"op" => "delete_instance", "host" => host}}) do + Instance.perform(:delete_instance, host) + end + + @impl Oban.Worker + def timeout(_job), do: :timer.seconds(900) +end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 940716558..652bf77e0 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index 8ce458d48..a7ab5883a 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MuteExpireWorker do - use Pleroma.Workers.WorkerHelper, queue: "mute_expire" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do @@ -14,7 +14,7 @@ defmodule Pleroma.Workers.MuteExpireWorker do def perform(%Job{ args: %{"op" => "unmute_conversation", "user_id" => user_id, "activity_id" => activity_id} }) do - Pleroma.Web.CommonAPI.remove_mute(user_id, activity_id) + Pleroma.Web.CommonAPI.remove_mute(activity_id, user_id) :ok end diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index 022d026f8..af8997e70 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do @moduledoc """ Generates notifications when a poll ends. """ - use Pleroma.Workers.WorkerHelper, queue: "poll_notifications" + use Pleroma.Workers.WorkerHelper, queue: "background" alias Pleroma.Activity alias Pleroma.Notification @@ -14,8 +14,12 @@ defmodule Pleroma.Workers.PollWorker do @impl Oban.Worker def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do - with %Activity{} = activity <- find_poll_activity(activity_id) do - Notification.create_poll_notifications(activity) + with %Activity{} = activity <- find_poll_activity(activity_id), + {:ok, notifications} <- Notification.create_poll_notifications(activity) do + Notification.stream(notifications) + else + {:error, :poll_activity_not_found} = e -> {:cancel, e} + e -> {:error, e} end end diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index e554684fe..f48e34042 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do @moduledoc """ Worker which purges expired activity. """ - - use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity] + @queue :background + use Oban.Worker, queue: @queue, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -46,20 +46,22 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do defp find_activity(id) do with nil <- Activity.get_by_id_with_object(id) do - {:error, :activity_not_found} + {:cancel, :activity_not_found} end end defp find_user(ap_id) do with nil <- Pleroma.User.get_by_ap_id(ap_id) do - {:error, :user_not_found} + {:cancel, :user_not_found} end end def get_expiration(id) do + queue = Atom.to_string(@queue) + from(j in Oban.Job, where: j.state == "scheduled", - where: j.queue == "activity_expiration", + where: j.queue == ^queue, where: fragment("?->>'activity_id' = ?", j.args, ^id) ) |> Pleroma.Repo.one() diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index 9114aeb7f..1f6931e4c 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do Worker which purges expired filters """ - use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -38,7 +38,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do def get_expiration(id) do from(j in Job, where: j.state == "scheduled", - where: j.queue == "filter_expiration", + where: j.queue == "background", where: fragment("?->'filter_id' = ?", j.args, ^id) ) |> Repo.one() diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index 2ccd9e80b..1854bf561 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do Worker which purges expired OAuth tokens """ - use Oban.Worker, queue: :token_expiration, max_attempts: 1 + use Oban.Worker, queue: :background, max_attempts: 1 @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 8b2052c23..fd5c13fca 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -12,17 +12,30 @@ defmodule Pleroma.Workers.ReceiverWorker do @impl Oban.Worker def perform(%Job{ - args: %{"op" => "incoming_ap_doc", "req_headers" => req_headers, "params" => params} + args: %{ + "op" => "incoming_ap_doc", + "method" => method, + "params" => params, + "req_headers" => req_headers, + "request_path" => request_path, + "query_string" => query_string + } }) do # Oban's serialization converts our tuple headers to lists. # Revert it for the signature validation. req_headers = Enum.into(req_headers, [], &List.to_tuple(&1)) - conn_data = %{params: params, req_headers: req_headers} + conn_data = %Plug.Conn{ + method: method, + params: params, + req_headers: req_headers, + request_path: request_path, + query_string: query_string + } with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), {:ok, _public_key} <- Signature.refetch_public_key(conn_data), - {:signature, true} <- {:signature, HTTPSignatures.validate_conn(conn_data)}, + {:signature, true} <- {:signature, Signature.validate_signature(conn_data)}, {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do {:ok, res} else @@ -47,11 +60,13 @@ defmodule Pleroma.Workers.ReceiverWorker do case errors do {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} {:error, :already_present} -> {:cancel, :already_present} - {:error, {:validate_object, reason}} -> {:cancel, reason} - {:error, {:error, {:validate, reason}}} -> {:cancel, reason} - {:error, {:reject, reason}} -> {:cancel, reason} + {:error, {:validate_object, _} = reason} -> {:cancel, reason} + {:error, {:error, {:validate, {:error, _changeset} = reason}}} -> {:cancel, reason} + {:error, {:reject, _} = reason} -> {:cancel, reason} {:signature, false} -> {:cancel, :invalid_signature} - {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason} + {:error, "Object has been deleted"} = reason -> {:cancel, reason} + {:error, {:side_effects, {:error, :no_object_actor}} = reason} -> {:cancel, reason} + {:error, :not_found} = reason -> {:cancel, reason} {:error, _} = e -> e e -> {:error, e} end diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index c26418483..60096e14b 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do alias Pleroma.Object.Fetcher - use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do @@ -13,23 +13,23 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:ok, _object} -> :ok + {:reject, reason} -> + {:cancel, reason} + {:error, :forbidden} -> - {:discard, :forbidden} + {:cancel, :forbidden} {:error, :not_found} -> - {:discard, :not_found} + {:cancel, :not_found} {:error, :allowed_depth} -> - {:discard, :allowed_depth} + {:cancel, :allowed_depth} {:error, _} = e -> e - - e -> - {:error, e} end end @impl Oban.Worker - def timeout(_job), do: :timer.seconds(10) + def timeout(_job), do: :timer.seconds(15) end diff --git a/lib/pleroma/workers/rich_media_expiration_worker.ex b/lib/pleroma/workers/rich_media_expiration_worker.ex deleted file mode 100644 index d7ae497a7..000000000 --- a/lib/pleroma/workers/rich_media_expiration_worker.ex +++ /dev/null @@ -1,15 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.RichMediaExpirationWorker do - alias Pleroma.Web.RichMedia.Card - - use Oban.Worker, - queue: :rich_media_expiration - - @impl Oban.Worker - def perform(%Job{args: %{"url" => url} = _args}) do - Card.delete(url) - end -end diff --git a/lib/pleroma/workers/rich_media_worker.ex b/lib/pleroma/workers/rich_media_worker.ex new file mode 100644 index 000000000..2ebf42d4f --- /dev/null +++ b/lib/pleroma/workers/rich_media_worker.ex @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.RichMediaWorker do + alias Pleroma.Config + alias Pleroma.Web.RichMedia.Backfill + alias Pleroma.Web.RichMedia.Card + + use Oban.Worker, queue: :background, max_attempts: 3, unique: [period: 300] + + @impl Oban.Worker + def perform(%Job{args: %{"op" => "expire", "url" => url} = _args}) do + Card.delete(url) + end + + def perform(%Job{args: %{"op" => "backfill", "url" => _url} = args}) do + case Backfill.run(args) do + :ok -> + :ok + + {:error, type} + when type in [:invalid_metadata, :body_too_large, :content_type, :validate, :get, :head] -> + {:cancel, type} + + error -> + {:error, error} + end + end + + # There is timeout value enforced by Tesla.Middleware.Timeout + # which can be found in the RichMedia.Helpers module to allow us to detect + # a slow/infinite data stream and insert a negative cache entry for the URL + # We pad it by 2 seconds to be certain a slow connection is detected and we + # can inject a negative cache entry for the URL + @impl Oban.Worker + def timeout(_job) do + Config.get!([:rich_media, :timeout]) + :timer.seconds(2) + end +end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 4df84d00f..ab62686f4 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do The worker to post scheduled activity. """ - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" alias Pleroma.Repo alias Pleroma.ScheduledActivity diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 8476a2be5..8969ae378 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -20,4 +20,7 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module.remove_from_index(object) end + + @impl Oban.Worker + def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/user_refresh_worker.ex b/lib/pleroma/workers/user_refresh_worker.ex new file mode 100644 index 000000000..fb90e9c9c --- /dev/null +++ b/lib/pleroma/workers/user_refresh_worker.ex @@ -0,0 +1,17 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.UserRefreshWorker do + use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: 300] + + alias Pleroma.User + + @impl true + def perform(%Job{args: %{"ap_id" => ap_id}}) do + User.fetch_by_ap_id(ap_id) + end + + @impl Oban.Worker + def timeout(_job), do: :timer.seconds(15) +end diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index 67e84b0c9..c549d3cd6 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Workers.WebPusherWorker do alias Pleroma.Notification alias Pleroma.Repo + alias Pleroma.Web.Push.Impl use Pleroma.Workers.WorkerHelper, queue: "web_push" @@ -15,7 +16,8 @@ defmodule Pleroma.Workers.WebPusherWorker do |> Repo.get(notification_id) |> Repo.preload([:activity, :user]) - Pleroma.Web.Push.Impl.perform(notification) + Impl.build(notification) + |> Enum.each(&Impl.deliver(&1)) end @impl Oban.Worker |