From 8f285a787f7af0436de18ac140712801a69eff05 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 21 Jun 2024 13:24:47 -0400 Subject: Refactor backups to be fully controlled by Oban --- lib/pleroma/ecto_enums.ex | 8 - lib/pleroma/emails/user_email.ex | 35 +- lib/pleroma/user/backup.ex | 358 +++++++++------------ .../admin_api/controllers/admin_api_controller.ex | 5 +- .../operations/pleroma_backup_operation.ex | 10 +- .../pleroma_api/controllers/backup_controller.ex | 2 +- lib/pleroma/web/pleroma_api/views/backup_view.ex | 10 - lib/pleroma/workers/backup_worker.ex | 47 +-- 8 files changed, 183 insertions(+), 292 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex index b346b39d6..a4890b489 100644 --- a/lib/pleroma/ecto_enums.ex +++ b/lib/pleroma/ecto_enums.ex @@ -27,11 +27,3 @@ defenum(Pleroma.DataMigration.State, failed: 4, manual: 5 ) - -defenum(Pleroma.User.Backup.State, - pending: 1, - running: 2, - complete: 3, - failed: 4, - invalid: 5 -) diff --git a/lib/pleroma/emails/user_email.ex b/lib/pleroma/emails/user_email.ex index 95b963764..10d89d2f3 100644 --- a/lib/pleroma/emails/user_email.ex +++ b/lib/pleroma/emails/user_email.ex @@ -345,37 +345,22 @@ defmodule Pleroma.Emails.UserEmail do Router.Helpers.subscription_url(Endpoint, :unsubscribe, token) end - def backup_is_ready_email(backup, admin_user_id \\ nil) do + def backup_is_ready_email(backup) do %{user: user} = Pleroma.Repo.preload(backup, :user) Gettext.with_locale_or_default user.language do download_url = Pleroma.Web.PleromaAPI.BackupView.download_url(backup) html_body = - if is_nil(admin_user_id) do - Gettext.dpgettext( - "static_pages", - "account archive email body - self-requested", - """ -

You requested a full backup of your Pleroma account. It's ready for download:

-

%{download_url}

- """, - download_url: download_url - ) - else - admin = Pleroma.Repo.get(User, admin_user_id) - - Gettext.dpgettext( - "static_pages", - "account archive email body - admin requested", - """ -

Admin @%{admin_nickname} requested a full backup of your Pleroma account. It's ready for download:

-

%{download_url}

- """, - admin_nickname: admin.nickname, - download_url: download_url - ) - end + Gettext.dpgettext( + "static_pages", + "account archive email body", + """ +

A full backup of your Pleroma account was requested. It's ready for download:

+

%{download_url}

+ """, + download_url: download_url + ) new() |> to(recipient(user)) diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 1821de667..2568089a4 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -14,9 +14,10 @@ defmodule Pleroma.User.Backup do alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.Config alias Pleroma.Repo + alias Pleroma.Uploaders.Uploader alias Pleroma.User - alias Pleroma.User.Backup.State alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView @@ -29,71 +30,111 @@ defmodule Pleroma.User.Backup do field(:file_name, :string) field(:file_size, :integer, default: 0) field(:processed, :boolean, default: false) - field(:state, State, default: :invalid) - field(:processed_number, :integer, default: 0) + field(:tempdir, :string) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) timestamps() end - @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + @doc """ + Schedules a job to backup a user if the number of backup requests has not exceeded the limit. - def create(user, admin_id \\ nil) do - with :ok <- validate_limit(user, admin_id), - {:ok, backup} <- user |> new() |> Repo.insert() do - BackupWorker.process(backup, admin_id) + Admins can directly call new/1 and schedule_backup/1 to bypass the limit. + """ + @spec user(User.t()) :: {:ok, t()} | {:error, any()} + def user(user) do + days = Config.get([__MODULE__, :limit_days]) + + with true <- permitted?(user), + %__MODULE__{} = backup <- new(user), + {:ok, inserted_backup} <- Repo.insert(backup), + {:ok, %Oban.Job{}} <- schedule_backup(inserted_backup) do + {:ok, inserted_backup} + else + false -> + {:error, + dngettext( + "errors", + "Last export was less than a day ago", + "Last export was less than %{days} days ago", + days, + days: days + )} + + e -> + {:error, e} end end + @doc "Generates a %Backup{} for a user with a random file name" + @spec new(User.t()) :: t() def new(user) do rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" %__MODULE__{ - user_id: user.id, content_type: "application/zip", file_name: name, - state: :pending + tempdir: tempdir(), + user: user } end - def delete(backup) do - uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + @doc "Schedules the execution of the provided backup" + @spec schedule_backup(t()) :: {:ok, Oban.Job.t()} | {:error, any()} + def schedule_backup(backup) do + with false <- is_nil(backup.id) do + %{"op" => "process", "backup_id" => backup.id} + |> BackupWorker.new() + |> Oban.insert() + else + true -> + {:error, "Backup is missing id. Please insert it into the Repo first."} + + e -> + {:error, e} + end + end + + @doc "Deletes the backup archive file and removes the database record" + @spec delete_archive(t()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} + def delete_archive(backup) do + uploader = Config.get([Pleroma.Upload, :uploader]) with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do Repo.delete(backup) end end - defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok - - defp validate_limit(user, nil) do - case get_last(user.id) do - %__MODULE__{inserted_at: inserted_at} -> - days = Pleroma.Config.get([__MODULE__, :limit_days]) - diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) - - if diff > days do - :ok - else - {:error, - dngettext( - "errors", - "Last export was less than a day ago", - "Last export was less than %{days} days ago", - days, - days: days - )} - end + @doc "Schedules a job to delete the backup archive" + @spec schedule_delete(t()) :: {:ok, Oban.Job.t()} | {:error, any()} + def schedule_delete(backup) do + days = Config.get([__MODULE__, :purge_after_days]) + time = 60 * 60 * 24 * days + scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) - nil -> - :ok + %{"op" => "delete", "backup_id" => backup.id} + |> BackupWorker.new(scheduled_at: scheduled_at) + |> Oban.insert() + end + + defp permitted?(user) do + with {_, %__MODULE__{inserted_at: inserted_at}} <- {:last, get_last(user)}, + days = Config.get([__MODULE__, :limit_days]), + diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days), + {_, true} <- {:diff, diff > days} do + true + else + {:last, nil} -> true + {:diff, false} -> false end end - def get_last(user_id) do + @doc "Returns last backup for the provided user" + @spec get_last(User.t()) :: t() + def get_last(%User{id: user_id}) do __MODULE__ |> where(user_id: ^user_id) |> order_by(desc: :id) @@ -101,6 +142,8 @@ defmodule Pleroma.User.Backup do |> Repo.one() end + @doc "Lists all existing backups for a user" + @spec list(User.t()) :: [Ecto.Schema.t() | term()] def list(%User{id: user_id}) do __MODULE__ |> where(user_id: ^user_id) @@ -108,92 +151,35 @@ defmodule Pleroma.User.Backup do |> Repo.all() end - def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do - __MODULE__ - |> where(user_id: ^user_id) - |> where([b], b.id != ^latest_id) - |> Repo.all() - |> Enum.each(&BackupWorker.delete/1) + @doc "Schedules deletion of all but the the most recent backup" + @spec remove_outdated(User.t()) :: :ok + def remove_outdated(user) do + with %__MODULE__{} = latest_backup <- get_last(user) do + __MODULE__ + |> where(user_id: ^user.id) + |> where([b], b.id != ^latest_backup.id) + |> Repo.all() + |> Enum.each(&schedule_delete/1) + else + _ -> :ok + end end - def get(id), do: Repo.get(__MODULE__, id) - - defp set_state(backup, state, processed_number \\ nil) do - struct = - %{state: state} - |> Pleroma.Maps.put_if_present(:processed_number, processed_number) + def get_by_id(id), do: Repo.get(__MODULE__, id) + @doc "Generates changeset for %Pleroma.User.Backup{}" + @spec changeset(%__MODULE__{}, map()) :: %Ecto.Changeset{} + def changeset(backup \\ %__MODULE__{}, attrs) do backup - |> cast(struct, [:state, :processed_number]) - |> Repo.update() - end - - def process( - %__MODULE__{} = backup, - processor_module \\ __MODULE__.Processor - ) do - set_state(backup, :running, 0) - - current_pid = self() - - task = - Task.Supervisor.async_nolink( - Pleroma.TaskSupervisor, - processor_module, - :do_process, - [backup, current_pid] - ) - - wait_backup(backup, backup.processed_number, task) + |> cast(attrs, [:content_type, :file_name, :file_size, :processed, :tempdir]) end - defp wait_backup(backup, current_processed, task) do - wait_time = @config_impl.get([__MODULE__, :process_wait_time]) - - receive do - {:progress, new_processed} -> - total_processed = current_processed + new_processed - - set_state(backup, :running, total_processed) - wait_backup(backup, total_processed, task) - - {:DOWN, _ref, _proc, _pid, reason} -> - backup = get(backup.id) - - if reason != :normal do - Logger.error("Backup #{backup.id} process ended abnormally: #{inspect(reason)}") - - {:ok, backup} = set_state(backup, :failed) - - cleanup(backup) - - {:error, - %{ - backup: backup, - reason: :exit, - details: reason - }} - else - {:ok, backup} - end - after - wait_time -> - Logger.error( - "Backup #{backup.id} timed out after no response for #{wait_time}ms, terminating" - ) - - Task.Supervisor.terminate_child(Pleroma.TaskSupervisor, task.pid) - - {:ok, backup} = set_state(backup, :failed) - - cleanup(backup) - - {:error, - %{ - backup: backup, - reason: :timeout - }} - end + @doc "Updates the backup record" + @spec update_record(%__MODULE__{}, map()) :: {:ok, %__MODULE__{}} | {:error, %Ecto.Changeset{}} + def update_record(%__MODULE__{} = backup, attrs) do + backup + |> changeset(attrs) + |> Repo.update() end @files [ @@ -204,53 +190,66 @@ defmodule Pleroma.User.Backup do ~c"followers.json", ~c"following.json" ] - @spec export(Pleroma.User.Backup.t(), pid()) :: {:ok, String.t()} | :error - def export(%__MODULE__{} = backup, caller_pid) do + + @spec run(t()) :: {:ok, t()} | {:error, :failed} + def run(%__MODULE__{} = backup) do backup = Repo.preload(backup, :user) - dir = backup_tempdir(backup) - - with :ok <- File.mkdir(dir), - :ok <- actor(dir, backup.user, caller_pid), - :ok <- statuses(dir, backup.user, caller_pid), - :ok <- likes(dir, backup.user, caller_pid), - :ok <- bookmarks(dir, backup.user, caller_pid), - :ok <- followers(dir, backup.user, caller_pid), - :ok <- following(dir, backup.user, caller_pid), - {:ok, zip_path} <- :zip.create(backup.file_name, @files, cwd: dir), - {:ok, _} <- File.rm_rf(dir) do - {:ok, zip_path} + tempfile = Path.join([backup.tempdir, backup.file_name]) + + with {_, :ok} <- {:mkdir, File.mkdir_p(backup.tempdir)}, + {_, :ok} <- {:actor, actor(backup.tempdir, backup.user)}, + {_, :ok} <- {:statuses, statuses(backup.tempdir, backup.user)}, + {_, :ok} <- {:likes, likes(backup.tempdir, backup.user)}, + {_, :ok} <- {:bookmarks, bookmarks(backup.tempdir, backup.user)}, + {_, :ok} <- {:followers, followers(backup.tempdir, backup.user)}, + {_, :ok} <- {:following, following(backup.tempdir, backup.user)}, + {_, {:ok, _zip_path}} <- + {:zip, :zip.create(to_charlist(tempfile), @files, cwd: to_charlist(backup.tempdir))}, + {_, {:ok, %File.Stat{size: zip_size}}} <- {:filestat, File.stat(tempfile)}, + {:ok, updated_backup} <- update_record(backup, %{file_size: zip_size}) do + {:ok, updated_backup} else - _ -> :error + _ -> + File.rm_rf(backup.tempdir) + {:error, :failed} end end - def dir(name) do - dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!() - Path.join(dir, name) + defp tempdir do + case Config.get([__MODULE__, :tempdir]) do + nil -> + System.tmp_dir!() + + path -> + rand = :crypto.strong_rand_bytes(8) |> Base.url_encode64(padding: false) + Path.join([path, rand]) + end end - def upload(%__MODULE__{} = backup, zip_path) do - uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + @doc "Uploads the completed backup and marks it as processed" + @spec upload(t()) :: {:ok, t()} + def upload(%__MODULE__{tempdir: tempdir} = backup) when is_binary(tempdir) do + uploader = Config.get([Pleroma.Upload, :uploader]) upload = %Pleroma.Upload{ name: backup.file_name, - tempfile: zip_path, + tempfile: Path.join([tempdir, backup.file_name]), content_type: backup.content_type, path: Path.join("backups", backup.file_name) } - with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), - :ok <- File.rm(zip_path) do - {:ok, upload} + with {:ok, _} <- Uploader.put_file(uploader, upload), + {:ok, uploaded_backup} <- update_record(backup, %{processed: true}), + {:ok, _} <- File.rm_rf(tempdir) do + {:ok, uploaded_backup} end end - defp actor(dir, user, caller_pid) do + defp actor(dir, user) do with {:ok, json} <- UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) |> Jason.encode() do - send(caller_pid, {:progress, 1}) File.write(Path.join(dir, "actor.json"), json) end end @@ -269,22 +268,10 @@ defmodule Pleroma.User.Backup do ) end - defp should_report?(num, chunk_size), do: rem(num, chunk_size) == 0 - - defp backup_tempdir(backup) do - name = String.trim_trailing(backup.file_name, ".zip") - dir(name) - end - - defp cleanup(backup) do - dir = backup_tempdir(backup) - File.rm_rf(dir) - end - - defp write(query, dir, name, fun, caller_pid) do + defp write(query, dir, name, fun) do path = Path.join(dir, "#{name}.json") - chunk_size = Pleroma.Config.get([__MODULE__, :process_chunk_size]) + chunk_size = Config.get([__MODULE__, :process_chunk_size]) with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do @@ -300,10 +287,6 @@ defmodule Pleroma.User.Backup do end), {:ok, str} <- Jason.encode(data), :ok <- IO.write(file, str <> ",\n") do - if should_report?(acc + 1, chunk_size) do - send(caller_pid, {:progress, chunk_size}) - end - acc + 1 else {:error, e} -> @@ -318,31 +301,29 @@ defmodule Pleroma.User.Backup do end end) - send(caller_pid, {:progress, rem(total, chunk_size)}) - with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do File.close(file) end end end - defp bookmarks(dir, %{id: user_id} = _user, caller_pid) do + defp bookmarks(dir, %{id: user_id} = _user) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) - |> write(dir, "bookmarks", fn a -> {:ok, a.object} end, caller_pid) + |> write(dir, "bookmarks", fn a -> {:ok, a.object} end) end - defp likes(dir, user, caller_pid) do + defp likes(dir, user) do user.ap_id |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) - |> write(dir, "likes", fn a -> {:ok, a.object} end, caller_pid) + |> write(dir, "likes", fn a -> {:ok, a.object} end) end - defp statuses(dir, user, caller_pid) do + defp statuses(dir, user) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) @@ -362,52 +343,17 @@ defmodule Pleroma.User.Backup do with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do {:ok, Map.delete(activity, "@context")} end - end, - caller_pid + end ) end - defp followers(dir, user, caller_pid) do + defp followers(dir, user) do User.get_followers_query(user) - |> write(dir, "followers", fn a -> {:ok, a.ap_id} end, caller_pid) + |> write(dir, "followers", fn a -> {:ok, a.ap_id} end) end - defp following(dir, user, caller_pid) do + defp following(dir, user) do User.get_friends_query(user) - |> write(dir, "following", fn a -> {:ok, a.ap_id} end, caller_pid) - end -end - -defmodule Pleroma.User.Backup.ProcessorAPI do - @callback do_process(%Pleroma.User.Backup{}, pid()) :: - {:ok, %Pleroma.User.Backup{}} | {:error, any()} -end - -defmodule Pleroma.User.Backup.Processor do - @behaviour Pleroma.User.Backup.ProcessorAPI - - alias Pleroma.Repo - alias Pleroma.User.Backup - - import Ecto.Changeset - - @impl true - def do_process(backup, current_pid) do - with {:ok, zip_file} <- Backup.export(backup, current_pid), - {:ok, %{size: size}} <- File.stat(zip_file), - {:ok, _upload} <- Backup.upload(backup, zip_file) do - backup - |> cast( - %{ - file_size: size, - processed: true, - state: :complete - }, - [:file_size, :processed, :state] - ) - |> Repo.update() - else - e -> {:error, e} - end + |> write(dir, "following", fn a -> {:ok, a.ap_id} end) end end diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index 1894000ff..0f22dd538 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -13,6 +13,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do alias Pleroma.ModerationLog alias Pleroma.Stats alias Pleroma.User + alias Pleroma.User.Backup alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.AccountView @@ -429,7 +430,9 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_by_nickname(nickname), - {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do + %Backup{} = backup <- Backup.new(user), + {:ok, inserted_backup} <- Pleroma.Repo.insert(backup), + {:ok, %Oban.Job{}} <- Backup.schedule_backup(inserted_backup) do ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"}) json(conn, "") diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex index 400f3825d..86f709515 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -65,12 +65,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do file_name: %Schema{type: :string}, file_size: %Schema{type: :integer}, processed: %Schema{type: :boolean, description: "whether this backup has succeeded"}, - state: %Schema{ - type: :string, - description: "the state of the backup", - enum: ["pending", "running", "complete", "failed"] - }, - processed_number: %Schema{type: :integer, description: "the number of records processed"} + tempdir: %Schema{type: :string} }, example: %{ "content_type" => "application/zip", @@ -79,8 +74,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do "file_size" => 4105, "inserted_at" => "2020-09-08T16:42:07.000Z", "processed" => true, - "state" => "complete", - "processed_number" => 20 + "tempdir" => "/tmp/PZIMw40vmpM" } } end diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index b9daed22b..0115ec645 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do end def create(%{assigns: %{user: user}} = conn, _params) do - with {:ok, _} <- Backup.create(user) do + with {:ok, _} <- Backup.user(user) do backups = Backup.list(user) render(conn, "index.json", backups: backups) end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index 20403aeee..d778590f0 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -9,22 +9,12 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do alias Pleroma.Web.CommonAPI.Utils def render("show.json", %{backup: %Backup{} = backup}) do - # To deal with records before the migration - state = - if backup.state == :invalid do - if backup.processed, do: :complete, else: :failed - else - backup.state - end - %{ id: backup.id, content_type: backup.content_type, url: download_url(backup), file_size: backup.file_size, processed: backup.processed, - state: to_string(state), - processed_number: backup.processed_number, inserted_at: Utils.to_masto_date(backup.inserted_at) } end diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 54ac31a3c..41f404e69 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -8,44 +8,25 @@ defmodule Pleroma.Workers.BackupWorker do alias Oban.Job alias Pleroma.User.Backup - def process(backup, admin_user_id \\ nil) do - %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id} - |> new() - |> Oban.insert() - end - - def schedule_deletion(backup) do - days = Pleroma.Config.get([Backup, :purge_after_days]) - time = 60 * 60 * 24 * days - scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) - - %{"op" => "delete", "backup_id" => backup.id} - |> new(scheduled_at: scheduled_at) - |> Oban.insert() - end - - def delete(backup) do - %{"op" => "delete", "backup_id" => backup.id} - |> new() - |> Oban.insert() - end - @impl Oban.Worker def perform(%Job{ - args: %{"op" => "process", "backup_id" => backup_id, "admin_user_id" => admin_user_id} + args: %{"op" => "process", "backup_id" => backup_id} }) do - with {:ok, %Backup{} = backup} <- - backup_id |> Backup.get() |> Backup.process(), - {:ok, _job} <- schedule_deletion(backup), - :ok <- Backup.remove_outdated(backup), - :ok <- maybe_deliver_email(backup, admin_user_id) do - {:ok, backup} + with {_, %Backup{} = backup} <- {:get, Backup.get_by_id(backup_id)}, + {_, {:ok, updated_backup}} <- {:run, Backup.run(backup)}, + {_, {:ok, uploaded_backup}} <- {:upload, Backup.upload(updated_backup)}, + {_, {:ok, _job}} <- {:delete, Backup.schedule_delete(uploaded_backup)}, + {_, :ok} <- {:outdated, Backup.remove_outdated(uploaded_backup.user)}, + {_, :ok} <- {:email, maybe_deliver_email(uploaded_backup)} do + {:ok, uploaded_backup} + else + e -> {:error, e} end end def perform(%Job{args: %{"op" => "delete", "backup_id" => backup_id}}) do - case Backup.get(backup_id) do - %Backup{} = backup -> Backup.delete(backup) + case Backup.get_by_id(backup_id) do + %Backup{} = backup -> Backup.delete_archive(backup) nil -> :ok end end @@ -57,13 +38,13 @@ defmodule Pleroma.Workers.BackupWorker do not is_nil(user.email) and user.email != "" end - defp maybe_deliver_email(backup, admin_user_id) do + defp maybe_deliver_email(backup) do has_mailer = Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) backup = backup |> Pleroma.Repo.preload(:user) if has_email?(backup.user) and has_mailer do backup - |> Pleroma.Emails.UserEmail.backup_is_ready_email(admin_user_id) + |> Pleroma.Emails.UserEmail.backup_is_ready_email() |> Pleroma.Emails.Mailer.deliver() :ok -- cgit v1.2.3 From ece063586b5e2c2e8c4eb6e92e50b3dcc09d2836 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 24 Jun 2024 16:36:43 -0400 Subject: Limit backup jobs to 5 minutes --- lib/pleroma/workers/backup_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 41f404e69..fb366de4a 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Workers.BackupWorker do end @impl Oban.Worker - def timeout(_job), do: :infinity + def timeout(_job), do: :timer.minutes(5) defp has_email?(user) do not is_nil(user.email) and user.email != "" -- cgit v1.2.3 From 3f60d7bf644fc38e8e5d9f525a1675526ade0037 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 24 Jun 2024 22:52:21 -0400 Subject: Better random tempdir format --- lib/pleroma/user/backup.ex | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 2568089a4..7feaa22bf 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -216,13 +216,15 @@ defmodule Pleroma.User.Backup do end defp tempdir do + rand = :crypto.strong_rand_bytes(8) |> Base.url_encode64(padding: false) + subdir = "backup-#{rand}" + case Config.get([__MODULE__, :tempdir]) do nil -> - System.tmp_dir!() + Path.join([System.tmp_dir!(), subdir]) path -> - rand = :crypto.strong_rand_bytes(8) |> Base.url_encode64(padding: false) - Path.join([path, rand]) + Path.join([path, subdir]) end end -- cgit v1.2.3 From e5cbbaf3f0385492580b27a624b936c1d74757a1 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 24 Jun 2024 22:58:38 -0400 Subject: Extend the backup job time limit to 30 minutes --- lib/pleroma/workers/backup_worker.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index fb366de4a..1791c558d 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -32,7 +32,7 @@ defmodule Pleroma.Workers.BackupWorker do end @impl Oban.Worker - def timeout(_job), do: :timer.minutes(5) + def timeout(_job), do: :timer.minutes(30) defp has_email?(user) do not is_nil(user.email) and user.email != "" -- cgit v1.2.3 From 187897874bddc27ef23606730e79522e3583ec43 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 22 Jul 2024 14:00:54 -0400 Subject: Make backup timeout configurable --- lib/pleroma/workers/backup_worker.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 1791c558d..d1b6fcdad 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Workers.BackupWorker do use Oban.Worker, queue: :slow, max_attempts: 1 alias Oban.Job + alias Pleroma.Config.Getting, as: Config alias Pleroma.User.Backup @impl Oban.Worker @@ -32,7 +33,7 @@ defmodule Pleroma.Workers.BackupWorker do end @impl Oban.Worker - def timeout(_job), do: :timer.minutes(30) + def timeout(_job), do: Config.get([Backup, :timeout], :timer.minutes(30)) defp has_email?(user) do not is_nil(user.email) and user.email != "" -- cgit v1.2.3 From 59309a9eff5c2e61b2195945eca21c5126eb3f5f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 28 Jul 2024 20:41:21 -0400 Subject: Publisher job simplification Publisher jobs now store the the activity id instead of inserting duplicate JSON data in the Oban queue for each delivery. --- lib/pleroma/web/activity_pub/publisher.ex | 41 +++++++++++++++++-------------- 1 file changed, 22 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index c8bdf2250..cb436e8d8 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -80,13 +80,26 @@ defmodule Pleroma.Web.ActivityPub.Publisher do parameters set: * `inbox`: the inbox to publish to - * `json`: the JSON message body representing the ActivityPub message * `actor`: the actor which is signing the message - * `id`: the ActivityStreams URI of the message + * `activity_id`: the internal activity id + * `cc`: the cc recipients relevant to this inbox (optional) """ - def publish_one(%{inbox: inbox, json: json, actor: %User{} = actor, id: id} = params) do - Logger.debug("Federating #{id} to #{inbox}") + def publish_one(%{inbox: inbox, actor: %User{} = actor, activity_id: activity_id} = params) do + activity = Activity.get_by_id(activity_id) + + ap_id = activity.data["id"] + Logger.debug("Federating #{ap_id} to #{inbox}") uri = %{path: path} = URI.parse(inbox) + + {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) + + cc = Map.get(params, :cc) + + json = + data + |> Map.put("cc", cc) + |> Jason.encode!() + digest = "SHA-256=" <> (:crypto.hash(:sha256, json) |> Base.encode64()) date = Pleroma.Signature.signed_date() @@ -119,7 +132,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do else {_post_result, %{status: code} = response} = e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) - Logger.metadata(activity: id, inbox: inbox, status: code) + Logger.metadata(id: activity_id, inbox: inbox, status: code) Logger.error("Publisher failed to inbox #{inbox} with status #{code}") case response do @@ -136,7 +149,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) - Logger.metadata(activity: id, inbox: inbox) + Logger.metadata(activity: activity_id, inbox: inbox) Logger.error("Publisher failed to inbox #{inbox} #{inspect(e)}") {:error, e} end @@ -251,7 +264,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do def publish(%User{} = actor, %{data: %{"bcc" => bcc}} = activity) when is_list(bcc) and bcc != [] do public = public?(activity) - {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) [priority_recipients, recipients] = recipients(actor, activity) @@ -276,16 +288,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do # instance would only accept a first message for the first recipient and ignore the rest. cc = get_cc_ap_ids(ap_id, recipients) - json = - data - |> Map.put("cc", cc) - |> Jason.encode!() - __MODULE__.enqueue_one(%{ inbox: inbox, - json: json, + cc: cc, actor_id: actor.id, - id: activity.data["id"], + activity_id: activity.id, unreachable_since: unreachable_since }) end) @@ -302,9 +309,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do Relay.publish(activity) end - {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - json = Jason.encode!(data) - [priority_inboxes, inboxes] = recipients(actor, activity) |> Enum.map(fn recipients -> @@ -326,9 +330,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do __MODULE__.enqueue_one( %{ inbox: inbox, - json: json, actor_id: actor.id, - id: activity.data["id"], + activity_id: activity.id, unreachable_since: unreachable_since }, priority: priority -- cgit v1.2.3 From 74072622e08dd1efdc7bf69c3278250ea1efb22e Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 Jul 2024 09:52:13 -0400 Subject: Remove actor and actor_id from the job as it can be inferred by the activity --- lib/pleroma/web/activity_pub/publisher.ex | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index cb436e8d8..373bf6e41 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -80,12 +80,12 @@ defmodule Pleroma.Web.ActivityPub.Publisher do parameters set: * `inbox`: the inbox to publish to - * `actor`: the actor which is signing the message * `activity_id`: the internal activity id * `cc`: the cc recipients relevant to this inbox (optional) """ - def publish_one(%{inbox: inbox, actor: %User{} = actor, activity_id: activity_id} = params) do - activity = Activity.get_by_id(activity_id) + def publish_one(%{inbox: inbox, activity_id: activity_id} = params) do + activity = Activity.get_by_id_with_user_actor(activity_id) + actor = activity.user_actor ap_id = activity.data["id"] Logger.debug("Federating #{ap_id} to #{inbox}") @@ -155,15 +155,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end - def publish_one(%{actor_id: actor_id} = params) do - actor = User.get_cached_by_id(actor_id) - - params - |> Map.delete(:actor_id) - |> Map.put(:actor, actor) - |> publish_one() - end - defp signature_host(%URI{port: port, scheme: scheme, host: host}) do if port == URI.default_port(scheme) do host @@ -291,7 +282,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do __MODULE__.enqueue_one(%{ inbox: inbox, cc: cc, - actor_id: actor.id, activity_id: activity.id, unreachable_since: unreachable_since }) @@ -330,7 +320,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do __MODULE__.enqueue_one( %{ inbox: inbox, - actor_id: actor.id, activity_id: activity.id, unreachable_since: unreachable_since }, -- cgit v1.2.3 From 8893ad98997197bd89e98f7dd18825dcb1206aa4 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 Jul 2024 09:59:35 -0400 Subject: Fix cancelling jobs --- lib/pleroma/web/common_api.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 06faf845e..b90b6a6d9 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -714,11 +714,11 @@ defmodule Pleroma.Web.CommonAPI do end end - defp maybe_cancel_jobs(%Activity{data: %{"id" => ap_id}}) do + defp maybe_cancel_jobs(%Activity{id: activity_id}) do Oban.Job |> where([j], j.worker == "Pleroma.Workers.PublisherWorker") |> where([j], j.args["op"] == "publish_one") - |> where([j], j.args["params"]["id"] == ^ap_id) + |> where([j], j.args["params"]["activity_id"] == ^activity_id) |> Oban.cancel_all_jobs() end -- cgit v1.2.3 From b48fd89a41ad766b79a7a2336737196216cede22 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 29 Jul 2024 10:03:22 -0400 Subject: Revert unintended change to the Logger metadata tag name --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 373bf6e41..e040753dc 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -132,7 +132,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do else {_post_result, %{status: code} = response} = e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) - Logger.metadata(id: activity_id, inbox: inbox, status: code) + Logger.metadata(activity: activity_id, inbox: inbox, status: code) Logger.error("Publisher failed to inbox #{inbox} with status #{code}") case response do -- cgit v1.2.3 From 52e9bec15655dd3ba75c133b85266a1ea65a8eef Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 28 Jun 2024 11:47:31 -0400 Subject: Remove WorkerHelper --- lib/mix/tasks/pleroma/database.ex | 10 +++-- lib/pleroma/emails/mailer.ex | 3 +- lib/pleroma/filter.ex | 11 +++-- lib/pleroma/instances/instance.ex | 3 +- lib/pleroma/mfa/token.ex | 13 +++--- lib/pleroma/object.ex | 3 +- lib/pleroma/search.ex | 6 ++- lib/pleroma/user.ex | 19 +++++---- lib/pleroma/user/import.ex | 30 ++++++++------ lib/pleroma/web/activity_pub/activity_pub.ex | 18 +++++--- lib/pleroma/web/activity_pub/publisher.ex | 6 +-- lib/pleroma/web/activity_pub/side_effects.ex | 14 ++++--- lib/pleroma/web/common_api.ex | 6 +-- lib/pleroma/web/federator.ex | 27 ++++++++---- lib/pleroma/web/o_auth/token.ex | 9 ++-- lib/pleroma/web/push.ex | 2 +- lib/pleroma/workers/attachments_cleanup_worker.ex | 6 +-- lib/pleroma/workers/background_worker.ex | 8 ++-- lib/pleroma/workers/backup_worker.ex | 4 +- lib/pleroma/workers/cron/digest_emails_worker.ex | 4 +- .../workers/cron/new_users_digest_worker.ex | 6 +-- lib/pleroma/workers/delete_worker.ex | 7 ++-- lib/pleroma/workers/mailer_worker.ex | 6 +-- lib/pleroma/workers/mute_expire_worker.ex | 6 +-- lib/pleroma/workers/poll_worker.ex | 6 +-- lib/pleroma/workers/publisher_worker.ex | 22 ++++++---- lib/pleroma/workers/purge_expired_activity.ex | 11 ++--- lib/pleroma/workers/purge_expired_filter.ex | 2 +- lib/pleroma/workers/purge_expired_token.ex | 12 +----- lib/pleroma/workers/receiver_worker.ex | 6 +-- lib/pleroma/workers/remote_fetcher_worker.ex | 6 +-- lib/pleroma/workers/rich_media_worker.ex | 4 +- lib/pleroma/workers/scheduled_activity_worker.ex | 6 +-- lib/pleroma/workers/search_indexing_worker.ex | 6 +-- lib/pleroma/workers/user_refresh_worker.ex | 2 +- lib/pleroma/workers/web_pusher_worker.ex | 6 +-- lib/pleroma/workers/worker_helper.ex | 48 ---------------------- 37 files changed, 176 insertions(+), 188 deletions(-) delete mode 100644 lib/pleroma/workers/worker_helper.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index b82d1f079..e52b5e0a7 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -295,10 +295,12 @@ defmodule Mix.Tasks.Pleroma.Database do |> DateTime.from_naive!("Etc/UTC") |> Timex.shift(days: days) - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: expires_at - }) + Pleroma.Workers.PurgeExpiredActivity.enqueue( + %{ + activity_id: activity.id + }, + scheduled_at: expires_at + ) end) end) |> Stream.run() diff --git a/lib/pleroma/emails/mailer.ex b/lib/pleroma/emails/mailer.ex index 101442130..2a80f8547 100644 --- a/lib/pleroma/emails/mailer.ex +++ b/lib/pleroma/emails/mailer.ex @@ -25,7 +25,8 @@ defmodule Pleroma.Emails.Mailer do |> :erlang.term_to_binary() |> Base.encode64() - MailerWorker.enqueue("email", %{"encoded_email" => encoded_email, "config" => config}) + MailerWorker.new(%{"op" => "email", "encoded_email" => encoded_email, "config" => config}) + |> Oban.insert() end @doc "callback to perform send email from queue" diff --git a/lib/pleroma/filter.ex b/lib/pleroma/filter.ex index e827d3cbc..77ed64d4f 100644 --- a/lib/pleroma/filter.ex +++ b/lib/pleroma/filter.ex @@ -133,10 +133,13 @@ defmodule Pleroma.Filter do defp maybe_add_expires_at(changeset, _), do: changeset defp maybe_add_expiration_job(%{expires_at: %NaiveDateTime{} = expires_at} = filter) do - Pleroma.Workers.PurgeExpiredFilter.enqueue(%{ - filter_id: filter.id, - expires_at: DateTime.from_naive!(expires_at, "Etc/UTC") - }) + Pleroma.Workers.PurgeExpiredFilter.new( + %{ + filter_id: filter.id + }, + scheduled_at: DateTime.from_naive!(expires_at, "Etc/UTC") + ) + |> Oban.insert() end defp maybe_add_expiration_job(_), do: {:ok, nil} diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 288555146..33f1229d0 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -297,7 +297,8 @@ defmodule Pleroma.Instances.Instance do all of those users' activities and notifications. """ def delete_users_and_activities(host) when is_binary(host) do - DeleteWorker.enqueue("delete_instance", %{"host" => host}) + DeleteWorker.new(%{"op" => "delete_instance", "host" => host}) + |> Oban.insert() end def perform(:delete_instance, host) when is_binary(host) do diff --git a/lib/pleroma/mfa/token.ex b/lib/pleroma/mfa/token.ex index 57bc11ed5..b53e1c7d0 100644 --- a/lib/pleroma/mfa/token.ex +++ b/lib/pleroma/mfa/token.ex @@ -52,11 +52,14 @@ defmodule Pleroma.MFA.Token do @spec create(User.t(), Authorization.t() | nil) :: {:ok, t()} | {:error, Ecto.Changeset.t()} def create(user, authorization \\ nil) do with {:ok, token} <- do_create(user, authorization) do - Pleroma.Workers.PurgeExpiredToken.enqueue(%{ - token_id: token.id, - valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), - mod: __MODULE__ - }) + Pleroma.Workers.PurgeExpiredToken.new( + %{ + token_id: token.id, + mod: __MODULE__ + }, + scheduled_at: DateTime.from_naive!(token.valid_until, "Etc/UTC") + ) + |> Oban.insert() {:ok, token} end diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index eb44b3855..748f18e6c 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -255,7 +255,8 @@ defmodule Pleroma.Object do @spec cleanup_attachments(boolean(), Object.t()) :: {:ok, Oban.Job.t() | nil} def cleanup_attachments(true, %Object{} = object) do - AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{"object" => object}) + AttachmentsCleanupWorker.new(%{"op" => "cleanup_attachments", "object" => object}) + |> Oban.insert() end def cleanup_attachments(_, _), do: {:ok, nil} diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index b9d2a0188..30b3ba958 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -2,11 +2,13 @@ defmodule Pleroma.Search do alias Pleroma.Workers.SearchIndexingWorker def add_to_index(%Pleroma.Activity{id: activity_id}) do - SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id}) + SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => activity_id}) + |> Oban.insert() end def remove_from_index(%Pleroma.Object{id: object_id}) do - SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id}) + SearchIndexingWorker.new(%{"op" => "remove_from_index", "object" => object_id}) + |> Oban.insert() end def search(query, options) do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e28d76a7c..0e9d70831 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -736,7 +736,8 @@ defmodule Pleroma.User do end def force_password_reset_async(user) do - BackgroundWorker.enqueue("force_password_reset", %{"user_id" => user.id}) + BackgroundWorker.new(%{"op" => "force_password_reset", "user_id" => user.id}) + |> Oban.insert() end @spec force_password_reset(User.t()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} @@ -1218,7 +1219,8 @@ defmodule Pleroma.User do def update_and_set_cache(changeset) do with {:ok, user} <- Repo.update(changeset, stale_error_field: :id) do if get_change(changeset, :raw_fields) do - BackgroundWorker.enqueue("verify_fields_links", %{"user_id" => user.id}) + BackgroundWorker.new(%{"op" => "verify_fields_links", "user_id" => user.id}) + |> Oban.insert() end set_cache(user) @@ -1589,11 +1591,11 @@ defmodule Pleroma.User do )) || {:ok, nil} do if duration > 0 do - Pleroma.Workers.MuteExpireWorker.enqueue( - "unmute_user", - %{"muter_id" => muter.id, "mutee_id" => mutee.id}, + Pleroma.Workers.MuteExpireWorker.new( + %{"op" => "unmute_user", "muter_id" => muter.id, "mutee_id" => mutee.id}, scheduled_at: expires_at ) + |> Oban.insert() end @cachex.del(:user_cache, "muted_users_ap_ids:#{muter.ap_id}") @@ -1836,7 +1838,8 @@ defmodule Pleroma.User do defp maybe_filter_on_ap_id(query, _ap_ids), do: query def set_activation_async(user, status \\ true) do - BackgroundWorker.enqueue("user_activation", %{"user_id" => user.id, "status" => status}) + BackgroundWorker.new(%{"op" => "user_activation", "user_id" => user.id, "status" => status}) + |> Oban.insert() end @spec set_activation([User.t()], boolean()) :: {:ok, User.t()} | {:error, Ecto.Changeset.t()} @@ -1983,7 +1986,9 @@ defmodule Pleroma.User do def delete(%User{} = user) do # Purge the user immediately purge(user) - DeleteWorker.enqueue("delete_user", %{"user_id" => user.id}) + + DeleteWorker.new(%{"op" => "delete_user", "user_id" => user.id}) + |> Oban.insert() end # *Actually* delete the user from the DB diff --git a/lib/pleroma/user/import.ex b/lib/pleroma/user/import.ex index 53ffd1ab3..11905237c 100644 --- a/lib/pleroma/user/import.ex +++ b/lib/pleroma/user/import.ex @@ -63,23 +63,29 @@ defmodule Pleroma.User.Import do end def blocks_import(%User{} = blocker, [_ | _] = identifiers) do - BackgroundWorker.enqueue( - "blocks_import", - %{"user_id" => blocker.id, "identifiers" => identifiers} - ) + BackgroundWorker.new(%{ + "op" => "blocks_import", + "user_id" => blocker.id, + "identifiers" => identifiers + }) + |> Oban.insert() end def follow_import(%User{} = follower, [_ | _] = identifiers) do - BackgroundWorker.enqueue( - "follow_import", - %{"user_id" => follower.id, "identifiers" => identifiers} - ) + BackgroundWorker.new(%{ + "op" => "follow_import", + "user_id" => follower.id, + "identifiers" => identifiers + }) + |> Oban.insert() end def mutes_import(%User{} = user, [_ | _] = identifiers) do - BackgroundWorker.enqueue( - "mutes_import", - %{"user_id" => user.id, "identifiers" => identifiers} - ) + BackgroundWorker.new(%{ + "op" => "mutes_import", + "user_id" => user.id, + "identifiers" => identifiers + }) + |> Oban.insert() end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b30b0cabe..a2a94a0ff 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -222,10 +222,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do %{data: %{"expires_at" => %DateTime{} = expires_at}} = activity ) do with {:ok, _job} <- - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: activity.id, - expires_at: expires_at - }) do + Pleroma.Workers.PurgeExpiredActivity.enqueue( + %{ + activity_id: activity.id + }, + scheduled_at: expires_at + ) do {:ok, activity} end end @@ -446,10 +448,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do _ <- notify_and_stream(activity) do maybe_federate(activity) - BackgroundWorker.enqueue("move_following", %{ + BackgroundWorker.new(%{ + "op" => "move_following", "origin_id" => origin.id, "target_id" => target.id }) + |> Oban.insert() {:ok, activity} else @@ -1797,10 +1801,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # enqueue a task to fetch all pinned objects Enum.each(pins, fn {ap_id, _} -> if is_nil(Object.get_cached_by_ap_id(ap_id)) do - Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ + Pleroma.Workers.RemoteFetcherWorker.new(%{ + "op" => "fetch_remote", "id" => ap_id, "depth" => 1 }) + |> Oban.insert() end end) end diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e040753dc..f71652cb7 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -30,11 +30,11 @@ defmodule Pleroma.Web.ActivityPub.Publisher do """ @spec enqueue_one(map(), Keyword.t()) :: {:ok, %Oban.Job{}} def enqueue_one(%{} = params, worker_args \\ []) do - PublisherWorker.enqueue( - "publish_one", - %{"params" => params}, + PublisherWorker.new( + %{"op" => "publish_one", "params" => params}, worker_args ) + |> Oban.insert() end @doc """ diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index cc1c7a0af..d6d403671 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -223,10 +223,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do if Pleroma.Web.Federator.allowed_thread_distance?(reply_depth) and object.data["replies"] != nil do for reply_id <- object.data["replies"] do - Pleroma.Workers.RemoteFetcherWorker.enqueue("fetch_remote", %{ + Pleroma.Workers.RemoteFetcherWorker.new(%{ + "op" => "fetch_remote", "id" => reply_id, "depth" => reply_depth }) + |> Oban.insert() end end @@ -410,10 +412,12 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do {:ok, expires_at} = Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime.cast(meta[:expires_at]) - Pleroma.Workers.PurgeExpiredActivity.enqueue(%{ - activity_id: meta[:activity_id], - expires_at: expires_at - }) + Pleroma.Workers.PurgeExpiredActivity.enqueue( + %{ + activity_id: meta[:activity_id] + }, + scheduled_at: expires_at + ) end {:ok, object, meta} diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index b90b6a6d9..1ed905d6c 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -559,11 +559,11 @@ defmodule Pleroma.Web.CommonAPI do with {:ok, _} <- ThreadMute.add_mute(user.id, activity.data["context"]), _ <- Pleroma.Notification.mark_context_as_read(user, activity.data["context"]) do if expires_in > 0 do - Pleroma.Workers.MuteExpireWorker.enqueue( - "unmute_conversation", - %{"user_id" => user.id, "activity_id" => activity.id}, + Pleroma.Workers.MuteExpireWorker.new( + %{"op" => "unmute_conversation", "user_id" => user.id, "activity_id" => activity.id}, schedule_in: expires_in ) + |> Oban.insert() end {:ok, activity} diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 3d3101d61..c740fc85f 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -35,22 +35,30 @@ defmodule Pleroma.Web.Federator do end # Client API - def incoming_ap_doc(%{params: _params, req_headers: _req_headers} = args) do - job_args = Enum.into(args, %{}, fn {k, v} -> {Atom.to_string(k), v} end) - - ReceiverWorker.enqueue( - "incoming_ap_doc", - Map.put(job_args, "timeout", :timer.seconds(20)), + def incoming_ap_doc(%{params: params, req_headers: req_headers}) do + ReceiverWorker.new( + %{ + "op" => "incoming_ap_doc", + "req_headers" => req_headers, + "params" => params, + "timeout" => :timer.seconds(20) + }, priority: 2 ) + |> Oban.insert() end def incoming_ap_doc(%{"type" => "Delete"} = params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow) + ReceiverWorker.new(%{"op" => "incoming_ap_doc", "params" => params}, + priority: 3, + queue: :slow + ) + |> Oban.insert() end def incoming_ap_doc(params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) + ReceiverWorker.new(%{"op" => "incoming_ap_doc", "params" => params}) + |> Oban.insert() end @impl true @@ -60,9 +68,10 @@ defmodule Pleroma.Web.Federator do @impl true def publish(%Pleroma.Activity{data: %{"type" => type}} = activity) do - PublisherWorker.enqueue("publish", %{"activity_id" => activity.id}, + PublisherWorker.new(%{"op" => "publish", "activity_id" => activity.id}, priority: publish_priority(type) ) + |> Oban.insert() end defp publish_priority("Delete"), do: 3 diff --git a/lib/pleroma/web/o_auth/token.ex b/lib/pleroma/web/o_auth/token.ex index 9b1198b42..d96425094 100644 --- a/lib/pleroma/web/o_auth/token.ex +++ b/lib/pleroma/web/o_auth/token.ex @@ -100,11 +100,10 @@ defmodule Pleroma.Web.OAuth.Token do def create(%App{} = app, %User{} = user, attrs \\ %{}) do with {:ok, token} <- do_create(app, user, attrs) do if Pleroma.Config.get([:oauth2, :clean_expired_tokens]) do - Pleroma.Workers.PurgeExpiredToken.enqueue(%{ - token_id: token.id, - valid_until: DateTime.from_naive!(token.valid_until, "Etc/UTC"), - mod: __MODULE__ - }) + Pleroma.Workers.PurgeExpiredToken.new(%{token_id: token.id, mod: __MODULE__}, + scheduled_at: DateTime.from_naive!(token.valid_until, "Etc/UTC") + ) + |> Oban.insert() end {:ok, token} diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex index d4693f63e..d783f776a 100644 --- a/lib/pleroma/web/push.ex +++ b/lib/pleroma/web/push.ex @@ -28,6 +28,6 @@ defmodule Pleroma.Web.Push do @spec send(Pleroma.Notification.t()) :: {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset() | term()} def send(notification) do - WebPusherWorker.enqueue("web_push", %{"notification_id" => notification.id}) + WebPusherWorker.new(%{"op" => "web_push", "notification_id" => notification.id}) end end diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 0b570b70b..e2f92b1fd 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -8,9 +8,9 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo - use Pleroma.Workers.WorkerHelper, queue: "slow" + use Oban.Worker, queue: :slow - @impl Oban.Worker + @impl true def perform(%Job{ args: %{ "op" => "cleanup_attachments", @@ -31,7 +31,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do def perform(%Job{args: %{"op" => "cleanup_attachments", "object" => _object}}), do: {:ok, :skip} - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(900) defp do_clean({object_ids, attachment_urls}) do diff --git a/lib/pleroma/workers/background_worker.ex b/lib/pleroma/workers/background_worker.ex index 870aef3c6..60da2d5ca 100644 --- a/lib/pleroma/workers/background_worker.ex +++ b/lib/pleroma/workers/background_worker.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Workers.BackgroundWorker do alias Pleroma.User - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "user_activation", "user_id" => user_id, "status" => status}}) do user = User.get_cached_by_id(user_id) @@ -39,6 +39,6 @@ defmodule Pleroma.Workers.BackgroundWorker do User.perform(:verify_fields_links, user) end - @impl Oban.Worker - def timeout(_job), do: :timer.seconds(15) + @impl true + def timeout(_job), do: :timer.seconds(900) end diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index d1b6fcdad..6466d8d73 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.BackupWorker do alias Pleroma.Config.Getting, as: Config alias Pleroma.User.Backup - @impl Oban.Worker + @impl true def perform(%Job{ args: %{"op" => "process", "backup_id" => backup_id} }) do @@ -32,7 +32,7 @@ defmodule Pleroma.Workers.BackupWorker do end end - @impl Oban.Worker + @impl true def timeout(_job), do: Config.get([Backup, :timeout], :timer.minutes(30)) defp has_email?(user) do diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 17e92d10b..b50b52a7b 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do require Logger - @impl Oban.Worker + @impl true def perform(_job) do config = Config.get([:email_notifications, :digest]) @@ -59,6 +59,6 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do User.touch_last_digest_emailed_at(user) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 1f57aad4a..787649983 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,9 +9,9 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(_job) do if Pleroma.Config.get([Pleroma.Emails.NewUsersDigestEmail, :enabled]) do today = NaiveDateTime.utc_now() |> Timex.beginning_of_day() @@ -61,6 +61,6 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do :ok end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/delete_worker.ex b/lib/pleroma/workers/delete_worker.ex index 97003fb69..6a1c7bb38 100644 --- a/lib/pleroma/workers/delete_worker.ex +++ b/lib/pleroma/workers/delete_worker.ex @@ -6,10 +6,9 @@ defmodule Pleroma.Workers.DeleteWorker do alias Pleroma.Instances.Instance alias Pleroma.User - use Pleroma.Workers.WorkerHelper, queue: "slow" - - @impl Oban.Worker + use Oban.Worker, queue: :slow + @impl true def perform(%Job{args: %{"op" => "delete_user", "user_id" => user_id}}) do user = User.get_cached_by_id(user_id) User.perform(:delete, user) @@ -19,6 +18,6 @@ defmodule Pleroma.Workers.DeleteWorker do Instance.perform(:delete_instance, host) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(900) end diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 652bf77e0..b0259b191 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,9 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do encoded_email |> Base.decode64!() @@ -13,6 +13,6 @@ defmodule Pleroma.Workers.MailerWorker do |> Pleroma.Emails.Mailer.deliver(config) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index a7ab5883a..8356a775d 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -3,9 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MuteExpireWorker do - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do Pleroma.User.unmute(muter_id, mutee_id) :ok @@ -18,6 +18,6 @@ defmodule Pleroma.Workers.MuteExpireWorker do :ok end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index af8997e70..d263aa1b9 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -6,13 +6,13 @@ defmodule Pleroma.Workers.PollWorker do @moduledoc """ Generates notifications when a poll ends. """ - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background alias Pleroma.Activity alias Pleroma.Notification alias Pleroma.Object - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "poll_end", "activity_id" => activity_id}}) do with %Activity{} = activity <- find_poll_activity(activity_id), {:ok, notifications} <- Notification.create_poll_notifications(activity) do @@ -23,7 +23,7 @@ defmodule Pleroma.Workers.PollWorker do end end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) defp find_poll_activity(activity_id) do diff --git a/lib/pleroma/workers/publisher_worker.ex b/lib/pleroma/workers/publisher_worker.ex index 63fcf4ac2..7d9b022de 100644 --- a/lib/pleroma/workers/publisher_worker.ex +++ b/lib/pleroma/workers/publisher_worker.ex @@ -6,13 +6,9 @@ defmodule Pleroma.Workers.PublisherWorker do alias Pleroma.Activity alias Pleroma.Web.Federator - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" + use Oban.Worker, queue: :federator_outgoing, max_attempts: 5 - def backoff(%Job{attempt: attempt}) when is_integer(attempt) do - Pleroma.Workers.WorkerHelper.sidekiq_backoff(attempt, 5) - end - - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "publish", "activity_id" => activity_id}}) do activity = Activity.get_by_id(activity_id) Federator.perform(:publish, activity) @@ -23,6 +19,18 @@ defmodule Pleroma.Workers.PublisherWorker do Federator.perform(:publish_one, params) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(10) + + @base_backoff 15 + @pow 5 + @impl true + def backoff(%Job{attempt: attempt}) when is_integer(attempt) do + backoff = + :math.pow(attempt, @pow) + + @base_backoff + + :rand.uniform(2 * @base_backoff) * attempt + + trunc(backoff) + end end diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index f48e34042..f05e75f46 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -13,16 +13,13 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do alias Pleroma.Activity - @spec enqueue(map()) :: + @spec enqueue(map(), list()) :: {:ok, Oban.Job.t()} | {:error, :expired_activities_disabled} | {:error, :expiration_too_close} - def enqueue(args) do + def enqueue(params, worker_args) do with true <- enabled?() do - {scheduled_at, args} = Map.pop(args, :expires_at) - - args - |> new(scheduled_at: scheduled_at) + new(params, worker_args) |> Oban.insert() end end @@ -35,7 +32,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do end end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) defp enabled? do diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index 1f6931e4c..0405f6684 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do |> Repo.delete() end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) @spec get_expiration(pos_integer()) :: Job.t() | nil diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index 1854bf561..ff962f21b 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -9,16 +9,6 @@ defmodule Pleroma.Workers.PurgeExpiredToken do use Oban.Worker, queue: :background, max_attempts: 1 - @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: - {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} - def enqueue(args) do - {scheduled_at, args} = Map.pop(args, :valid_until) - - args - |> __MODULE__.new(scheduled_at: scheduled_at) - |> Oban.insert() - end - @impl true def perform(%Oban.Job{args: %{"token_id" => id, "mod" => module}}) do module @@ -27,6 +17,6 @@ defmodule Pleroma.Workers.PurgeExpiredToken do |> Pleroma.Repo.delete() end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index fd5c13fca..d4db97b63 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -7,9 +7,9 @@ defmodule Pleroma.Workers.ReceiverWorker do alias Pleroma.User alias Pleroma.Web.Federator - use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" + use Oban.Worker, queue: :federator_incoming, max_attempts: 5 - @impl Oban.Worker + @impl true def perform(%Job{ args: %{ @@ -51,7 +51,7 @@ defmodule Pleroma.Workers.ReceiverWorker do end end - @impl Oban.Worker + @impl true def timeout(%_{args: %{"timeout" => timeout}}), do: timeout def timeout(_job), do: :timer.seconds(5) diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index 60096e14b..e43765733 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do alias Pleroma.Object.Fetcher - use Pleroma.Workers.WorkerHelper, queue: "background" + use Oban.Worker, queue: :background - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do case Fetcher.fetch_object_from_id(id, depth: args["depth"]) do {:ok, _object} -> @@ -30,6 +30,6 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do end end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(15) end diff --git a/lib/pleroma/workers/rich_media_worker.ex b/lib/pleroma/workers/rich_media_worker.ex index 2ebf42d4f..d5ba7b63e 100644 --- a/lib/pleroma/workers/rich_media_worker.ex +++ b/lib/pleroma/workers/rich_media_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.RichMediaWorker do use Oban.Worker, queue: :background, max_attempts: 3, unique: [period: 300] - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "expire", "url" => url} = _args}) do Card.delete(url) end @@ -33,7 +33,7 @@ defmodule Pleroma.Workers.RichMediaWorker do # a slow/infinite data stream and insert a negative cache entry for the URL # We pad it by 2 seconds to be certain a slow connection is detected and we # can inject a negative cache entry for the URL - @impl Oban.Worker + @impl true def timeout(_job) do Config.get!([:rich_media, :timeout]) + :timer.seconds(2) end diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index ab62686f4..da386e0c3 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do The worker to post scheduled activity. """ - use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" + use Oban.Worker, queue: :federator_outgoing, max_attempts: 5 alias Pleroma.Repo alias Pleroma.ScheduledActivity @@ -15,7 +15,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do require Logger - @impl Oban.Worker + @impl true def perform(%Job{args: %{"activity_id" => activity_id}}) do with %ScheduledActivity{} = scheduled_activity <- find_scheduled_activity(activity_id), %User{} = user <- find_user(scheduled_activity.user_id) do @@ -37,7 +37,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do end end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) defp find_scheduled_activity(id) do diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex index 8969ae378..001f5254d 100644 --- a/lib/pleroma/workers/search_indexing_worker.ex +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -1,7 +1,7 @@ defmodule Pleroma.Workers.SearchIndexingWorker do - use Pleroma.Workers.WorkerHelper, queue: "search_indexing" + use Oban.Worker, queue: :search_indexing, max_attempts: 2 - @impl Oban.Worker + @impl true alias Pleroma.Config.Getting, as: Config @@ -21,6 +21,6 @@ defmodule Pleroma.Workers.SearchIndexingWorker do search_module.remove_from_index(object) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/user_refresh_worker.ex b/lib/pleroma/workers/user_refresh_worker.ex index fb90e9c9c..222a4a8f7 100644 --- a/lib/pleroma/workers/user_refresh_worker.ex +++ b/lib/pleroma/workers/user_refresh_worker.ex @@ -12,6 +12,6 @@ defmodule Pleroma.Workers.UserRefreshWorker do User.fetch_by_ap_id(ap_id) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(15) end diff --git a/lib/pleroma/workers/web_pusher_worker.ex b/lib/pleroma/workers/web_pusher_worker.ex index c549d3cd6..f4232d02a 100644 --- a/lib/pleroma/workers/web_pusher_worker.ex +++ b/lib/pleroma/workers/web_pusher_worker.ex @@ -7,9 +7,9 @@ defmodule Pleroma.Workers.WebPusherWorker do alias Pleroma.Repo alias Pleroma.Web.Push.Impl - use Pleroma.Workers.WorkerHelper, queue: "web_push" + use Oban.Worker, queue: :web_push - @impl Oban.Worker + @impl true def perform(%Job{args: %{"op" => "web_push", "notification_id" => notification_id}}) do notification = Notification @@ -20,6 +20,6 @@ defmodule Pleroma.Workers.WebPusherWorker do |> Enum.each(&Impl.deliver(&1)) end - @impl Oban.Worker + @impl true def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/worker_helper.ex b/lib/pleroma/workers/worker_helper.ex deleted file mode 100644 index 1d20cbd89..000000000 --- a/lib/pleroma/workers/worker_helper.ex +++ /dev/null @@ -1,48 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.WorkerHelper do - alias Pleroma.Config - alias Pleroma.Workers.WorkerHelper - - def worker_args(queue) do - case Config.get([:workers, :retries, queue]) do - nil -> [] - max_attempts -> [max_attempts: max_attempts] - end - end - - def sidekiq_backoff(attempt, pow \\ 4, base_backoff \\ 15) do - backoff = - :math.pow(attempt, pow) + - base_backoff + - :rand.uniform(2 * base_backoff) * attempt - - trunc(backoff) - end - - defmacro __using__(opts) do - caller_module = __CALLER__.module - queue = Keyword.fetch!(opts, :queue) - - quote do - # Note: `max_attempts` is intended to be overridden in `new/2` call - use Oban.Worker, - queue: unquote(queue), - max_attempts: 1 - - alias Oban.Job - - def enqueue(op, params, worker_args \\ []) do - params = Map.merge(%{"op" => op}, params) - queue_atom = String.to_atom(unquote(queue)) - worker_args = worker_args ++ WorkerHelper.worker_args(queue_atom) - - unquote(caller_module) - |> apply(:new, [params, worker_args]) - |> Oban.insert() - end - end - end -end -- cgit v1.2.3 From e6951e7e409471a5fa68bd2387a758d6abb1f728 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 31 Jul 2024 14:14:31 -0400 Subject: Fix User.disclose_client never working correctly Our test environment cheats by constructing a conn with a custom oauth_access/2 function. This assigns a :token to the conn but due to the way it is constructed it has the :user preloaded. When the OAuth Plug fetches a token it does not preload the user, so the check for user.disclose_client was always nil and assumed to be false. Preloading the :user ensures the test environment matches reality. --- lib/pleroma/web/plugs/o_auth_plug.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/plugs/o_auth_plug.ex b/lib/pleroma/web/plugs/o_auth_plug.ex index b59ac9d3e..488968691 100644 --- a/lib/pleroma/web/plugs/o_auth_plug.ex +++ b/lib/pleroma/web/plugs/o_auth_plug.ex @@ -52,7 +52,7 @@ defmodule Pleroma.Web.Plugs.OAuthPlug do where: t.token == ^token ) - with %Token{user_id: user_id} = token_record <- Repo.one(token_query), + with %Token{user_id: user_id} = token_record <- Repo.one(token_query) |> Repo.preload(:user), false <- is_nil(user_id), %User{} = user <- User.get_cached_by_id(user_id) do {:ok, user, token_record} -- cgit v1.2.3 From 1f986ec7138ce95a102a84c75a5b39dd885bf451 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 1 Aug 2024 22:02:04 -0400 Subject: Gun: Publisher job behavior improvement Gun's connection pool also returns an error if duplicate workers are launched simultaneously. Snooze on this error as well, and lower the snooze to 3 seconds with the optimism that the connection will still be open by then and the delivery can be completed quickly. The original setting of 30 seconds is pretty high and means there's an unnatural lag between deliveries of activities destined to the same server that were created at nearly the same time. This configuration should be more efficient. --- lib/pleroma/web/activity_pub/publisher.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e040753dc..e63b8ff1f 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -143,9 +143,13 @@ defmodule Pleroma.Web.ActivityPub.Publisher do _ -> {:error, e} end + {:error, {:already_started, _}} -> + Logger.debug("Publisher snoozing worker job due worker :already_started race condition") + connection_pool_snooze() + {:error, :pool_full} -> Logger.debug("Publisher snoozing worker job due to full connection pool") - {:snooze, 30} + connection_pool_snooze() e -> unless params[:unreachable_since], do: Instances.set_unreachable(inbox) @@ -155,6 +159,8 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end + defp connection_pool_snooze, do: {:snooze, 3} + defp signature_host(%URI{port: port, scheme: scheme, host: host}) do if port == URI.default_port(scheme) do host -- cgit v1.2.3 From de9194893e811eaa5e14e07ae208da84006b6f38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sat, 3 Aug 2024 18:28:43 +0200 Subject: Support `id` param in `GET /api/v1/statuses` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec/operations/status_operation.ex | 8 +++++++- lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 3 ++- 2 files changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 1717c68c8..ef828feee 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -31,11 +31,17 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do security: [%{"oAuth" => ["read:statuses"]}], parameters: [ Operation.parameter( - :ids, + :id, :query, %Schema{type: :array, items: FlakeID}, "Array of status IDs" ), + Operation.parameter( + :ids, + :query, + %Schema{type: :array, items: FlakeID}, + "Deprecated, use `id` instead" + ), Operation.parameter( :with_muted, :query, diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index b9b236920..d5aef5ad2 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -111,10 +111,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do `ids` query param is required """ def index( - %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{ids: ids} = params}}} = + %{assigns: %{user: user}, private: %{open_api_spex: %{params: params}}} = conn, _ ) do + ids = Map.get(params, :id, Map.get(params, :ids)) limit = 100 activities = -- cgit v1.2.3 From 52f7033f7ac82155fc927f4b0a3f4f9e8ae11114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Sun, 4 Aug 2024 16:02:44 +0200 Subject: StreamerView: Do not leak follows count if hidden MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/views/streamer_view.ex | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index f97570b0a..6016d821b 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -109,7 +109,11 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("follow_relationships_update.json", item, topic) do + def render( + "follow_relationships_update.json", + %{follower: follower, following: following} = item, + topic + ) do %{ stream: render("stream.json", %{topic: topic}), event: "pleroma:follow_relationships_update", @@ -117,14 +121,22 @@ defmodule Pleroma.Web.StreamerView do %{ state: item.state, follower: %{ - id: item.follower.id, - follower_count: item.follower.follower_count, - following_count: item.follower.following_count + id: follower.id, + follower_count: follower.follower_count, + following_count: follower.following_count }, following: %{ - id: item.following.id, - follower_count: item.following.follower_count, - following_count: item.following.following_count + id: following.id, + follower_count: + if(!following.hide_followers_count or !following.hide_followers, + do: following.follower_count, + else: 0 + ), + following_count: + if(!following.hide_follows_count or !following.hide_follows, + do: following.following_count, + else: 0 + ) } } |> Jason.encode!() -- cgit v1.2.3 From f8bdcaa161575e40097a82481009620edc5a0696 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Aug 2024 11:15:35 -0400 Subject: Split Federator.publish_one/1 into a second function called prepare_one/1 --- lib/pleroma/web/activity_pub/publisher.ex | 52 +++++++++++++++++++++++++------ lib/pleroma/web/federator.ex | 5 ++- 2 files changed, 47 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index e63b8ff1f..2d2c09f1c 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -76,14 +76,12 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end @doc """ - Publish a single message to a peer. Takes a struct with the following - parameters set: - + Prepare an activity for publishing from an Oban job * `inbox`: the inbox to publish to * `activity_id`: the internal activity id * `cc`: the cc recipients relevant to this inbox (optional) """ - def publish_one(%{inbox: inbox, activity_id: activity_id} = params) do + def prepare_one(%{inbox: inbox, activity_id: activity_id} = params) do activity = Activity.get_by_id_with_user_actor(activity_id) actor = activity.user_actor @@ -113,6 +111,38 @@ defmodule Pleroma.Web.ActivityPub.Publisher do date: date }) + %{ + activity_id: activity_id, + json: json, + date: date, + signature: signature, + digest: digest, + inbox: inbox, + unreachable_since: params[:unreachable_since] + } + end + + @doc """ + Publish a single message to a peer. Takes a struct with the following + parameters set: + * `activity_id`: the activity id + * `json`: the json payload + * `date`: the signed date from Pleroma.Signature.signed_date() + * `signature`: the signature from Pleroma.Signature.sign/2 + * `digest`: base64 encoded the hash of the json payload prefixed with "SHA-256=" + * `inbox`: the inbox URI of this delivery + * `unreachable_since`: timestamp the instance was marked unreachable + + """ + def publish_one(%{ + activity_id: activity_id, + json: json, + date: date, + signature: signature, + digest: digest, + inbox: inbox, + unreachable_since: unreachable_since + }) do with {:ok, %{status: code}} = result when code in 200..299 <- HTTP.post( inbox, @@ -124,14 +154,12 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {"digest", digest} ] ) do - if not Map.has_key?(params, :unreachable_since) || params[:unreachable_since] do - Instances.set_reachable(inbox) - end + maybe_set_reachable(unreachable_since, inbox) result else {_post_result, %{status: code} = response} = e -> - unless params[:unreachable_since], do: Instances.set_unreachable(inbox) + maybe_set_unreachable(unreachable_since, inbox) Logger.metadata(activity: activity_id, inbox: inbox, status: code) Logger.error("Publisher failed to inbox #{inbox} with status #{code}") @@ -152,7 +180,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do connection_pool_snooze() e -> - unless params[:unreachable_since], do: Instances.set_unreachable(inbox) + maybe_set_unreachable(unreachable_since, inbox) Logger.metadata(activity: activity_id, inbox: inbox) Logger.error("Publisher failed to inbox #{inbox} #{inspect(e)}") {:error, e} @@ -161,6 +189,12 @@ defmodule Pleroma.Web.ActivityPub.Publisher do defp connection_pool_snooze, do: {:snooze, 3} + defp maybe_set_reachable(%NaiveDateTime{}, inbox), do: Instances.set_reachable(inbox) + defp maybe_set_reachable(_, _), do: :ok + + defp maybe_set_unreachable(nil, inbox), do: Instances.set_unreachable(inbox) + defp maybe_set_unreachable(%NaiveDateTime{}, _), do: :ok + defp signature_host(%URI{port: port, scheme: scheme, host: host}) do if port == URI.default_port(scheme) do host diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 3d3101d61..242cf4bfd 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -71,7 +71,10 @@ defmodule Pleroma.Web.Federator do # Job Worker Callbacks @spec perform(atom(), any()) :: {:ok, any()} | {:error, any()} - def perform(:publish_one, params), do: Publisher.publish_one(params) + def perform(:publish_one, params) do + Publisher.prepare_one(params) + |> Publisher.publish_one() + end def perform(:publish, activity) do Logger.debug(fn -> "Running publish for #{activity.data["id"]}" end) -- cgit v1.2.3 From 83fcf42c709c390888f13a37f2d381b071a65231 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Aug 2024 12:00:50 -0400 Subject: Force cc to an empty list if undefined --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 2d2c09f1c..d06f49f40 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -91,7 +91,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - cc = Map.get(params, :cc) + cc = Map.get(params, :cc, []) json = data -- cgit v1.2.3 From 9ae9e2fc5ce61200712ea02201c2ba87b174f06a Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Aug 2024 12:16:06 -0400 Subject: Use a struct to hold the prepared data passed to publish_one/1 --- lib/pleroma/web/activity_pub/publisher.ex | 38 +++++++++------------- lib/pleroma/web/activity_pub/publisher/prepared.ex | 8 +++++ 2 files changed, 24 insertions(+), 22 deletions(-) create mode 100644 lib/pleroma/web/activity_pub/publisher/prepared.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index d06f49f40..7d105ea4f 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Web.ActivityPub.Publisher.Prepared alias Pleroma.Web.ActivityPub.Relay alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Workers.PublisherWorker @@ -81,6 +82,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do * `activity_id`: the internal activity id * `cc`: the cc recipients relevant to this inbox (optional) """ + @spec prepare_one(map()) :: Prepared.t() def prepare_one(%{inbox: inbox, activity_id: activity_id} = params) do activity = Activity.get_by_id_with_user_actor(activity_id) actor = activity.user_actor @@ -111,7 +113,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do date: date }) - %{ + %Prepared{ activity_id: activity_id, json: json, date: date, @@ -134,34 +136,26 @@ defmodule Pleroma.Web.ActivityPub.Publisher do * `unreachable_since`: timestamp the instance was marked unreachable """ - def publish_one(%{ - activity_id: activity_id, - json: json, - date: date, - signature: signature, - digest: digest, - inbox: inbox, - unreachable_since: unreachable_since - }) do + def publish_one(p = %Prepared{}) do with {:ok, %{status: code}} = result when code in 200..299 <- HTTP.post( - inbox, - json, + p.inbox, + p.json, [ {"Content-Type", "application/activity+json"}, - {"Date", date}, - {"signature", signature}, - {"digest", digest} + {"Date", p.date}, + {"signature", p.signature}, + {"digest", p.digest} ] ) do - maybe_set_reachable(unreachable_since, inbox) + maybe_set_reachable(p.unreachable_since, p.inbox) result else {_post_result, %{status: code} = response} = e -> - maybe_set_unreachable(unreachable_since, inbox) - Logger.metadata(activity: activity_id, inbox: inbox, status: code) - Logger.error("Publisher failed to inbox #{inbox} with status #{code}") + maybe_set_unreachable(p.unreachable_since, p.inbox) + Logger.metadata(activity: p.activity_id, inbox: p.inbox, status: code) + Logger.error("Publisher failed to inbox #{p.inbox} with status #{code}") case response do %{status: 400} -> {:cancel, :bad_request} @@ -180,9 +174,9 @@ defmodule Pleroma.Web.ActivityPub.Publisher do connection_pool_snooze() e -> - maybe_set_unreachable(unreachable_since, inbox) - Logger.metadata(activity: activity_id, inbox: inbox) - Logger.error("Publisher failed to inbox #{inbox} #{inspect(e)}") + maybe_set_unreachable(p.unreachable_since, p.inbox) + Logger.metadata(activity: p.activity_id, inbox: p.inbox) + Logger.error("Publisher failed to inbox #{p.inbox} #{inspect(e)}") {:error, e} end end diff --git a/lib/pleroma/web/activity_pub/publisher/prepared.ex b/lib/pleroma/web/activity_pub/publisher/prepared.ex new file mode 100644 index 000000000..ddd8167e1 --- /dev/null +++ b/lib/pleroma/web/activity_pub/publisher/prepared.ex @@ -0,0 +1,8 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.Publisher.Prepared do + @type t :: %__MODULE__{} + defstruct [:activity_id, :json, :date, :signature, :digest, :inbox, :unreachable_since] +end -- cgit v1.2.3 From 0bfe5920486cd84fd17ea405fb49a6a40ca5bc2c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 6 Aug 2024 12:48:36 -0400 Subject: Credo --- lib/pleroma/web/activity_pub/publisher.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 7d105ea4f..e31dec059 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -136,7 +136,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do * `unreachable_since`: timestamp the instance was marked unreachable """ - def publish_one(p = %Prepared{}) do + def publish_one(%Prepared{} = p) do with {:ok, %{status: code}} = result when code in 200..299 <- HTTP.post( p.inbox, -- cgit v1.2.3 From c284c4e3e697dda5c5965588667317090a51bfca Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 09:19:33 -0400 Subject: Extract the logic from the map --- lib/pleroma/web/views/streamer_view.ex | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 6016d821b..8e5a9f2a4 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -114,6 +114,20 @@ defmodule Pleroma.Web.StreamerView do %{follower: follower, following: following} = item, topic ) do + follower_count = + if Enum.any?([following.hide_followers_count, following.hide_followers]) do + 0 + else + following.follower_count + end + + following_count = + if Enum.any?([following.hide_follows_count, following.hide_follows]) do + 0 + else + following.following_count + end + %{ stream: render("stream.json", %{topic: topic}), event: "pleroma:follow_relationships_update", @@ -127,16 +141,8 @@ defmodule Pleroma.Web.StreamerView do }, following: %{ id: following.id, - follower_count: - if(!following.hide_followers_count or !following.hide_followers, - do: following.follower_count, - else: 0 - ), - following_count: - if(!following.hide_follows_count or !following.hide_follows, - do: following.following_count, - else: 0 - ) + follower_count: follower_count, + following_count: following_count } } |> Jason.encode!() -- cgit v1.2.3 From 7d33b53908242bf420d9f84550cabb5f86bc4738 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 11:47:43 -0400 Subject: Improve the variable naming --- lib/pleroma/web/views/streamer_view.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 8e5a9f2a4..079a37351 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -114,14 +114,14 @@ defmodule Pleroma.Web.StreamerView do %{follower: follower, following: following} = item, topic ) do - follower_count = + following_follower_count = if Enum.any?([following.hide_followers_count, following.hide_followers]) do 0 else following.follower_count end - following_count = + following_following_count = if Enum.any?([following.hide_follows_count, following.hide_follows]) do 0 else @@ -141,8 +141,8 @@ defmodule Pleroma.Web.StreamerView do }, following: %{ id: following.id, - follower_count: follower_count, - following_count: following_count + follower_count: following_follower_count, + following_count: following_following_count } } |> Jason.encode!() -- cgit v1.2.3 From 06e8ece4cc3956d991d48dbd338604b7940d167c Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 12:37:58 -0400 Subject: Fix CommonAPI.follow/2 which returned users in the reverse order they were provided to the function --- lib/pleroma/web/common_api.ex | 2 +- lib/pleroma/web/mastodon_api/mastodon_api.ex | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 1ed905d6c..921e414c3 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -130,7 +130,7 @@ defmodule Pleroma.Web.CommonAPI do if activity.data["state"] == "reject" do {:error, :rejected} else - {:ok, follower, followed, activity} + {:ok, followed, follower, activity} end end end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 6dcbfb097..c9e045d23 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -18,10 +18,10 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do if not User.following?(follower, followed) do CommonAPI.follow(followed, follower) else - {:ok, follower, followed, nil} + {:ok, followed, follower, nil} end - with {:ok, follower, _followed, _} <- result do + with {:ok, _followed, follower, _} <- result do options = cast_params(params) set_reblogs_visibility(options[:reblogs], result) set_subscription(options[:notify], result) @@ -29,19 +29,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do end end - defp set_reblogs_visibility(false, {:ok, follower, followed, _}) do + defp set_reblogs_visibility(false, {:ok, followed, follower, _}) do CommonAPI.hide_reblogs(followed, follower) end - defp set_reblogs_visibility(_, {:ok, follower, followed, _}) do + defp set_reblogs_visibility(_, {:ok, followed, follower, _}) do CommonAPI.show_reblogs(followed, follower) end - defp set_subscription(true, {:ok, follower, followed, _}) do + defp set_subscription(true, {:ok, followed, follower, _}) do User.subscribe(follower, followed) end - defp set_subscription(false, {:ok, follower, followed, _}) do + defp set_subscription(false, {:ok, followed, follower, _}) do User.unsubscribe(follower, followed) end -- cgit v1.2.3 From 721005b3126e0920ab861fcc83195f160b0ee9a0 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 13:00:31 -0400 Subject: Fix WebPush notifications not generating jobs Dialyzer pointed this one out. The WorkerHelper removal in !4166 was missing this Oban.insert() and tests were not noticing any problems because we mocked the Push.send function instead of executing it and checking for the Oban job. --- lib/pleroma/web/push.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex index d783f776a..6d777142e 100644 --- a/lib/pleroma/web/push.ex +++ b/lib/pleroma/web/push.ex @@ -29,5 +29,6 @@ defmodule Pleroma.Web.Push do {:ok, Oban.Job.t()} | {:error, Oban.Job.changeset() | term()} def send(notification) do WebPusherWorker.new(%{"op" => "web_push", "notification_id" => notification.id}) + |> Oban.insert() end end -- cgit v1.2.3 From 8f15000c0f4c81b27b3a7077092f1ba1b37da205 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 14:38:29 -0400 Subject: Do not require a cc field when validating an incoming Follow activity The cc field is not required and the code was keeping the cc if it existed on an activity or replacing it with the default of an empty list when casting. If any Follow activity was received with a cc field, it would attempt to keep it. This was noticed in !4208 where we would craft Follow requests with a cc value of nil. --- lib/pleroma/web/activity_pub/object_validators/follow_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex index b3ca5b691..e4e97bf72 100644 --- a/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/follow_validator.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.FollowValidator do defp validate_data(cng) do cng - |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_required([:id, :type, :actor, :to, :object]) |> validate_inclusion(:type, ["Follow"]) |> validate_inclusion(:state, ~w{pending reject accept}) |> validate_actor_presence() -- cgit v1.2.3 From 526a57ff9fe2be729c6e4d95f3eb5991047c41a2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 21:51:22 -0400 Subject: Remove validation for cc fields on Follow Accept/Reject --- .../web/activity_pub/object_validators/accept_reject_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex index d611da051..03ab83347 100644 --- a/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/accept_reject_validator.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator do defp validate_data(cng) do cng - |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_required([:id, :type, :actor, :to, :object]) |> validate_inclusion(:type, ["Accept", "Reject"]) |> validate_actor_presence() |> validate_object_presence(allowed_types: ["Follow"]) -- cgit v1.2.3 From ca934b744f9c6095315421990975c1e9a7b9cd85 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 7 Aug 2024 21:51:43 -0400 Subject: Remove validation for cc fields on Blocks --- lib/pleroma/web/activity_pub/object_validators/block_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex index 0de87a27e..98340545c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/block_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/block_validator.ex @@ -29,7 +29,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator do defp validate_data(cng) do cng - |> validate_required([:id, :type, :actor, :to, :cc, :object]) + |> validate_required([:id, :type, :actor, :to, :object]) |> validate_inclusion(:type, ["Block"]) |> CommonValidations.validate_actor_presence() |> CommonValidations.validate_actor_presence(field_name: :object) -- cgit v1.2.3 From a0af6cba09f6c85f524d31f7d1a68948b77d9e53 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Aug 2024 14:12:44 -0400 Subject: Added MRF.QuietReply which prevents replies to public posts from being published to the timelines --- lib/pleroma/web/activity_pub/mrf/quiet_reply.ex | 55 +++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/quiet_reply.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex new file mode 100644 index 000000000..8a9b2beb8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -0,0 +1,55 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do + require Pleroma.Constants + + alias Pleroma.User + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + @impl true + def history_awareness, do: :auto + + @impl true + def filter( + %{ + "type" => "Create", + "object" => %{ + "actor" => actor, + "type" => "Note", + "to" => to, + "cc" => cc, + "inReplyTo" => in_reply_to + } + } = object + ) do + with true <- is_binary(in_reply_to), + false <- match?([], cc), + %User{follower_address: followers_collection, local: true} <- + User.get_by_ap_id(actor) do + updated_to = + to + |> Kernel.++([followers_collection]) + |> Kernel.--([Pleroma.Constants.as_public()]) + + updated_cc = [Pleroma.Constants.as_public()] + + updated_object = + object + |> put_in(["object", "to"], updated_to) + |> put_in(["object", "cc"], updated_cc) + + {:ok, updated_object} + else + _ -> {:ok, object} + end + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe, do: {:ok, %{}} +end -- cgit v1.2.3 From 5a134a46f7a59eba131c9e484d49e09394e341a8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 8 Aug 2024 15:23:59 -0400 Subject: We must change to/cc in the activity and inner object --- lib/pleroma/web/activity_pub/mrf/quiet_reply.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index 8a9b2beb8..52d92f267 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -16,11 +16,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do def filter( %{ "type" => "Create", + "to" => to, + "cc" => cc, "object" => %{ "actor" => actor, "type" => "Note", - "to" => to, - "cc" => cc, "inReplyTo" => in_reply_to } } = object @@ -38,6 +38,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do updated_object = object + |> Map.put("to", updated_to) + |> Map.put("cc", updated_cc) |> put_in(["object", "to"], updated_to) |> put_in(["object", "cc"], updated_cc) -- cgit v1.2.3 From 6e53e94bdab91044ffa24e9d6585fdde1727b0f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 12 Aug 2024 12:23:38 +0200 Subject: Remove stub for /api/v1/accounts/:id/identity_proofs (deprecated by Mastodon 3.5.0) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec/operations/account_operation.ex | 16 ---------------- .../web/mastodon_api/controllers/account_controller.ex | 5 +---- lib/pleroma/web/router.ex | 1 - 3 files changed, 1 insertion(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 85f02166f..d9614bc48 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -498,22 +498,6 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end - def identity_proofs_operation do - %Operation{ - tags: ["Retrieve account information"], - summary: "Identity proofs", - operationId: "AccountController.identity_proofs", - # Validators complains about unused path params otherwise - parameters: [ - %Reference{"$ref": "#/components/parameters/accountIdOrNickname"} - ], - description: "Not implemented", - responses: %{ - 200 => empty_array_response() - } - } - end - def familiar_followers_operation do %Operation{ tags: ["Retrieve account information"], diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 80ab95a57..6a6911727 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -51,7 +51,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do plug( OAuthScopesPlug, %{scopes: ["read:accounts"]} - when action in [:verify_credentials, :endorsements, :identity_proofs] + when action in [:verify_credentials, :endorsements] ) plug( @@ -660,7 +660,4 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do defp get_familiar_followers(user, current_user) do User.get_familiar_followers(user, current_user) end - - @doc "GET /api/v1/identity_proofs" - def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params) end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index fc40a1143..6492e3861 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -648,7 +648,6 @@ defmodule Pleroma.Web.Router do get("/accounts/relationships", AccountController, :relationships) get("/accounts/familiar_followers", AccountController, :familiar_followers) get("/accounts/:id/lists", AccountController, :lists) - get("/accounts/:id/identity_proofs", AccountController, :identity_proofs) get("/endorsements", AccountController, :endorsements) get("/blocks", AccountController, :blocks) get("/mutes", AccountController, :mutes) -- cgit v1.2.3 From c1c0f0b1443aa486f7867b9fa02cc41dbbc74261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 12 Aug 2024 12:33:20 +0200 Subject: Mark `/api/v1/pleroma/accounts/:id/subscribe`/`unsubscribe` as deprecated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex index 7340653fb..b8b37d7cf 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_account_operation.ex @@ -85,9 +85,11 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do def subscribe_operation do %Operation{ + deprecated: true, tags: ["Account actions"], summary: "Subscribe", - description: "Receive notifications for all statuses posted by the account.", + description: + "Receive notifications for all statuses posted by the account. Deprecated, use `notify: true` in follow operation instead.", operationId: "PleromaAPI.AccountController.subscribe", parameters: [id_param()], security: [%{"oAuth" => ["follow", "write:follows"]}], @@ -100,9 +102,11 @@ defmodule Pleroma.Web.ApiSpec.PleromaAccountOperation do def unsubscribe_operation do %Operation{ + deprecated: true, tags: ["Account actions"], summary: "Unsubscribe", - description: "Stop receiving notifications for all statuses posted by the account.", + description: + "Stop receiving notifications for all statuses posted by the account. Deprecated, use `notify: false` in follow operation instead.", operationId: "PleromaAPI.AccountController.unsubscribe", parameters: [id_param()], security: [%{"oAuth" => ["follow", "write:follows"]}], -- cgit v1.2.3 From 34715b8581daca64b474438b590730ee53e86017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Mon, 12 Aug 2024 13:38:33 +0200 Subject: remove unused alias MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/controllers/account_controller.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 6a6911727..54d46c86b 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -22,7 +22,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do alias Pleroma.Web.CommonAPI alias Pleroma.Web.MastodonAPI.ListView alias Pleroma.Web.MastodonAPI.MastodonAPI - alias Pleroma.Web.MastodonAPI.MastodonAPIController alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.OAuth.OAuthController alias Pleroma.Web.Plugs.OAuthScopesPlug -- cgit v1.2.3 From 471f5c81fd279ccbc0cee7196573485b8608786b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 12 Aug 2024 16:07:45 -0400 Subject: Add module documentation --- lib/pleroma/web/activity_pub/mrf/quiet_reply.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index 52d92f267..ae5e2cdc7 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -3,6 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do + @moduledoc """ + QuietReply alters the scope of activities from local users when replying by enforcing them to be "Unlisted" or "Quiet Public". This delivers the activity to all the expected recipients and instances, but it will not be published in the Federated / The Whole Known Network timelines. It will still be published to the Home timelines of the user's followers and visible to anyone who opens the thread. + """ require Pleroma.Constants alias Pleroma.User -- cgit v1.2.3 From 488c4b8b983f99d036e3f3bf67dec782f9959319 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 15 Mar 2024 15:10:45 -0400 Subject: MRF.FODirectReply Force replies to followers-only posts to always be direct --- .../web/activity_pub/mrf/fo_direct_reply.ex | 65 ++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex b/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex new file mode 100644 index 000000000..4eb97afa8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex @@ -0,0 +1,65 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do + @moduledoc """ + FODirectReply alters the scope of replies to activities which are Followers Only to be Direct. The purpose of this policy is to prevent broken threads for followers of the reply author because their response was to a user that they are not also following. + """ + + alias Pleroma.User + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + @impl true + def filter( + %{ + "type" => "Create", + "to" => to, + "object" => %{ + "actor" => actor, + "type" => "Note", + "inReplyTo" => in_reply_to + } + } = activity + ) do + with true <- is_binary(in_reply_to), + %User{follower_address: followers_collection, local: true} <- User.get_by_ap_id(actor), + true <- followers_only?(in_reply_to) do + direct_to = to -- [followers_collection] + + updated_activity = + activity + |> Map.put("cc", []) + |> Map.put("to", direct_to) + |> Map.put("directMessage", true) + |> put_in(["object", "cc"], []) + |> put_in(["object", "to"], direct_to) + + {:ok, updated_activity} + else + _ -> {:ok, activity} + end + end + + @impl true + def filter(activity), do: {:ok, activity} + + @impl true + def describe, do: {:ok, %{}} + + defp followers_only?(parent_ap_id) do + with %Pleroma.Object{} = object <- Pleroma.Object.get_by_ap_id(parent_ap_id), + object_data <- Map.get(object, :data), + %Pleroma.User{} = user <- User.get_cached_by_ap_id(object_data["actor"]) do + if user.follower_address in object_data["to"] do + true + else + false + end + else + _ -> + false + end + end +end -- cgit v1.2.3 From b0c64945c2cfd622b9f2c68d594bda4fd4c1b9eb Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 12 Aug 2024 20:17:35 -0400 Subject: MRF.FODirectReply: use Visibility module to verify the scope --- lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex b/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex index 4eb97afa8..2cf22745a 100644 --- a/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/fo_direct_reply.ex @@ -7,7 +7,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do FODirectReply alters the scope of replies to activities which are Followers Only to be Direct. The purpose of this policy is to prevent broken threads for followers of the reply author because their response was to a user that they are not also following. """ + alias Pleroma.Object alias Pleroma.User + alias Pleroma.Web.ActivityPub.Visibility @behaviour Pleroma.Web.ActivityPub.MRF.Policy @@ -25,7 +27,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do ) do with true <- is_binary(in_reply_to), %User{follower_address: followers_collection, local: true} <- User.get_by_ap_id(actor), - true <- followers_only?(in_reply_to) do + %Object{} = in_reply_to_object <- Object.get_by_ap_id(in_reply_to), + "private" <- Visibility.get_visibility(in_reply_to_object) do direct_to = to -- [followers_collection] updated_activity = @@ -47,19 +50,4 @@ defmodule Pleroma.Web.ActivityPub.MRF.FODirectReply do @impl true def describe, do: {:ok, %{}} - - defp followers_only?(parent_ap_id) do - with %Pleroma.Object{} = object <- Pleroma.Object.get_by_ap_id(parent_ap_id), - object_data <- Map.get(object, :data), - %Pleroma.User{} = user <- User.get_cached_by_ap_id(object_data["actor"]) do - if user.follower_address in object_data["to"] do - true - else - false - end - else - _ -> - false - end - end end -- cgit v1.2.3 From 2ba5ad8eb51c9a87da3062328468ea0824d41cc9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 13 Aug 2024 13:59:07 -0400 Subject: MRF cleanup MRFs written over time have been inconsistent with the terminology of what is being processed. MRFs work on Activities, but generally we always named the assignments "message" or "object" which is really confusing when trying to debug or write tests. --- .../web/activity_pub/mrf/anti_followbot_policy.ex | 8 +- .../web/activity_pub/mrf/anti_link_spam_policy.ex | 10 +- .../activity_pub/mrf/anti_mention_spam_policy.ex | 8 +- lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex | 14 +- lib/pleroma/web/activity_pub/mrf/drop_policy.ex | 6 +- lib/pleroma/web/activity_pub/mrf/emoji_policy.ex | 30 ++--- .../web/activity_pub/mrf/ensure_re_prepended.ex | 14 +- .../web/activity_pub/mrf/follow_bot_policy.ex | 20 +-- .../activity_pub/mrf/force_bot_unlisted_policy.ex | 12 +- lib/pleroma/web/activity_pub/mrf/force_mention.ex | 2 +- .../activity_pub/mrf/force_mentions_in_content.ex | 12 +- lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex | 51 +++---- .../web/activity_pub/mrf/hellthread_policy.ex | 54 ++++---- .../web/activity_pub/mrf/inline_quote_policy.ex | 4 +- lib/pleroma/web/activity_pub/mrf/keyword_policy.ex | 48 +++---- .../activity_pub/mrf/media_proxy_warming_policy.ex | 10 +- lib/pleroma/web/activity_pub/mrf/mention_policy.ex | 12 +- .../web/activity_pub/mrf/no_empty_policy.ex | 14 +- lib/pleroma/web/activity_pub/mrf/no_op_policy.ex | 4 +- .../activity_pub/mrf/no_placeholder_text_policy.ex | 8 +- .../web/activity_pub/mrf/normalize_markup.ex | 10 +- .../web/activity_pub/mrf/nsfw_api_policy.ex | 48 +++---- .../web/activity_pub/mrf/object_age_policy.ex | 58 ++++---- lib/pleroma/web/activity_pub/mrf/policy.ex | 2 +- lib/pleroma/web/activity_pub/mrf/quiet_reply.ex | 12 +- .../activity_pub/mrf/quote_to_link_tag_policy.ex | 10 +- lib/pleroma/web/activity_pub/mrf/simple_policy.ex | 150 ++++++++++----------- .../web/activity_pub/mrf/steal_emoji_policy.ex | 6 +- .../web/activity_pub/mrf/subchain_policy.ex | 10 +- lib/pleroma/web/activity_pub/mrf/tag_policy.ex | 62 ++++----- .../web/activity_pub/mrf/user_allow_list_policy.ex | 12 +- .../web/activity_pub/mrf/vocabulary_policy.ex | 30 ++--- 32 files changed, 376 insertions(+), 375 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex index df4ba819c..8ea61aec2 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_followbot_policy.ex @@ -63,20 +63,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiFollowbotPolicy do end @impl true - def filter(%{"type" => "Follow", "actor" => actor_id} = message) do + def filter(%{"type" => "Follow", "actor" => actor_id} = activity) do %User{} = actor = normalize_by_ap_id(actor_id) score = determine_if_followbot(actor) - if score < 0.8 || bot_allowed?(message, actor) do - {:ok, message} + if score < 0.8 || bot_allowed?(activity, actor) do + {:ok, activity} else {:reject, "[AntiFollowbotPolicy] Scored #{actor_id} as #{score}"} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex index 3ec9c52ee..2be6d8df4 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_link_spam_policy.ex @@ -29,17 +29,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do defp contains_links?(_), do: false @impl true - def filter(%{"type" => "Create", "actor" => actor, "object" => object} = message) do + def filter(%{"type" => "Create", "actor" => actor, "object" => object} = activity) do with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), {:contains_links, true} <- {:contains_links, contains_links?(object)}, {:old_user, true} <- {:old_user, old_user?(u)} do - {:ok, message} + {:ok, activity} else {:ok, %User{local: true}} -> - {:ok, message} + {:ok, activity} {:contains_links, false} -> - {:ok, message} + {:ok, activity} {:old_user, false} -> {:reject, "[AntiLinkSpamPolicy] User has no posts nor followers"} @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiLinkSpamPolicy do end # in all other cases, pass through - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex index 531e75ce8..1d76a307b 100644 --- a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -22,11 +22,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do end # copied from HellthreadPolicy - defp get_recipient_count(message) do - recipients = (message["to"] || []) ++ (message["cc"] || []) + defp get_recipient_count(activity) do + recipients = (activity["to"] || []) ++ (activity["cc"] || []) follower_collection = - User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address + User.get_cached_by_ap_id(activity["actor"] || activity["attributedTo"]).follower_address if Enum.member?(recipients, Pleroma.Constants.as_public()) do recipients = @@ -80,7 +80,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do end # in all other cases, pass through - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex index 7c6bb888f..ca41c464c 100644 --- a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex @@ -38,18 +38,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do @query_timeout 500 @impl true - def filter(%{"actor" => actor} = object) do + def filter(%{"actor" => actor} = activity) do actor_info = URI.parse(actor) - with {:ok, object} <- check_rbl(actor_info, object) do - {:ok, object} + with {:ok, activity} <- check_rbl(actor_info, activity) do + {:ok, activity} else _ -> {:reject, "[DNSRBLPolicy]"} end end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe do @@ -90,7 +90,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do } end - defp check_rbl(%{host: actor_host}, object) do + defp check_rbl(%{host: actor_host}, activity) do with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()), zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do query = @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do rbl_response = rblquery(query) if Enum.empty?(rbl_response) do - {:ok, object} + {:ok, activity} else Task.start(fn -> reason = @@ -117,7 +117,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do :error end else - _ -> {:ok, object} + _ -> {:ok, activity} end end diff --git a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex index ad0936839..e4fcc9935 100644 --- a/lib/pleroma/web/activity_pub/mrf/drop_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/drop_policy.ex @@ -8,9 +8,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.DropPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(object) do - Logger.debug("REJECTING #{inspect(object)}") - {:reject, object} + def filter(activity) do + Logger.debug("REJECTING #{inspect(activity)}") + {:reject, activity} end @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex index f884962b9..1de5280d9 100644 --- a/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/emoji_policy.ex @@ -28,11 +28,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do Pleroma.Config.get([:mrf_emoji, :federated_timeline_removal_shortcode], []) end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :manual - @impl Pleroma.Web.ActivityPub.MRF.Policy - def filter(%{"type" => type, "object" => %{"type" => objtype} = object} = message) + @impl true + def filter(%{"type" => type, "object" => %{"type" => objtype} = object} = activity) when type in ["Create", "Update"] and objtype in Pleroma.Constants.status_object_types() do with {:ok, object} <- Updater.do_with_history(object, fn object -> @@ -42,13 +42,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do Updater.do_with_history(object, fn object -> {:ok, process_remove(object, :shortcode, config_remove_shortcode())} end), - activity <- Map.put(message, "object", object), + activity <- Map.put(activity, "object", object), activity <- maybe_delist(activity) do {:ok, activity} end end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => type} = object) when type in Pleroma.Constants.actor_types() do with object <- process_remove(object, :url, config_remove_url()), object <- process_remove(object, :shortcode, config_remove_shortcode()) do @@ -56,7 +56,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do end end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"type" => "EmojiReact"} = object) do with {:ok, _} <- matched_emoji_checker(config_remove_url(), config_remove_shortcode()).(object) do @@ -67,9 +67,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do end end - @impl Pleroma.Web.ActivityPub.MRF.Policy - def filter(message) do - {:ok, message} + @impl true + def filter(activity) do + {:ok, activity} end defp match_string?(string, pattern) when is_binary(pattern) do @@ -214,7 +214,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do ) end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def describe do mrf_emoji = Pleroma.Config.get(:mrf_emoji, []) @@ -226,7 +226,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do {:ok, %{mrf_emoji: mrf_emoji}} end - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def config_description do %{ key: :mrf_emoji, @@ -239,7 +239,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do key: :remove_url, type: {:list, :string}, description: """ - A list of patterns which result in emoji whose URL matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles. + A list of patterns which result in emoji whose URL matches being removed from the activity. This will apply to statuses, emoji reactions, and user profiles. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, @@ -249,7 +249,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do key: :remove_shortcode, type: {:list, :string}, description: """ - A list of patterns which result in emoji whose shortcode matches being removed from the message. This will apply to statuses, emoji reactions, and user profiles. + A list of patterns which result in emoji whose shortcode matches being removed from the activity. This will apply to statuses, emoji reactions, and user profiles. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, @@ -259,7 +259,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do key: :federated_timeline_removal_url, type: {:list, :string}, description: """ - A list of patterns which result in message with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. + A list of patterns which result in activity with emojis whose URLs match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, @@ -269,7 +269,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.EmojiPolicy do key: :federated_timeline_removal_shortcode, type: {:list, :string}, description: """ - A list of patterns which result in message with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. + A list of patterns which result in activities with emojis whose shortcodes match being removed from federated timelines (a.k.a unlisted). This will apply only to statuses. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, diff --git a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex index a148cc1e7..f5983c8a7 100644 --- a/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex +++ b/lib/pleroma/web/activity_pub/mrf/ensure_re_prepended.ex @@ -29,19 +29,19 @@ defmodule Pleroma.Web.ActivityPub.MRF.EnsureRePrepended do def filter_by_summary(_in_reply_to, child), do: child - def filter(%{"type" => type, "object" => child_object} = object) - when type in ["Create", "Update"] and is_map(child_object) do + def filter(%{"type" => type, "object" => object} = activity) + when type in ["Create", "Update"] and is_map(object) do child = - child_object["inReplyTo"] + object["inReplyTo"] |> Object.normalize(fetch: false) - |> filter_by_summary(child_object) + |> filter_by_summary(object) - object = Map.put(object, "object", child) + activity = Map.put(activity, "object", child) - {:ok, object} + {:ok, activity} end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} def describe, do: {:ok, %{}} end diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 55ea2683c..480a03ef6 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -11,12 +11,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do require Logger @impl true - def filter(message) do + def filter(activity) do with follower_nickname <- Config.get([:mrf_follow_bot, :follower_nickname]), %User{actor_type: "Service"} = follower <- User.get_cached_by_nickname(follower_nickname), - %{"type" => "Create", "object" => %{"type" => "Note"}} <- message do - try_follow(follower, message) + %{"type" => "Create", "object" => %{"type" => "Note"}} <- activity do + try_follow(follower, activity) else nil -> Logger.warning( @@ -24,17 +24,17 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do account does not exist, or the account is not correctly configured as a bot." ) - {:ok, message} + {:ok, activity} _ -> - {:ok, message} + {:ok, activity} end end - defp try_follow(follower, message) do - to = Map.get(message, "to", []) - cc = Map.get(message, "cc", []) - actor = [message["actor"]] + defp try_follow(follower, activity) do + to = Map.get(activity, "to", []) + cc = Map.get(activity, "cc", []) + actor = [activity["actor"]] Enum.concat([to, cc, actor]) |> List.flatten() @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do end end) - {:ok, message} + {:ok, activity} end @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex index 8cec8eabe..3b3251dc3 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_bot_unlisted_policy.ex @@ -22,7 +22,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do "cc" => cc, "actor" => actor, "object" => object - } = message + } = activity ) do user = User.get_cached_by_ap_id(actor) isbot = check_if_bot(user) @@ -36,20 +36,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceBotUnlistedPolicy do |> Map.put("to", to) |> Map.put("cc", cc) - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Map.put("object", object) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/force_mention.ex b/lib/pleroma/web/activity_pub/mrf/force_mention.ex index 3853489fc..4ea23540d 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mention.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mention.ex @@ -52,7 +52,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMention do end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex index 5532093cb..caae365e5 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex @@ -79,18 +79,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do %{ "type" => type, "object" => %{"type" => "Note", "to" => to, "inReplyTo" => in_reply_to} - } = object + } = activity ) when type in ["Create", "Update"] and is_list(to) and is_binary(in_reply_to) do # image-only posts from pleroma apparently reach this MRF without the content field - content = object["object"]["content"] || "" + content = activity["object"]["content"] || "" # Get the replied-to user for sorting - replied_to_user = get_replied_to_user(object["object"]) + replied_to_user = get_replied_to_user(activity["object"]) mention_users = to - |> clean_recipients(object) + |> clean_recipients(activity) |> Enum.map(&User.get_cached_by_ap_id/1) |> Enum.reject(&is_nil/1) |> sort_replied_user(replied_to_user) @@ -126,11 +126,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do content end - {:ok, put_in(object["object"]["content"], content)} + {:ok, put_in(activity["object"]["content"], content)} end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex index fdb9a9dba..72f2274ed 100644 --- a/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hashtag_policy.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do alias Pleroma.Object @moduledoc """ - Reject, TWKN-remove or Set-Sensitive messages with specific hashtags (without the leading #) + Reject, TWKN-remove or Set-Sensitive activities with specific hashtags (without the leading #) Note: This MRF Policy is always enabled, if you want to disable it you have to set empty lists. """ @@ -19,40 +19,40 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do @impl true def history_awareness, do: :manual - defp check_reject(message, hashtags) do + defp check_reject(activity, hashtags) do if Enum.any?(Config.get([:mrf_hashtag, :reject]), fn match -> match in hashtags end) do {:reject, "[HashtagPolicy] Matches with rejected keyword"} else - {:ok, message} + {:ok, activity} end end - defp check_ftl_removal(%{"to" => to} = message, hashtags) do + defp check_ftl_removal(%{"to" => to} = activity, hashtags) do if Pleroma.Constants.as_public() in to and Enum.any?(Config.get([:mrf_hashtag, :federated_timeline_removal]), fn match -> match in hashtags end) do to = List.delete(to, Pleroma.Constants.as_public()) - cc = [Pleroma.Constants.as_public() | message["cc"] || []] + cc = [Pleroma.Constants.as_public() | activity["cc"] || []] - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Kernel.put_in(["object", "to"], to) |> Kernel.put_in(["object", "cc"], cc) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end - defp check_ftl_removal(message, _hashtags), do: {:ok, message} + defp check_ftl_removal(activity, _hashtags), do: {:ok, activity} - defp check_sensitive(message) do + defp check_sensitive(activity) do {:ok, new_object} = - Object.Updater.do_with_history(message["object"], fn object -> + Object.Updater.do_with_history(activity["object"], fn object -> hashtags = Object.hashtags(%Object{data: object}) if Enum.any?(Config.get([:mrf_hashtag, :sensitive]), fn match -> match in hashtags end) do @@ -62,11 +62,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do end end) - {:ok, Map.put(message, "object", new_object)} + {:ok, Map.put(activity, "object", new_object)} end @impl true - def filter(%{"type" => type, "object" => object} = message) when type in ["Create", "Update"] do + def filter(%{"type" => type, "object" => object} = activity) + when type in ["Create", "Update"] do history_items = with %{"formerRepresentations" => %{"orderedItems" => items}} <- object do items @@ -82,23 +83,23 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do hashtags = Object.hashtags(%Object{data: object}) ++ historical_hashtags if hashtags != [] do - with {:ok, message} <- check_reject(message, hashtags), - {:ok, message} <- + with {:ok, activity} <- check_reject(activity, hashtags), + {:ok, activity} <- (if type == "Create" do - check_ftl_removal(message, hashtags) + check_ftl_removal(activity, hashtags) else - {:ok, message} + {:ok, activity} end), - {:ok, message} <- check_sensitive(message) do - {:ok, message} + {:ok, activity} <- check_sensitive(activity) do + {:ok, activity} end else - {:ok, message} + {:ok, activity} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe do @@ -120,21 +121,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.HashtagPolicy do %{ key: :reject, type: {:list, :string}, - description: "A list of hashtags which result in message being rejected.", + description: "A list of hashtags which result in the activity being rejected.", suggestions: ["foo"] }, %{ key: :federated_timeline_removal, type: {:list, :string}, description: - "A list of hashtags which result in message being removed from federated timelines (a.k.a unlisted).", + "A list of hashtags which result in the activity being removed from federated timelines (a.k.a unlisted).", suggestions: ["foo"] }, %{ key: :sensitive, type: {:list, :string}, description: - "A list of hashtags which result in message being set as sensitive (a.k.a NSFW/R-18)", + "A list of hashtags which result in the activity being set as sensitive (a.k.a NSFW/R-18)", suggestions: ["nsfw", "r18"] } ] diff --git a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex index 80e235d6e..3a80d0a69 100644 --- a/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/hellthread_policy.ex @@ -7,54 +7,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do require Pleroma.Constants - @moduledoc "Block messages with too much mentions (configurable)" + @moduledoc "Block activities with too much mentions (configurable)" @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp delist_message(message, threshold) when threshold > 0 do - follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address - to = message["to"] || [] - cc = message["cc"] || [] + defp delist_activity(activity, threshold) when threshold > 0 do + follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address + to = activity["to"] || [] + cc = activity["cc"] || [] follower_collection? = Enum.member?(to ++ cc, follower_collection) - message = - case get_recipient_count(message) do + activity = + case get_recipient_count(activity) do {:public, recipients} when follower_collection? and recipients > threshold -> - message + activity |> Map.put("to", [follower_collection]) |> Map.put("cc", [Pleroma.Constants.as_public()]) {:public, recipients} when recipients > threshold -> - message + activity |> Map.put("to", []) |> Map.put("cc", [Pleroma.Constants.as_public()]) _ -> - message + activity end - {:ok, message} + {:ok, activity} end - defp delist_message(message, _threshold), do: {:ok, message} + defp delist_activity(activity, _threshold), do: {:ok, activity} - defp reject_message(message, threshold) when threshold > 0 do - with {_, recipients} <- get_recipient_count(message) do + defp reject_activity(activity, threshold) when threshold > 0 do + with {_, recipients} <- get_recipient_count(activity) do if recipients > threshold do {:reject, "[HellthreadPolicy] #{recipients} recipients is over the limit of #{threshold}"} else - {:ok, message} + {:ok, activity} end end end - defp reject_message(message, _threshold), do: {:ok, message} + defp reject_activity(activity, _threshold), do: {:ok, activity} - defp get_recipient_count(message) do - recipients = (message["to"] || []) ++ (message["cc"] || []) - follower_collection = User.get_cached_by_ap_id(message["actor"]).follower_address + defp get_recipient_count(activity) do + recipients = (activity["to"] || []) ++ (activity["cc"] || []) + follower_collection = User.get_cached_by_ap_id(activity["actor"]).follower_address if Enum.member?(recipients, Pleroma.Constants.as_public()) do recipients = @@ -73,7 +73,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do end @impl true - def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = message) + def filter(%{"type" => "Create", "object" => %{"type" => object_type}} = activity) when object_type in ~w{Note Article} do reject_threshold = Pleroma.Config.get( @@ -83,16 +83,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do delist_threshold = Pleroma.Config.get([:mrf_hellthread, :delist_threshold]) - with {:ok, message} <- reject_message(message, reject_threshold), - {:ok, message} <- delist_message(message, delist_threshold) do - {:ok, message} + with {:ok, activity} <- reject_activity(activity, reject_threshold), + {:ok, activity} <- delist_activity(activity, delist_threshold) do + {:ok, activity} else e -> e end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, @@ -104,13 +104,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do key: :mrf_hellthread, related_policy: "Pleroma.Web.ActivityPub.MRF.HellthreadPolicy", label: "MRF Hellthread", - description: "Block messages with excessive user mentions", + description: "Block activities with excessive user mentions", children: [ %{ key: :delist_threshold, type: :integer, description: - "Number of mentioned users after which the message gets removed from timelines and" <> + "Number of mentioned users after which the activity gets removed from timelines and" <> "disables notifications. Set to 0 to disable.", suggestions: [10] }, @@ -118,7 +118,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.HellthreadPolicy do key: :reject_threshold, type: :integer, description: - "Number of mentioned users after which the messaged gets rejected. Set to 0 to disable.", + "Number of mentioned users after which the activity gets rejected. Set to 0 to disable.", suggestions: [20] } ] diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex index b7a01c27c..469d06ef6 100644 --- a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -48,12 +48,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :auto @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex index 729da4e9c..6ba6fd509 100644 --- a/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/keyword_policy.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do alias Pleroma.Web.ActivityPub.MRF.Utils - @moduledoc "Reject or Word-Replace messages with a keyword or regex" + @moduledoc "Reject or Word-Replace activities with a keyword or regex" @behaviour Pleroma.Web.ActivityPub.MRF.Policy @@ -25,7 +25,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do |> Enum.join("\n") end - defp check_reject(%{"object" => %{} = object} = message) do + defp check_reject(%{"object" => %{} = object} = activity) do with {:ok, _new_object} <- Pleroma.Object.Updater.do_with_history(object, fn object -> payload = object_payload(object) @@ -35,16 +35,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end) do {:reject, "[KeywordPolicy] Matches with rejected keyword"} else - {:ok, message} + {:ok, activity} end end) do - {:ok, message} + {:ok, activity} else e -> e end end - defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = message) do + defp check_ftl_removal(%{"type" => "Create", "to" => to, "object" => %{} = object} = activity) do check_keyword = fn object -> payload = object_payload(object) @@ -67,24 +67,24 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do if Pleroma.Constants.as_public() in to and should_delist?.(object) do to = List.delete(to, Pleroma.Constants.as_public()) - cc = [Pleroma.Constants.as_public() | message["cc"] || []] + cc = [Pleroma.Constants.as_public() | activity["cc"] || []] - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end - defp check_ftl_removal(message) do - {:ok, message} + defp check_ftl_removal(activity) do + {:ok, activity} end - defp check_replace(%{"object" => %{} = object} = message) do + defp check_replace(%{"object" => %{} = object} = activity) do replace_kw = fn object -> ["content", "name", "summary"] |> Enum.filter(fn field -> Map.has_key?(object, field) && object[field] end) @@ -103,18 +103,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do {:ok, object} = Pleroma.Object.Updater.do_with_history(object, replace_kw) - message = Map.put(message, "object", object) + activity = Map.put(activity, "object", object) - {:ok, message} + {:ok, activity} end @impl true - def filter(%{"type" => type, "object" => %{"content" => _content}} = message) + def filter(%{"type" => type, "object" => %{"content" => _content}} = activity) when type in ["Create", "Update"] do - with {:ok, message} <- check_reject(message), - {:ok, message} <- check_ftl_removal(message), - {:ok, message} <- check_replace(message) do - {:ok, message} + with {:ok, activity} <- check_reject(activity), + {:ok, activity} <- check_ftl_removal(activity), + {:ok, activity} <- check_replace(activity) do + {:ok, activity} else {:reject, nil} -> {:reject, "[KeywordPolicy] "} {:reject, _} = e -> e @@ -123,7 +123,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe do @@ -154,13 +154,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do related_policy: "Pleroma.Web.ActivityPub.MRF.KeywordPolicy", label: "MRF Keyword", description: - "Reject or Word-Replace messages matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).", + "Reject or Word-Replace activities matching a keyword or [Regex](https://hexdocs.pm/elixir/Regex.html).", children: [ %{ key: :reject, type: {:list, :string}, description: """ - A list of patterns which result in message being rejected. + A list of patterns which result in the activity being rejected. Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, @@ -170,7 +170,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.KeywordPolicy do key: :federated_timeline_removal, type: {:list, :string}, description: """ - A list of patterns which result in message being removed from federated timelines (a.k.a unlisted). + A list of patterns which result in the activity being removed from federated timelines (a.k.a unlisted). Each pattern can be a string or [Regex](https://hexdocs.pm/elixir/Regex.html) in the format of `~r/PATTERN/`. """, diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index 0c5b53def..b0d07a6f8 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -31,7 +31,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do HTTP.get(url, [], http_client_opts) end - defp preload(%{"object" => %{"attachment" => attachments}} = _message) do + defp preload(%{"object" => %{"attachment" => attachments}} = _activity) do Enum.each(attachments, fn %{"url" => url} when is_list(url) -> url @@ -49,15 +49,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do end @impl true - def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = message) + def filter(%{"type" => type, "object" => %{"attachment" => attachments} = _object} = activity) when type in ["Create", "Update"] and is_list(attachments) and length(attachments) > 0 do - preload(message) + preload(activity) - {:ok, message} + {:ok, activity} end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex index 8aa4f347f..f7bff121f 100644 --- a/lib/pleroma/web/activity_pub/mrf/mention_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/mention_policy.ex @@ -3,25 +3,25 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do - @moduledoc "Block messages which mention a user" + @moduledoc "Block activities which mention a user" @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(%{"type" => "Create"} = message) do + def filter(%{"type" => "Create"} = activity) do reject_actors = Pleroma.Config.get([:mrf_mention, :actors], []) - recipients = (message["to"] || []) ++ (message["cc"] || []) + recipients = (activity["to"] || []) ++ (activity["cc"] || []) if rejected_mention = Enum.find(recipients, fn recipient -> Enum.member?(reject_actors, recipient) end) do {:reject, "[MentionPolicy] Rejected for mention of #{rejected_mention}"} else - {:ok, message} + {:ok, activity} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} @@ -32,7 +32,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.MentionPolicy do key: :mrf_mention, related_policy: "Pleroma.Web.ActivityPub.MRF.MentionPolicy", label: "MRF Mention", - description: "Block messages which mention a specific user", + description: "Block activities which mention a specific user", children: [ %{ key: :actors, diff --git a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex index 12bf4ddd2..08dd39878 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_empty_policy.ex @@ -9,20 +9,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoEmptyPolicy do alias Pleroma.Web.Endpoint @impl true - def filter(%{"actor" => actor} = object) do + def filter(%{"actor" => actor} = activity) do with true <- local?(actor), - true <- eligible_type?(object), - true <- note?(object), - false <- has_attachment?(object), - true <- only_mentions?(object) do + true <- eligible_type?(activity), + true <- note?(activity), + false <- has_attachment?(activity), + true <- only_mentions?(activity) do {:reject, "[NoEmptyPolicy]"} else _ -> - {:ok, object} + {:ok, activity} end end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} defp local?(actor) do if actor |> String.starts_with?("#{Endpoint.url()}") do diff --git a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex index 8840c4fac..64a5872bc 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_op_policy.ex @@ -7,8 +7,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoOpPolicy do @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(object) do - {:ok, object} + def filter(activity) do + {:ok, activity} end @impl true diff --git a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex index f81e9e52a..c6f239a5e 100644 --- a/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/no_placeholder_text_policy.ex @@ -13,15 +13,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.NoPlaceholderTextPolicy do def filter( %{ "type" => type, - "object" => %{"content" => content, "attachment" => _} = _child_object - } = object + "object" => %{"content" => content, "attachment" => _} = _object + } = activity ) when type in ["Create", "Update"] and content in [".", "

.

"] do - {:ok, put_in(object, ["object", "content"], "")} + {:ok, put_in(activity, ["object", "content"], "")} end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex index 2dfc9a901..91855ef84 100644 --- a/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex +++ b/lib/pleroma/web/activity_pub/mrf/normalize_markup.ex @@ -12,20 +12,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.NormalizeMarkup do def history_awareness, do: :auto @impl true - def filter(%{"type" => type, "object" => child_object} = object) + def filter(%{"type" => type, "object" => object} = activity) when type in ["Create", "Update"] do scrub_policy = Pleroma.Config.get([:mrf_normalize_markup, :scrub_policy]) content = - child_object["content"] + object["content"] |> HTML.filter_tags(scrub_policy) - object = put_in(object, ["object", "content"], content) + activity = put_in(activity, ["object", "content"], content) - {:ok, object} + {:ok, activity} end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex index 451a212d4..52aaf05aa 100644 --- a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -122,52 +122,52 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end - def check_object_nsfw(%{"object" => %{} = child_object} = object) do - case check_object_nsfw(child_object) do - {:sfw, _} -> {:sfw, object} - {:nsfw, _} -> {:nsfw, object} + def check_object_nsfw(%{"object" => %{} = object} = activity) do + case check_object_nsfw(object) do + {:sfw, _} -> {:sfw, activity} + {:nsfw, _} -> {:nsfw, activity} end end def check_object_nsfw(object), do: {:sfw, object} @impl true - def filter(object) do - with {:sfw, object} <- check_object_nsfw(object) do - {:ok, object} + def filter(activity) do + with {:sfw, activity} <- check_object_nsfw(activity) do + {:ok, activity} else - {:nsfw, _data} -> handle_nsfw(object) + {:nsfw, _data} -> handle_nsfw(activity) end end - defp handle_nsfw(object) do + defp handle_nsfw(activity) do if Config.get([@policy, :reject]) do - {:reject, object} + {:reject, activity} else {:ok, - object + activity |> maybe_unlist() |> maybe_mark_sensitive()} end end - defp maybe_unlist(object) do + defp maybe_unlist(activity) do if Config.get([@policy, :unlist]) do - unlist(object) + unlist(activity) else - object + activity end end - defp maybe_mark_sensitive(object) do + defp maybe_mark_sensitive(activity) do if Config.get([@policy, :mark_sensitive]) do - mark_sensitive(object) + mark_sensitive(activity) else - object + activity end end - def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = activity) do with %User{} = user <- User.get_cached_by_ap_id(actor) do to = [user.follower_address | to] @@ -179,7 +179,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do |> List.delete(user.follower_address) |> Enum.uniq() - object + activity |> Map.put("to", to) |> Map.put("cc", cc) else @@ -187,14 +187,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do end end - def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do - Map.put(object, "object", mark_sensitive(child_object)) + def mark_sensitive(%{"object" => object} = activity) when is_map(object) do + Map.put(activity, "object", mark_sensitive(object)) end - def mark_sensitive(object) when is_map(object) do - tags = (object["tag"] || []) ++ ["nsfw"] + def mark_sensitive(activity) when is_map(activity) do + tags = (activity["tag"] || []) ++ ["nsfw"] - object + activity |> Map.put("tag", tags) |> Map.put("sensitive", true) end diff --git a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex index df1a6dcbb..34905fc21 100644 --- a/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/object_age_policy.ex @@ -11,12 +11,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do @moduledoc "Filter activities depending on their age" @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp check_date(%{"object" => %{"published" => published}} = message) do + defp check_date(%{"object" => %{"published" => published}} = activity) do with %DateTime{} = now <- DateTime.utc_now(), {:ok, %DateTime{} = then, _} <- DateTime.from_iso8601(published), max_ttl <- Config.get([:mrf_object_age, :threshold]), {:ttl, false} <- {:ttl, DateTime.diff(now, then) > max_ttl} do - {:ok, message} + {:ok, activity} else {:ttl, true} -> {:reject, nil} @@ -26,73 +26,73 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do end end - defp check_reject(message, actions) do + defp check_reject(activity, actions) do if :reject in actions do {:reject, "[ObjectAgePolicy]"} else - {:ok, message} + {:ok, activity} end end - defp check_delist(message, actions) do + defp check_delist(activity, actions) do if :delist in actions do - with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do + with %User{} = user <- User.get_cached_by_ap_id(activity["actor"]) do to = - List.delete(message["to"] || [], Pleroma.Constants.as_public()) ++ + List.delete(activity["to"] || [], Pleroma.Constants.as_public()) ++ [user.follower_address] cc = - List.delete(message["cc"] || [], user.follower_address) ++ + List.delete(activity["cc"] || [], user.follower_address) ++ [Pleroma.Constants.as_public()] - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Kernel.put_in(["object", "to"], to) |> Kernel.put_in(["object", "cc"], cc) - {:ok, message} + {:ok, activity} else _e -> {:reject, "[ObjectAgePolicy] Unhandled error"} end else - {:ok, message} + {:ok, activity} end end - defp check_strip_followers(message, actions) do + defp check_strip_followers(activity, actions) do if :strip_followers in actions do - with %User{} = user <- User.get_cached_by_ap_id(message["actor"]) do - to = List.delete(message["to"] || [], user.follower_address) - cc = List.delete(message["cc"] || [], user.follower_address) + with %User{} = user <- User.get_cached_by_ap_id(activity["actor"]) do + to = List.delete(activity["to"] || [], user.follower_address) + cc = List.delete(activity["cc"] || [], user.follower_address) - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Kernel.put_in(["object", "to"], to) |> Kernel.put_in(["object", "cc"], cc) - {:ok, message} + {:ok, activity} else _e -> {:reject, "[ObjectAgePolicy] Unhandled error"} end else - {:ok, message} + {:ok, activity} end end @impl true - def filter(%{"type" => "Create", "object" => %{"published" => _}} = message) do + def filter(%{"type" => "Create", "object" => %{"published" => _}} = activity) do with actions <- Config.get([:mrf_object_age, :actions]), - {:reject, _} <- check_date(message), - {:ok, message} <- check_reject(message, actions), - {:ok, message} <- check_delist(message, actions), - {:ok, message} <- check_strip_followers(message, actions) do - {:ok, message} + {:reject, _} <- check_date(activity), + {:ok, activity} <- check_reject(activity, actions), + {:ok, activity} <- check_delist(activity, actions), + {:ok, activity} <- check_strip_followers(activity, actions) do + {:ok, activity} else # check_date() is allowed to short-circuit the pipeline e -> e @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe do @@ -131,8 +131,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.ObjectAgePolicy do type: {:list, :atom}, description: "A list of actions to apply to the post. `:delist` removes the post from public timelines; " <> - "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct message; " <> - "`:reject` rejects the message entirely", + "`:strip_followers` removes followers from the ActivityPub recipient list ensuring they won't be delivered to home timelines, additionally for followers-only it degrades to a direct activity; " <> + "`:reject` rejects the activity entirely", suggestions: [:delist, :strip_followers, :reject] } ] diff --git a/lib/pleroma/web/activity_pub/mrf/policy.ex b/lib/pleroma/web/activity_pub/mrf/policy.ex index 1f34883e7..54ca4b735 100644 --- a/lib/pleroma/web/activity_pub/mrf/policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/policy.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.Policy do - @callback filter(map()) :: {:ok | :reject, map()} + @callback filter(Pleroma.Activity.t()) :: {:ok | :reject, Pleroma.Activity.t()} @callback describe() :: {:ok | :error, map()} @callback config_description() :: %{ optional(:children) => [map()], diff --git a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex index ae5e2cdc7..b07dc3b56 100644 --- a/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex +++ b/lib/pleroma/web/activity_pub/mrf/quiet_reply.ex @@ -26,7 +26,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do "type" => "Note", "inReplyTo" => in_reply_to } - } = object + } = activity ) do with true <- is_binary(in_reply_to), false <- match?([], cc), @@ -39,21 +39,21 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuietReply do updated_cc = [Pleroma.Constants.as_public()] - updated_object = - object + updated_activity = + activity |> Map.put("to", updated_to) |> Map.put("cc", updated_cc) |> put_in(["object", "to"], updated_to) |> put_in(["object", "cc"], updated_cc) - {:ok, updated_object} + {:ok, updated_activity} else - _ -> {:ok, object} + _ -> {:ok, activity} end end @impl true - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex index ac353f03f..2a17b6761 100644 --- a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex @@ -10,18 +10,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do require Pleroma.Constants - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do {:ok, Map.put(activity, "object", filter_object(object))} end - @impl Pleroma.Web.ActivityPub.MRF.Policy - def filter(object), do: {:ok, object} + @impl true + def filter(activity), do: {:ok, activity} - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def describe, do: {:ok, %{}} - @impl Pleroma.Web.ActivityPub.MRF.Policy + @impl true def history_awareness, do: :auto defp filter_object(%{"quoteUrl" => quote_url} = object) do diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index d708c99eb..ae7f18bfe 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -13,20 +13,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do require Pleroma.Constants - defp check_accept(%{host: actor_host} = _actor_info, object) do + defp check_accept(%{host: actor_host} = _actor_info, activity) do accepts = instance_list(:accept) |> MRF.subdomains_regex() cond do - accepts == [] -> {:ok, object} - actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, object} - MRF.subdomain_match?(accepts, actor_host) -> {:ok, object} + accepts == [] -> {:ok, activity} + actor_host == Config.get([Pleroma.Web.Endpoint, :url, :host]) -> {:ok, activity} + MRF.subdomain_match?(accepts, actor_host) -> {:ok, activity} true -> {:reject, "[SimplePolicy] host not in accept list"} end end - defp check_reject(%{host: actor_host} = _actor_info, object) do + defp check_reject(%{host: actor_host} = _actor_info, activity) do rejects = instance_list(:reject) |> MRF.subdomains_regex() @@ -34,109 +34,109 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do if MRF.subdomain_match?(rejects, actor_host) do {:reject, "[SimplePolicy] host in reject list"} else - {:ok, object} + {:ok, activity} end end defp check_media_removal( %{host: actor_host} = _actor_info, - %{"type" => type, "object" => %{"attachment" => child_attachment}} = object + %{"type" => type, "object" => %{"attachment" => object_attachment}} = activity ) - when length(child_attachment) > 0 and type in ["Create", "Update"] do + when length(object_attachment) > 0 and type in ["Create", "Update"] do media_removal = instance_list(:media_removal) |> MRF.subdomains_regex() - object = + activity = if MRF.subdomain_match?(media_removal, actor_host) do - child_object = Map.delete(object["object"], "attachment") - Map.put(object, "object", child_object) + object = Map.delete(activity["object"], "attachment") + Map.put(activity, "object", object) else - object + activity end - {:ok, object} + {:ok, activity} end - defp check_media_removal(_actor_info, object), do: {:ok, object} + defp check_media_removal(_actor_info, activity), do: {:ok, activity} defp check_media_nsfw( %{host: actor_host} = _actor_info, %{ "type" => type, - "object" => %{} = _child_object - } = object + "object" => %{} = _object + } = activity ) when type in ["Create", "Update"] do media_nsfw = instance_list(:media_nsfw) |> MRF.subdomains_regex() - object = + activity = if MRF.subdomain_match?(media_nsfw, actor_host) do - Kernel.put_in(object, ["object", "sensitive"], true) + Kernel.put_in(activity, ["object", "sensitive"], true) else - object + activity end - {:ok, object} + {:ok, activity} end - defp check_media_nsfw(_actor_info, object), do: {:ok, object} + defp check_media_nsfw(_actor_info, activity), do: {:ok, activity} - defp check_ftl_removal(%{host: actor_host} = _actor_info, object) do + defp check_ftl_removal(%{host: actor_host} = _actor_info, activity) do timeline_removal = instance_list(:federated_timeline_removal) |> MRF.subdomains_regex() - object = + activity = with true <- MRF.subdomain_match?(timeline_removal, actor_host), - user <- User.get_cached_by_ap_id(object["actor"]), - true <- Pleroma.Constants.as_public() in object["to"] do - to = List.delete(object["to"], Pleroma.Constants.as_public()) ++ [user.follower_address] + user <- User.get_cached_by_ap_id(activity["actor"]), + true <- Pleroma.Constants.as_public() in activity["to"] do + to = List.delete(activity["to"], Pleroma.Constants.as_public()) ++ [user.follower_address] - cc = List.delete(object["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()] + cc = List.delete(activity["cc"], user.follower_address) ++ [Pleroma.Constants.as_public()] - object + activity |> Map.put("to", to) |> Map.put("cc", cc) else - _ -> object + _ -> activity end - {:ok, object} + {:ok, activity} end defp intersection(list1, list2) do list1 -- list1 -- list2 end - defp check_followers_only(%{host: actor_host} = _actor_info, object) do + defp check_followers_only(%{host: actor_host} = _actor_info, activity) do followers_only = instance_list(:followers_only) |> MRF.subdomains_regex() - object = + activity = with true <- MRF.subdomain_match?(followers_only, actor_host), - user <- User.get_cached_by_ap_id(object["actor"]) do + user <- User.get_cached_by_ap_id(activity["actor"]) do # Don't use Map.get/3 intentionally, these must not be nil - fixed_to = object["to"] || [] - fixed_cc = object["cc"] || [] + fixed_to = activity["to"] || [] + fixed_cc = activity["cc"] || [] to = FollowingRelationship.followers_ap_ids(user, fixed_to) cc = FollowingRelationship.followers_ap_ids(user, fixed_cc) - object + activity |> Map.put("to", intersection([user.follower_address | to], fixed_to)) |> Map.put("cc", intersection([user.follower_address | cc], fixed_cc)) else - _ -> object + _ -> activity end - {:ok, object} + {:ok, activity} end - defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do + defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = activity) do report_removal = instance_list(:report_removal) |> MRF.subdomains_regex() @@ -144,39 +144,39 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do if MRF.subdomain_match?(report_removal, actor_host) do {:reject, "[SimplePolicy] host in report_removal list"} else - {:ok, object} + {:ok, activity} end end - defp check_report_removal(_actor_info, object), do: {:ok, object} + defp check_report_removal(_actor_info, activity), do: {:ok, activity} - defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = object) do + defp check_avatar_removal(%{host: actor_host} = _actor_info, %{"icon" => _icon} = activity) do avatar_removal = instance_list(:avatar_removal) |> MRF.subdomains_regex() if MRF.subdomain_match?(avatar_removal, actor_host) do - {:ok, Map.delete(object, "icon")} + {:ok, Map.delete(activity, "icon")} else - {:ok, object} + {:ok, activity} end end - defp check_avatar_removal(_actor_info, object), do: {:ok, object} + defp check_avatar_removal(_actor_info, activity), do: {:ok, activity} - defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = object) do + defp check_banner_removal(%{host: actor_host} = _actor_info, %{"image" => _image} = activity) do banner_removal = instance_list(:banner_removal) |> MRF.subdomains_regex() if MRF.subdomain_match?(banner_removal, actor_host) do - {:ok, Map.delete(object, "image")} + {:ok, Map.delete(activity, "image")} else - {:ok, object} + {:ok, activity} end end - defp check_banner_removal(_actor_info, object), do: {:ok, object} + defp check_banner_removal(_actor_info, activity), do: {:ok, activity} defp check_object(%{"object" => object} = activity) do with {:ok, _object} <- filter(object) do @@ -184,7 +184,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end end - defp check_object(object), do: {:ok, object} + defp check_object(activity), do: {:ok, activity} defp instance_list(config_key) do Config.get([:mrf_simple, config_key]) @@ -192,7 +192,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do end @impl true - def filter(%{"type" => "Delete", "actor" => actor} = object) do + def filter(%{"type" => "Delete", "actor" => actor} = activity) do %{host: actor_host} = URI.parse(actor) reject_deletes = @@ -202,54 +202,54 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do if MRF.subdomain_match?(reject_deletes, actor_host) do {:reject, "[SimplePolicy] host in reject_deletes list"} else - {:ok, object} + {:ok, activity} end end @impl true - def filter(%{"actor" => actor} = object) do + def filter(%{"actor" => actor} = activity) do actor_info = URI.parse(actor) - with {:ok, object} <- check_accept(actor_info, object), - {:ok, object} <- check_reject(actor_info, object), - {:ok, object} <- check_media_removal(actor_info, object), - {:ok, object} <- check_media_nsfw(actor_info, object), - {:ok, object} <- check_ftl_removal(actor_info, object), - {:ok, object} <- check_followers_only(actor_info, object), - {:ok, object} <- check_report_removal(actor_info, object), - {:ok, object} <- check_object(object) do - {:ok, object} + with {:ok, activity} <- check_accept(actor_info, activity), + {:ok, activity} <- check_reject(actor_info, activity), + {:ok, activity} <- check_media_removal(actor_info, activity), + {:ok, activity} <- check_media_nsfw(actor_info, activity), + {:ok, activity} <- check_ftl_removal(actor_info, activity), + {:ok, activity} <- check_followers_only(actor_info, activity), + {:ok, activity} <- check_report_removal(actor_info, activity), + {:ok, activity} <- check_object(activity) do + {:ok, activity} else {:reject, _} = e -> e end end - def filter(%{"id" => actor, "type" => obj_type} = object) - when obj_type in ["Application", "Group", "Organization", "Person", "Service"] do + def filter(%{"id" => actor, "type" => actor_type} = activity) + when actor_type in ["Application", "Group", "Organization", "Person", "Service"] do actor_info = URI.parse(actor) - with {:ok, object} <- check_accept(actor_info, object), - {:ok, object} <- check_reject(actor_info, object), - {:ok, object} <- check_avatar_removal(actor_info, object), - {:ok, object} <- check_banner_removal(actor_info, object) do - {:ok, object} + with {:ok, activity} <- check_accept(actor_info, activity), + {:ok, activity} <- check_reject(actor_info, activity), + {:ok, activity} <- check_avatar_removal(actor_info, activity), + {:ok, activity} <- check_banner_removal(actor_info, activity) do + {:ok, activity} else {:reject, _} = e -> e end end - def filter(object) when is_binary(object) do - uri = URI.parse(object) + def filter(activity) when is_binary(activity) do + uri = URI.parse(activity) - with {:ok, object} <- check_accept(uri, object), - {:ok, object} <- check_reject(uri, object) do - {:ok, object} + with {:ok, activity} <- check_accept(uri, activity), + {:ok, activity} <- check_reject(uri, activity) do + {:ok, activity} else {:reject, _} = e -> e end end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe do diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index fa6b595ea..6edfb124e 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -62,7 +62,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do end @impl true - def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do + def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = activity) do host = URI.parse(actor).host if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do @@ -97,10 +97,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do end end - {:ok, message} + {:ok, activity} end - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true @spec config_description :: %{ diff --git a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex index fdb9e5176..97acca7e8 100644 --- a/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/subchain_policy.ex @@ -20,20 +20,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do end @impl true - def filter(%{"actor" => actor} = message) do + def filter(%{"actor" => actor} = activity) do with {:ok, match, subchain} <- lookup_subchain(actor) do Logger.debug( "[SubchainPolicy] Matched #{actor} against #{inspect(match)} with subchain #{inspect(subchain)}" ) - MRF.filter(subchain, message) + MRF.filter(subchain, activity) else - _e -> {:ok, message} + _e -> {:ok, activity} end end @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} @@ -45,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SubchainPolicy do related_policy: "Pleroma.Web.ActivityPub.MRF.SubchainPolicy", label: "MRF Subchain", description: - "This policy processes messages through an alternate pipeline when a given message matches certain criteria." <> + "This policy processes activities through an alternate pipeline when a given activity matches certain criteria." <> " All criteria are configured as a map of regular expressions to lists of policy modules.", children: [ %{ diff --git a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex index 73760ca8f..c236a5a99 100644 --- a/lib/pleroma/web/activity_pub/mrf/tag_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/tag_policy.ex @@ -28,25 +28,25 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do "mrf_tag:media-force-nsfw", %{ "type" => type, - "object" => %{"attachment" => child_attachment} - } = message + "object" => %{"attachment" => object_attachment} + } = activity ) - when length(child_attachment) > 0 and type in ["Create", "Update"] do - {:ok, Kernel.put_in(message, ["object", "sensitive"], true)} + when length(object_attachment) > 0 and type in ["Create", "Update"] do + {:ok, Kernel.put_in(activity, ["object", "sensitive"], true)} end defp process_tag( "mrf_tag:media-strip", %{ "type" => type, - "object" => %{"attachment" => child_attachment} = object - } = message + "object" => %{"attachment" => object_attachment} = object + } = activity ) - when length(child_attachment) > 0 and type in ["Create", "Update"] do + when length(object_attachment) > 0 and type in ["Create", "Update"] do object = Map.delete(object, "attachment") - message = Map.put(message, "object", object) + activity = Map.put(activity, "object", object) - {:ok, message} + {:ok, activity} end defp process_tag( @@ -57,7 +57,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do "cc" => cc, "actor" => actor, "object" => object - } = message + } = activity ) do user = User.get_cached_by_ap_id(actor) @@ -70,15 +70,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do |> Map.put("to", to) |> Map.put("cc", cc) - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Map.put("object", object) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end @@ -90,7 +90,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do "cc" => cc, "actor" => actor, "object" => object - } = message + } = activity ) do user = User.get_cached_by_ap_id(actor) @@ -104,26 +104,26 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do |> Map.put("to", to) |> Map.put("cc", cc) - message = - message + activity = + activity |> Map.put("to", to) |> Map.put("cc", cc) |> Map.put("object", object) - {:ok, message} + {:ok, activity} else - {:ok, message} + {:ok, activity} end end defp process_tag( "mrf_tag:disable-remote-subscription", - %{"type" => "Follow", "actor" => actor} = message + %{"type" => "Follow", "actor" => actor} = activity ) do user = User.get_cached_by_ap_id(actor) if user.local == true do - {:ok, message} + {:ok, activity} else {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-remote-subscription"} @@ -133,14 +133,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do defp process_tag("mrf_tag:disable-any-subscription", %{"type" => "Follow", "actor" => actor}), do: {:reject, "[TagPolicy] Follow from #{actor} tagged with mrf_tag:disable-any-subscription"} - defp process_tag(_, message), do: {:ok, message} + defp process_tag(_, activity), do: {:ok, activity} - def filter_message(actor, message) do + def filter_activity(actor, activity) do User.get_cached_by_ap_id(actor) |> get_tags() - |> Enum.reduce({:ok, message}, fn - tag, {:ok, message} -> - process_tag(tag, message) + |> Enum.reduce({:ok, activity}, fn + tag, {:ok, activity} -> + process_tag(tag, activity) _, error -> error @@ -148,15 +148,15 @@ defmodule Pleroma.Web.ActivityPub.MRF.TagPolicy do end @impl true - def filter(%{"object" => target_actor, "type" => "Follow"} = message), - do: filter_message(target_actor, message) + def filter(%{"object" => target_actor, "type" => "Follow"} = activity), + do: filter_activity(target_actor, activity) @impl true - def filter(%{"actor" => actor, "type" => type} = message) when type in ["Create", "Update"], - do: filter_message(actor, message) + def filter(%{"actor" => actor, "type" => type} = activity) when type in ["Create", "Update"], + do: filter_activity(actor, activity) @impl true - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, do: {:ok, %{}} diff --git a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex index e14047d4e..10cc0e09d 100644 --- a/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/user_allow_list_policy.ex @@ -8,18 +8,18 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do @moduledoc "Accept-list of users from specified instances" @behaviour Pleroma.Web.ActivityPub.MRF.Policy - defp filter_by_list(object, []), do: {:ok, object} + defp filter_by_list(activity, []), do: {:ok, activity} - defp filter_by_list(%{"actor" => actor} = object, allow_list) do + defp filter_by_list(%{"actor" => actor} = activity, allow_list) do if actor in allow_list do - {:ok, object} + {:ok, activity} else {:reject, "[UserAllowListPolicy] #{actor} not in the list"} end end @impl true - def filter(%{"actor" => actor} = object) do + def filter(%{"actor" => actor} = activity) do actor_info = URI.parse(actor) allow_list = @@ -28,10 +28,10 @@ defmodule Pleroma.Web.ActivityPub.MRF.UserAllowListPolicy do [] ) - filter_by_list(object, allow_list) + filter_by_list(activity, allow_list) end - def filter(object), do: {:ok, object} + def filter(activity), do: {:ok, activity} @impl true def describe do diff --git a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex index 1c114558e..5671e4cf3 100644 --- a/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/vocabulary_policy.ex @@ -3,38 +3,38 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do - @moduledoc "Filter messages which belong to certain activity vocabularies" + @moduledoc "Filter activities which belong to certain activity vocabularies" @behaviour Pleroma.Web.ActivityPub.MRF.Policy @impl true - def filter(%{"type" => "Undo", "object" => child_message} = message) do - with {:ok, _} <- filter(child_message) do - {:ok, message} + def filter(%{"type" => "Undo", "object" => object} = activity) do + with {:ok, _} <- filter(object) do + {:ok, activity} else {:reject, _} = e -> e end end - def filter(%{"type" => message_type} = message) do + def filter(%{"type" => activity_type} = activity) do with accepted_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :accept]), rejected_vocabulary <- Pleroma.Config.get([:mrf_vocabulary, :reject]), {_, true} <- {:accepted, - Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, message_type)}, + Enum.empty?(accepted_vocabulary) || Enum.member?(accepted_vocabulary, activity_type)}, {_, false} <- {:rejected, - length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, message_type)}, - {:ok, _} <- filter(message["object"]) do - {:ok, message} + length(rejected_vocabulary) > 0 && Enum.member?(rejected_vocabulary, activity_type)}, + {:ok, _} <- filter(activity["object"]) do + {:ok, activity} else {:reject, _} = e -> e - {:accepted, _} -> {:reject, "[VocabularyPolicy] #{message_type} not in accept list"} - {:rejected, _} -> {:reject, "[VocabularyPolicy] #{message_type} in reject list"} + {:accepted, _} -> {:reject, "[VocabularyPolicy] #{activity_type} not in accept list"} + {:rejected, _} -> {:reject, "[VocabularyPolicy] #{activity_type} in reject list"} end end - def filter(message), do: {:ok, message} + def filter(activity), do: {:ok, activity} @impl true def describe, @@ -46,20 +46,20 @@ defmodule Pleroma.Web.ActivityPub.MRF.VocabularyPolicy do key: :mrf_vocabulary, related_policy: "Pleroma.Web.ActivityPub.MRF.VocabularyPolicy", label: "MRF Vocabulary", - description: "Filter messages which belong to certain activity vocabularies", + description: "Filter activities which belong to certain activity vocabularies", children: [ %{ key: :accept, type: {:list, :string}, description: - "A list of ActivityStreams terms to accept. If empty, all supported messages are accepted.", + "A list of ActivityStreams terms to accept. If empty, all supported activities are accepted.", suggestions: ["Create", "Follow", "Mention", "Announce", "Like"] }, %{ key: :reject, type: {:list, :string}, description: - "A list of ActivityStreams terms to reject. If empty, no messages are rejected.", + "A list of ActivityStreams terms to reject. If empty, no activities are rejected.", suggestions: ["Create", "Follow", "Mention", "Announce", "Like"] } ] -- cgit v1.2.3 From b281ad06de2de331450a5e319e3ba497071d4197 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 13 Aug 2024 15:34:56 -0400 Subject: Revert "Custom mix task to retry failed tests once in CI pipeline" This reverts commit ed2976b237b53a524247d564691e0a12d3231d68. --- lib/mix/tasks/pleroma/test_runner.ex | 25 ------------------------- 1 file changed, 25 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/test_runner.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex deleted file mode 100644 index 69fefb001..000000000 --- a/lib/mix/tasks/pleroma/test_runner.ex +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Mix.Tasks.Pleroma.TestRunner do - @shortdoc "Retries tests once if they fail" - - use Mix.Task - - def run(args \\ []) do - case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do - {_, 0} -> - :ok - - _ -> - retry(args) - end - end - - def retry(args) do - case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do - {_, 0} -> - :ok - - _ -> - exit(1) - end - end -end -- cgit v1.2.3 From 3b1e6ac8f0ea263beb422f11168ecc254f1bbf53 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 13 Aug 2024 20:06:01 +0200 Subject: User: truncate remote user fields instead of rejecting --- lib/pleroma/user.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 0e9d70831..c6c536943 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -463,6 +463,7 @@ defmodule Pleroma.User do def remote_user_changeset(struct \\ %User{local: false}, params) do bio_limit = Config.get([:instance, :user_bio_length], 5000) name_limit = Config.get([:instance, :user_name_length], 100) + fields_limit = Config.get([:instance, :max_remote_account_fields], 0) name = case params[:name] do @@ -476,6 +477,7 @@ defmodule Pleroma.User do |> Map.put_new(:last_refreshed_at, NaiveDateTime.utc_now()) |> truncate_if_exists(:name, name_limit) |> truncate_if_exists(:bio, bio_limit) + |> Map.update(:fields, [], &Enum.take(&1, fields_limit)) |> truncate_fields_param() |> fix_follower_address() -- cgit v1.2.3 From 8cd8cea3fb5ce87e5f92dfb45a667c47f78b6b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 14 Aug 2024 19:25:51 +0200 Subject: Fix 'Setting a marker should mark notifications as read' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/pleroma/web/mastodon_api/controllers/marker_controller.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex b/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex index 4ad30f330..42b2a201d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/marker_controller.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.MastodonAPI.MarkerController do use Pleroma.Web, :controller + alias Pleroma.Web.Plugs.OAuthScopesPlug plug(Pleroma.Web.ApiSpec.CastAndValidate) @@ -30,9 +31,16 @@ defmodule Pleroma.Web.MastodonAPI.MarkerController do def upsert(%{assigns: %{user: user}, body_params: params} = conn, _) do params = Map.new(params, fn {key, value} -> {to_string(key), value} end) - with {:ok, result} <- Pleroma.Marker.upsert(user, params), + with {:ok, _} <- mark_notifications_read(user, params), + {:ok, result} <- Pleroma.Marker.upsert(user, params), markers <- Map.values(result) do render(conn, "markers.json", %{markers: markers}) end end + + defp mark_notifications_read(user, %{"notifications" => %{last_read_id: last_read_id}}) do + Pleroma.Notification.set_read_up_to(user, last_read_id) + end + + defp mark_notifications_read(_, _), do: {:ok, :noop} end -- cgit v1.2.3 From 089fa4d1463dcc9b64d9a536d9dcfc4287c150c3 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 17 Aug 2024 20:33:42 -0400 Subject: Improve Remote Object Fetcher error handling, Oban --- lib/pleroma/object/fetcher.ex | 38 ++++------------------------ lib/pleroma/workers/remote_fetcher_worker.ex | 23 ++++++++++++----- 2 files changed, 21 insertions(+), 40 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex index c0f671dd4..9d9a201ca 100644 --- a/lib/pleroma/object/fetcher.ex +++ b/lib/pleroma/object/fetcher.ex @@ -73,50 +73,22 @@ defmodule Pleroma.Object.Fetcher do {:object, data, Object.normalize(activity, fetch: false)} do {:ok, object} else - {:allowed_depth, false} = e -> - log_fetch_error(id, e) - {:error, :allowed_depth} - - {:containment, reason} = e -> - log_fetch_error(id, e) - {:error, reason} - - {:transmogrifier, {:error, {:reject, reason}}} = e -> - log_fetch_error(id, e) - {:reject, reason} - - {:transmogrifier, {:reject, reason}} = e -> - log_fetch_error(id, e) - {:reject, reason} - - {:transmogrifier, reason} = e -> - log_fetch_error(id, e) - {:error, reason} - - {:object, data, nil} -> - reinject_object(%Object{}, data) - {:normalize, object = %Object{}} -> {:ok, object} {:fetch_object, %Object{} = object} -> {:ok, object} - {:fetch, {:error, reason}} = e -> - log_fetch_error(id, e) - {:error, reason} + {:object, data, nil} -> + reinject_object(%Object{}, data) e -> - log_fetch_error(id, e) - {:error, e} + Logger.metadata(object: id) + Logger.error("Object rejected while fetching #{id} #{inspect(e)}") + e end end - defp log_fetch_error(id, error) do - Logger.metadata(object: id) - Logger.error("Object rejected while fetching #{id} #{inspect(error)}") - end - defp prepare_activity_params(data) do %{ "type" => "Create", diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index e43765733..9d3f1ec53 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -13,17 +13,26 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do {:ok, _object} -> :ok - {:reject, reason} -> + {:allowed_depth, false} -> + {:cancel, :allowed_depth} + + {:containment, reason} -> {:cancel, reason} - {:error, :forbidden} -> - {:cancel, :forbidden} + {:transmogrifier, reason} -> + {:cancel, reason} - {:error, :not_found} -> - {:cancel, :not_found} + {:fetch, {:error, :forbidden = reason}} -> + {:cancel, reason} - {:error, :allowed_depth} -> - {:cancel, :allowed_depth} + {:fetch, {:error, :not_found = reason}} -> + {:cancel, reason} + + {:fetch, {:error, {:content_type, _}} = reason} -> + {:cancel, reason} + + {:fetch, {:error, reason}} -> + {:error, reason} {:error, _} = e -> e -- cgit v1.2.3 From 1b8141b506df1cc78e01f24881bed6257c9e5931 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sun, 18 Aug 2024 23:13:35 -0400 Subject: Address case where instance reachability status couldn't be updated --- lib/pleroma/web/activity_pub/publisher.ex | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 5cd982c6a..0de3a0d43 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -148,12 +148,17 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {"digest", p.digest} ] ) do - maybe_set_reachable(p.unreachable_since, p.inbox) + if not is_nil(p.unreachable_since) do + Instances.set_reachable(p.inbox) + end result else {_post_result, %{status: code} = response} = e -> - maybe_set_unreachable(p.unreachable_since, p.inbox) + if is_nil(p.unreachable_since) do + Instances.set_unreachable(p.inbox) + end + Logger.metadata(activity: p.activity_id, inbox: p.inbox, status: code) Logger.error("Publisher failed to inbox #{p.inbox} with status #{code}") @@ -174,7 +179,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do connection_pool_snooze() e -> - maybe_set_unreachable(p.unreachable_since, p.inbox) + if is_nil(p.unreachable_since) do + Instances.set_unreachable(p.inbox) + end + Logger.metadata(activity: p.activity_id, inbox: p.inbox) Logger.error("Publisher failed to inbox #{p.inbox} #{inspect(e)}") {:error, e} @@ -183,12 +191,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do defp connection_pool_snooze, do: {:snooze, 3} - defp maybe_set_reachable(%NaiveDateTime{}, inbox), do: Instances.set_reachable(inbox) - defp maybe_set_reachable(_, _), do: :ok - - defp maybe_set_unreachable(nil, inbox), do: Instances.set_unreachable(inbox) - defp maybe_set_unreachable(%NaiveDateTime{}, _), do: :ok - defp signature_host(%URI{port: port, scheme: scheme, host: host}) do if port == URI.default_port(scheme) do host -- cgit v1.2.3