diff options
author | Lain Soykaf <lain@lain.com> | 2024-05-28 16:51:19 +0400 |
---|---|---|
committer | Lain Soykaf <lain@lain.com> | 2024-05-28 16:51:19 +0400 |
commit | cc42b50c5bbbeee066d3d26907eebb488ec6aa3d (patch) | |
tree | 3b67771624d237bdb58f0931098d6682191cc214 /lib | |
parent | fdc3cbb8cbefad2161cc408b4440420d6e4b2a88 (diff) | |
parent | bef15cde6141a977aebdfc998d6091a31c4fc2d6 (diff) | |
download | pleroma-cc42b50c5bbbeee066d3d26907eebb488ec6aa3d.tar.gz pleroma-cc42b50c5bbbeee066d3d26907eebb488ec6aa3d.zip |
Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into pleroma-feature/akkoma-prune-old-posts
Diffstat (limited to 'lib')
75 files changed, 1936 insertions, 176 deletions
diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex new file mode 100644 index 000000000..81a9fced6 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -0,0 +1,80 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Indexer do + import Mix.Pleroma + import Ecto.Query + + alias Pleroma.Workers.SearchIndexingWorker + + def run(["create_index"]) do + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do + IO.puts("Index created") + else + e -> IO.puts("Could not create index: #{inspect(e)}") + end + end + + def run(["drop_index"]) do + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do + IO.puts("Index dropped") + else + e -> IO.puts("Could not drop index: #{inspect(e)}") + end + end + + def run(["index" | options]) do + {options, [], []} = + OptionParser.parse( + options, + strict: [ + limit: :integer + ] + ) + + start_pleroma() + + limit = Keyword.get(options, :limit, 100_000) + + per_step = 1000 + chunks = max(div(limit, per_step), 1) + + 1..chunks + |> Enum.each(fn step -> + q = + from(a in Pleroma.Activity, + limit: ^per_step, + offset: ^per_step * (^step - 1), + select: [:id], + order_by: [desc: :id] + ) + + {:ok, ids} = + Pleroma.Repo.transaction(fn -> + Pleroma.Repo.stream(q, timeout: :infinity) + |> Enum.map(fn a -> + a.id + end) + end) + + IO.puts("Got #{length(ids)} activities, adding to indexer") + + ids + |> Enum.chunk_every(100) + |> Enum.each(fn chunk -> + IO.puts("Adding #{length(chunk)} activities to indexing queue") + + chunk + |> Enum.map(fn id -> + SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) + end) + |> Oban.insert_all() + end) + end) + end +end diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 75154f94c..0d9757b44 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -14,6 +14,7 @@ defmodule Pleroma.Application do @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] + @compile_env Mix.env() def name, do: @name def version, do: @version @@ -51,7 +52,11 @@ defmodule Pleroma.Application do Pleroma.HTML.compile_scrubbers() Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() - Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + + if @compile_env != :test do + Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() + end + Pleroma.ApplicationRequirements.verify!() load_custom_modules() Pleroma.Docs.JSON.compile() @@ -109,7 +114,8 @@ defmodule Pleroma.Application do streamer_registry() ++ background_migrators() ++ shout_child(shout_enabled?()) ++ - [Pleroma.Gopher.Server] + [Pleroma.Gopher.Server] ++ + [Pleroma.Search.Healthcheck] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options @@ -162,7 +168,8 @@ defmodule Pleroma.Application do expiration: chat_message_id_idempotency_key_expiration(), limit: 500_000 ), - build_cachex("rel_me", limit: 2500) + build_cachex("rel_me", limit: 2500), + build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000) ] end diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 819245481..8c0df64fc 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -28,6 +28,7 @@ defmodule Pleroma.ApplicationRequirements do |> check_welcome_message_config!() |> check_rum!() |> check_repo_pool_size!() + |> check_mrfs() |> handle_result() end @@ -234,4 +235,25 @@ defmodule Pleroma.ApplicationRequirements do true end end + + defp check_mrfs(:ok) do + mrfs = Config.get!([:mrf, :policies]) + + missing_mrfs = + Enum.reduce(mrfs, [], fn x, acc -> + if Code.ensure_compiled(x) do + acc + else + acc ++ [x] + end + end) + + if Enum.empty?(missing_mrfs) do + :ok + else + {:error, "The following MRF modules are configured but missing: #{inspect(missing_mrfs)}"} + end + end + + defp check_mrfs(result), do: result end diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index d814b4931..3a5e35301 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Constants do "context_id", "deleted_activity_id", "pleroma_internal", - "generator" + "generator", + "rules" ] ) diff --git a/lib/pleroma/helpers/inet_helper.ex b/lib/pleroma/helpers/inet_helper.ex index 704d37f8a..00e18649e 100644 --- a/lib/pleroma/helpers/inet_helper.ex +++ b/lib/pleroma/helpers/inet_helper.ex @@ -16,4 +16,15 @@ defmodule Pleroma.Helpers.InetHelper do def parse_address(ip) do :inet.parse_address(ip) end + + def parse_cidr(proxy) when is_binary(proxy) do + proxy = + cond do + "/" in String.codepoints(proxy) -> proxy + InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" + InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" + end + + InetCidr.parse_cidr!(proxy, true) + end end diff --git a/lib/pleroma/http_signatures_api.ex b/lib/pleroma/http_signatures_api.ex new file mode 100644 index 000000000..8e73dc98e --- /dev/null +++ b/lib/pleroma/http_signatures_api.ex @@ -0,0 +1,4 @@ +defmodule Pleroma.HTTPSignaturesAPI do + @callback validate_conn(conn :: Plug.Conn.t()) :: boolean + @callback signature_for_conn(conn :: Plug.Conn.t()) :: map +end diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 710b19866..4f714b25f 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -73,6 +73,7 @@ defmodule Pleroma.Notification do pleroma:report reblog poll + status } def changeset(%Notification{} = notification, attrs) do @@ -280,15 +281,10 @@ defmodule Pleroma.Notification do select: n.id ) - {:ok, %{ids: {_, notification_ids}}} = - Multi.new() - |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) - |> Marker.multi_set_last_read_id(user, "notifications") - |> Repo.transaction() - - for_user_query(user) - |> where([n], n.id in ^notification_ids) - |> Repo.all() + Multi.new() + |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) + |> Marker.multi_set_last_read_id(user, "notifications") + |> Repo.transaction() end @spec read_one(User.t(), String.t()) :: @@ -299,10 +295,6 @@ defmodule Pleroma.Notification do |> Multi.update(:update, changeset(notification, %{seen: true})) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - |> case do - {:ok, %{update: notification}} -> {:ok, notification} - {:error, :update, changeset, _} -> {:error, changeset} - end end end @@ -361,37 +353,38 @@ defmodule Pleroma.Notification do end end - @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []} - def create_notifications(activity, options \\ []) + @spec create_notifications(Activity.t()) :: {:ok, [Notification.t()] | []} + def create_notifications(activity) - def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do + def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do object = Object.normalize(activity, fetch: false) if object && object.data["type"] == "Answer" do {:ok, []} else - do_create_notifications(activity, options) + do_create_notifications(activity) end end - def create_notifications(%Activity{data: %{"type" => type}} = activity, options) + def create_notifications(%Activity{data: %{"type" => type}} = activity) when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do - do_create_notifications(activity, options) + do_create_notifications(activity) end - def create_notifications(_, _), do: {:ok, []} + def create_notifications(_), do: {:ok, []} - defp do_create_notifications(%Activity{} = activity, options) do - do_send = Keyword.get(options, :do_send, true) + defp do_create_notifications(%Activity{} = activity) do + enabled_receivers = get_notified_from_activity(activity) - {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) - potential_receivers = enabled_receivers ++ disabled_receivers + enabled_subscribers = get_notified_subscribers_from_activity(activity) notifications = - Enum.map(potential_receivers, fn user -> - do_send = do_send && user in enabled_receivers - create_notification(activity, user, do_send: do_send) - end) + (Enum.map(enabled_receivers, fn user -> + create_notification(activity, user) + end) ++ + Enum.map(enabled_subscribers -- enabled_receivers, fn user -> + create_notification(activity, user, type: "status") + end)) |> Enum.reject(&is_nil/1) {:ok, notifications} @@ -450,7 +443,6 @@ defmodule Pleroma.Notification do # TODO move to sql, too. def create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do - do_send = Keyword.get(opts, :do_send, true) type = Keyword.get(opts, :type, type_from_activity(activity)) unless skip?(activity, user, opts) do @@ -465,11 +457,6 @@ defmodule Pleroma.Notification do |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - if do_send do - Streamer.stream(["user", "user:notification"], notification) - Push.send(notification) - end - notification end end @@ -527,13 +514,28 @@ defmodule Pleroma.Notification do |> exclude_relationship_restricted_ap_ids(activity) |> exclude_thread_muter_ap_ids(activity) - notification_enabled_users = - Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + end + + def get_notified_from_activity(_, _local_only), do: [] + + def get_notified_subscribers_from_activity(activity, local_only \\ true) - {notification_enabled_users, potential_receivers -- notification_enabled_users} + def get_notified_subscribers_from_activity( + %Activity{data: %{"type" => "Create"}} = activity, + local_only + ) do + notification_enabled_ap_ids = + [] + |> Utils.maybe_notify_subscribers(activity) + + potential_receivers = + User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only) + + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) end - def get_notified_from_activity(_, _local_only), do: {[], []} + def get_notified_subscribers_from_activity(_, _), do: [] # For some activities, only notify the author of the object def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}}) @@ -576,7 +578,6 @@ defmodule Pleroma.Notification do [] |> Utils.maybe_notify_to_recipients(activity) |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) |> Utils.maybe_notify_followers(activity) |> Enum.uniq() end @@ -643,6 +644,7 @@ defmodule Pleroma.Notification do def skip?(%Activity{} = activity, %User{} = user, opts) do [ :self, + :internal, :invisible, :block_from_strangers, :recently_followed, @@ -662,6 +664,12 @@ defmodule Pleroma.Notification do end end + def skip?(:internal, %Activity{} = activity, _user, _opts) do + actor = activity.data["actor"] + user = User.get_cached_by_ap_id(actor) + User.internal?(user) + end + def skip?(:invisible, %Activity{} = activity, _user, _opts) do actor = activity.data["actor"] user = User.get_cached_by_ap_id(actor) @@ -748,4 +756,12 @@ defmodule Pleroma.Notification do ) |> Repo.update_all(set: [seen: true]) end + + @spec send(list(Notification.t())) :: :ok + def send(notifications) do + Enum.each(notifications, fn notification -> + Streamer.stream(["user", "user:notification"], notification) + Push.send(notification) + end) + end end diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex new file mode 100644 index 000000000..3ba413214 --- /dev/null +++ b/lib/pleroma/rule.ex @@ -0,0 +1,68 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Rule do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.Repo + alias Pleroma.Rule + + schema "rules" do + field(:priority, :integer, default: 0) + field(:text, :string) + field(:hint, :string) + + timestamps() + end + + def changeset(%Rule{} = rule, params \\ %{}) do + rule + |> cast(params, [:priority, :text, :hint]) + |> validate_required([:text]) + end + + def query do + Rule + |> order_by(asc: :priority) + |> order_by(asc: :id) + end + + def get(ids) when is_list(ids) do + from(r in __MODULE__, where: r.id in ^ids) + |> Repo.all() + end + + def get(id), do: Repo.get(__MODULE__, id) + + def exists?(id) do + from(r in __MODULE__, where: r.id == ^id) + |> Repo.exists?() + end + + def create(params) do + {:ok, rule} = + %Rule{} + |> changeset(params) + |> Repo.insert() + + rule + end + + def update(params, id) do + {:ok, rule} = + get(id) + |> changeset(params) + |> Repo.update() + + rule + end + + def delete(id) do + get(id) + |> Repo.delete() + end +end diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index 63c6cb45b..c361d7d89 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -204,7 +204,7 @@ defmodule Pleroma.ScheduledActivity do def job_query(scheduled_activity_id) do from(j in Oban.Job, - where: j.queue == "scheduled_activities", + where: j.queue == "federator_outgoing", where: fragment("args ->> 'activity_id' = ?::text", ^to_string(scheduled_activity_id)) ) end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 3b266e59b..fd0218cb8 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -10,8 +10,12 @@ defmodule Pleroma.Search do end def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - + search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.search(options[:for_user], query, options) end + + def healthcheck_endpoints do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + search_module.healthcheck_endpoints + end end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 31bfc7e33..c6fe8a9bd 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -48,6 +48,15 @@ defmodule Pleroma.Search.DatabaseSearch do @impl true def remove_from_index(_object), do: :ok + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + + @impl true + def healthcheck_endpoints, do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex new file mode 100644 index 000000000..e562c8478 --- /dev/null +++ b/lib/pleroma/search/healthcheck.ex @@ -0,0 +1,86 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.Search.Healthcheck do + @doc """ + Monitors health of search backend to control processing of events based on health and availability. + """ + use GenServer + require Logger + + @queue :search_indexing + @tick :timer.seconds(5) + @timeout :timer.seconds(2) + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_) do + state = %{healthy: false} + {:ok, state, {:continue, :start}} + end + + @impl true + def handle_continue(:start, state) do + tick() + {:noreply, state} + end + + @impl true + def handle_info(:check, state) do + urls = Pleroma.Search.healthcheck_endpoints() + + new_state = + if check(urls) do + Oban.resume_queue(queue: @queue) + Map.put(state, :healthy, true) + else + Oban.pause_queue(queue: @queue) + Map.put(state, :healthy, false) + end + + maybe_log_state_change(state, new_state) + + tick() + {:noreply, new_state} + end + + @impl true + def handle_call(:state, _from, state) do + {:reply, state, state, :hibernate} + end + + def state, do: GenServer.call(__MODULE__, :state) + + def check([]), do: true + + def check(urls) when is_list(urls) do + Enum.all?( + urls, + fn url -> + case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do + {:ok, %{status: 200}} -> true + _ -> false + end + end + ) + end + + def check(_), do: true + + defp tick do + Process.send_after(self(), :check, @tick) + end + + defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do + Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure") + end + + defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do + Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass") + end + + defp maybe_log_state_change(_, _), do: :ok +end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2bff663e8..9bba5b30f 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -10,6 +10,12 @@ defmodule Pleroma.Search.Meilisearch do @behaviour Pleroma.Search.SearchBackend + @impl true + def create_index, do: :ok + + @impl true + def drop_index, do: :ok + defp meili_headers do private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) @@ -178,4 +184,15 @@ defmodule Pleroma.Search.Meilisearch do def remove_from_index(object) do meili_delete("/indexes/objects/documents/#{object.id}") end + + @impl true + def healthcheck_endpoints do + endpoint = + Config.get([Pleroma.Search.Meilisearch, :url]) + |> URI.parse() + |> Map.put(:path, "/health") + |> URI.to_string() + + [endpoint] + end end diff --git a/lib/pleroma/search/qdrant_search.ex b/lib/pleroma/search/qdrant_search.ex new file mode 100644 index 000000000..b659bb682 --- /dev/null +++ b/lib/pleroma/search/qdrant_search.ex @@ -0,0 +1,182 @@ +defmodule Pleroma.Search.QdrantSearch do + @behaviour Pleroma.Search.SearchBackend + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config + + alias __MODULE__.OpenAIClient + alias __MODULE__.QdrantClient + + import Pleroma.Search.Meilisearch, only: [object_to_search_data: 1] + import Pleroma.Search.DatabaseSearch, only: [maybe_fetch: 3] + + @impl true + def create_index do + payload = Config.get([Pleroma.Search.QdrantSearch, :qdrant_index_configuration]) + + with {:ok, %{status: 200}} <- QdrantClient.put("/collections/posts", payload) do + :ok + else + e -> {:error, e} + end + end + + @impl true + def drop_index do + with {:ok, %{status: 200}} <- QdrantClient.delete("/collections/posts") do + :ok + else + e -> {:error, e} + end + end + + def get_embedding(text) do + with {:ok, %{body: %{"data" => [%{"embedding" => embedding}]}}} <- + OpenAIClient.post("/v1/embeddings", %{ + input: text, + model: Config.get([Pleroma.Search.QdrantSearch, :openai_model]) + }) do + {:ok, embedding} + else + _ -> + {:error, "Failed to get embedding"} + end + end + + defp actor_from_activity(%{data: %{"actor" => actor}}) do + actor + end + + defp actor_from_activity(_), do: nil + + defp build_index_payload(activity, embedding) do + actor = actor_from_activity(activity) + published_at = activity.data["published"] + + %{ + points: [ + %{ + id: activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!(), + vector: embedding, + payload: %{actor: actor, published_at: published_at} + } + ] + } + end + + defp build_search_payload(embedding, options) do + base = %{ + vector: embedding, + limit: options[:limit] || 20, + offset: options[:offset] || 0 + } + + if author = options[:author] do + Map.put(base, :filter, %{ + must: [%{key: "actor", match: %{value: author.ap_id}}] + }) + else + base + end + end + + @impl true + def add_to_index(activity) do + # This will only index public or unlisted notes + maybe_search_data = object_to_search_data(activity.object) + + if activity.data["type"] == "Create" and maybe_search_data do + with {:ok, embedding} <- get_embedding(maybe_search_data.content), + {:ok, %{status: 200}} <- + QdrantClient.put( + "/collections/posts/points", + build_index_payload(activity, embedding) + ) do + :ok + else + e -> {:error, e} + end + else + :ok + end + end + + @impl true + def remove_from_index(object) do + activity = Activity.get_by_object_ap_id_with_object(object.data["id"]) + id = activity.id |> FlakeId.from_string() |> Ecto.UUID.cast!() + + with {:ok, %{status: 200}} <- + QdrantClient.post("/collections/posts/points/delete", %{"points" => [id]}) do + :ok + else + e -> {:error, e} + end + end + + @impl true + def search(user, original_query, options) do + query = "Represent this sentence for searching relevant passages: #{original_query}" + + with {:ok, embedding} <- get_embedding(query), + {:ok, %{body: %{"result" => result}}} <- + QdrantClient.post( + "/collections/posts/points/search", + build_search_payload(embedding, options) + ) do + ids = + Enum.map(result, fn %{"id" => id} -> + Ecto.UUID.dump!(id) + end) + + from(a in Activity, where: a.id in ^ids) + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> Ecto.Query.order_by([a], fragment("array_position(?, ?)", ^ids, a.id)) + |> Pleroma.Repo.all() + |> maybe_fetch(user, original_query) + else + _ -> + [] + end + end + + @impl true + def healthcheck_endpoints do + qdrant_health = + Config.get([Pleroma.Search.QdrantSearch, :qdrant_url]) + |> URI.parse() + |> Map.put(:path, "/healthz") + |> URI.to_string() + + openai_health = Config.get([Pleroma.Search.QdrantSearch, :openai_healthcheck_url]) + + [qdrant_health, openai_health] |> Enum.filter(& &1) + end +end + +defmodule Pleroma.Search.QdrantSearch.OpenAIClient do + use Tesla + alias Pleroma.Config.Getting, as: Config + + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :openai_url])) + plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"Authorization", + "Bearer #{Pleroma.Config.get([Pleroma.Search.QdrantSearch, :openai_api_key])}"} + ]) +end + +defmodule Pleroma.Search.QdrantSearch.QdrantClient do + use Tesla + alias Pleroma.Config.Getting, as: Config + + plug(Tesla.Middleware.BaseUrl, Config.get([Pleroma.Search.QdrantSearch, :qdrant_url])) + plug(Tesla.Middleware.JSON) + + plug(Tesla.Middleware.Headers, [ + {"api-key", Pleroma.Config.get([Pleroma.Search.QdrantSearch, :qdrant_api_key])} + ]) +end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 68bc48cec..f4ed13c36 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -21,4 +21,22 @@ defmodule Pleroma.Search.SearchBackend do from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} + + @doc """ + Create the index + """ + @callback create_index() :: :ok | {:error, any()} + + @doc """ + Drop the index + """ + @callback drop_index() :: :ok | {:error, any()} + + @doc """ + Healthcheck endpoints of search backend infrastructure to monitor for controlling + processing of jobs in the Oban queue. + + It is expected a 200 response is healthy and other responses are unhealthy. + """ + @callback healthcheck_endpoints :: list() | nil end diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index 8fd422a6e..900d40c4b 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -44,8 +44,7 @@ defmodule Pleroma.Signature do defp remove_suffix(uri, []), do: uri def fetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} else @@ -55,8 +54,7 @@ defmodule Pleroma.Signature do end def refetch_public_key(conn) do - with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), - {:ok, actor_id} <- key_id_to_actor_id(kid), + with {:ok, actor_id} <- get_actor_id(conn), {:ok, _user} <- ActivityPub.make_user_from_ap_id(actor_id), {:ok, public_key} <- User.get_public_key_for_ap_id(actor_id) do {:ok, public_key} @@ -66,6 +64,16 @@ defmodule Pleroma.Signature do end end + def get_actor_id(conn) do + with %{"keyId" => kid} <- HTTPSignatures.signature_for_conn(conn), + {:ok, actor_id} <- key_id_to_actor_id(kid) do + {:ok, actor_id} + else + e -> + {:error, e} + end + end + def sign(%User{keys: keys} = user, headers) do with {:ok, private_key, _} <- Keys.keys_from_pem(keys) do HTTPSignatures.sign(private_key, user.ap_id <> "#main-key", headers) diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index e6c484548..35c7c02a5 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -239,8 +239,12 @@ defmodule Pleroma.Upload do "" end - [base_url, path] - |> Path.join() + if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do + String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path) + else + [base_url, path] + |> Path.join() + end end defp url_from_spec(_upload, _base_url, {:url, url}), do: url @@ -277,6 +281,9 @@ defmodule Pleroma.Upload do Path.join([upload_base_url, bucket_with_namespace]) end + Pleroma.Uploaders.IPFS -> + @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + _ -> public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" end diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex new file mode 100644 index 000000000..d171e4652 --- /dev/null +++ b/lib/pleroma/uploaders/ipfs.ex @@ -0,0 +1,77 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFS do + @behaviour Pleroma.Uploaders.Uploader + require Logger + + alias Tesla.Multipart + + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + + defp get_final_url(method) do + config = @config_impl.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + Path.join([post_base_url, method]) + end + + def put_file_endpoint do + get_final_url("/api/v0/add") + end + + def delete_file_endpoint do + get_final_url("/api/v0/files/rm") + end + + @placeholder "{CID}" + def placeholder, do: @placeholder + + @impl true + def get_file(file) do + b_url = Pleroma.Upload.base_url() + + if String.contains?(b_url, @placeholder) do + {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}} + else + {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} + end + end + + @impl true + def put_file(%Pleroma.Upload{} = upload) do + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(upload.tempfile) + + case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do + {:ok, ret} -> + case Jason.decode(ret.body) do + {:ok, ret} -> + if Map.has_key?(ret, "Hash") do + {:ok, {:file, ret["Hash"]}} + else + {:error, "JSON doesn't contain Hash key"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "JSON decode failed"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "IPFS Gateway upload failed"} + end + end + + @impl true + def delete_file(file) do + case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do + {:ok, %{status: 204}} -> :ok + error -> {:error, inspect(error)} + end + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 778e20526..6d6aa98b5 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1404,6 +1404,40 @@ defmodule Pleroma.User do |> Repo.all() end + @spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do + friends = + get_friends_query(current_user) + |> where([u], not u.hide_follows) + |> select([u], u.id) + + User.Query.build(%{is_active: true}) + |> where([u], u.id not in ^[user.id, current_user.id]) + |> join(:inner, [u], r in FollowingRelationship, + as: :followers_relationships, + on: r.following_id == ^user.id and r.follower_id == u.id + ) + |> where([followers_relationships: r], r.state == ^:follow_accept) + |> where([followers_relationships: r], r.follower_id in subquery(friends)) + end + + def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do + user + |> get_familiar_followers_query(current_user, nil) + |> User.Query.paginate(page, 20) + end + + @spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user), + do: get_familiar_followers_query(user, current_user, nil) + + @spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())} + def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do + user + |> get_familiar_followers_query(current_user, page) + |> Repo.all() + end + def increase_note_count(%User{} = user) do User |> where(id: ^user.id) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a1fccc705..5bb0fba6e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -200,7 +200,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def notify_and_stream(activity) do - Notification.create_notifications(activity) + {:ok, notifications} = Notification.create_notifications(activity) + Notification.send(notifications) original_activity = case activity do @@ -978,8 +979,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_replies(query, %{exclude_replies: true}) do from( - [_activity, object] in query, - where: fragment("?->>'inReplyTo' is null", object.data) + [activity, object] in query, + where: + fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data) ) end @@ -1259,6 +1261,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_quote_url(query, _), do: query + defp restrict_rule(query, %{rule_id: rule_id}) do + from( + activity in query, + where: fragment("(?)->'rules' \\? (?)", activity.data, ^rule_id) + ) + end + + defp restrict_rule(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1421,6 +1432,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_instance(opts) |> restrict_announce_object_actor(opts) |> restrict_filtered(opts) + |> restrict_rule(opts) |> restrict_quote_url(opts) |> maybe_restrict_deactivated_users(opts) |> exclude_poll_votes(opts) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index e38a94966..e6161455d 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -52,6 +52,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do when action in [:activity, :object] ) + plug(:log_inbox_metadata when action in [:inbox]) plug(:set_requester_reachable when action in [:inbox]) plug(:relay_active? when action in [:relay]) @@ -521,6 +522,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do conn end + defp log_inbox_metadata(%{params: %{"actor" => actor, "type" => type}} = conn, _) do + Logger.metadata(actor: actor, type: type) + conn + end + + defp log_inbox_metadata(conn, _), do: conn + def upload_media(%{assigns: %{user: %User{} = user}} = conn, %{"file" => file} = data) do with {:ok, object} <- ActivityPub.upload( diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex new file mode 100644 index 000000000..531e75ce8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + alias Pleroma.Config + alias Pleroma.User + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + defp user_has_posted?(%User{} = u), do: u.note_count > 0 + + defp user_has_age?(%User{} = u) do + user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000) + diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond) + diff >= user_age_limit + end + + defp good_reputation?(%User{} = u) do + user_has_age?(u) and user_has_posted?(u) + end + + # copied from HellthreadPolicy + defp get_recipient_count(message) do + recipients = (message["to"] || []) ++ (message["cc"] || []) + + follower_collection = + User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address + + if Enum.member?(recipients, Pleroma.Constants.as_public()) do + recipients = + recipients + |> List.delete(Pleroma.Constants.as_public()) + |> List.delete(follower_collection) + + {:public, length(recipients)} + else + recipients = + recipients + |> List.delete(follower_collection) + + {:not_public, length(recipients)} + end + end + + defp object_has_recipients?(%{"object" => object} = activity) do + {_, object_count} = get_recipient_count(object) + {_, activity_count} = get_recipient_count(activity) + object_count + activity_count > 0 + end + + defp object_has_recipients?(object) do + {_, count} = get_recipient_count(object) + count > 0 + end + + @impl true + def filter(%{"type" => "Create", "actor" => actor} = activity) do + with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), + {:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)}, + {:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do + {:ok, activity} + else + {:ok, %User{local: true}} -> + {:ok, activity} + + {:has_mentions, false} -> + {:ok, activity} + + {:good_reputation, false} -> + {:reject, "[AntiMentionSpamPolicy] User rejected"} + + {:error, _} -> + {:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"} + + e -> + {:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"} + end + end + + # in all other cases, pass through + def filter(message), do: {:ok, message} + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex new file mode 100644 index 000000000..9543cc545 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex @@ -0,0 +1,142 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do + @moduledoc """ + Dynamic activity filtering based on an RBL database + + This MRF makes queries to a custom DNS server which will + respond with values indicating the classification of the domain + the activity originated from. This method has been widely used + in the email anti-spam industry for very fast reputation checks. + + e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK + Other values such as 127.0.0.2 may be used for specific classifications. + + Information for why the host is blocked can be stored in a corresponding TXT record. + + This method is fail-open so if the queries fail the activites are accepted. + + An example of software meant for this purpsoe is rbldnsd which can be found + at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at + https://git.pleroma.social/feld/rbldnsd + + It is highly recommended that you run your own copy of rbldnsd and use an + external mechanism to sync/share the contents of the zone file. This is + important to keep the latency on the queries as low as possible and prevent + your DNS server from being attacked so it fails and content is permitted. + """ + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Config + + require Logger + + @query_retries 1 + @query_timeout 500 + + @impl true + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) + + with {:ok, object} <- check_rbl(actor_info, object) do + {:ok, object} + else + _ -> {:reject, "[DNSRBLPolicy]"} + end + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe do + mrf_dnsrbl = + Config.get(:mrf_dnsrbl) + |> Enum.into(%{}) + + {:ok, %{mrf_dnsrbl: mrf_dnsrbl}} + end + + @impl true + def config_description do + %{ + key: :mrf_dnsrbl, + related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy", + label: "MRF DNSRBL", + description: "DNS RealTime Blackhole Policy", + children: [ + %{ + key: :nameserver, + type: {:string}, + description: "DNSRBL Nameserver to Query (IP or hostame)", + suggestions: ["127.0.0.1"] + }, + %{ + key: :port, + type: {:string}, + description: "Nameserver port", + suggestions: ["53"] + }, + %{ + key: :zone, + type: {:string}, + description: "Root zone for querying", + suggestions: ["bl.pleroma.com"] + } + ] + } + end + + defp check_rbl(%{host: actor_host}, object) do + with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()), + zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do + query = + Enum.join([actor_host, zone], ".") + |> String.to_charlist() + + rbl_response = rblquery(query) + + if Enum.empty?(rbl_response) do + {:ok, object} + else + Task.start(fn -> + reason = rblquery(query, :txt) || "undefined" + + Logger.warning( + "DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}" + ) + end) + + :error + end + else + _ -> {:ok, object} + end + end + + defp get_rblhost_ip(rblhost) do + case rblhost |> String.to_charlist() |> :inet_parse.address() do + {:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address() + _ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()} + end + end + + defp rblquery(query, type \\ :a) do + config = Config.get([:mrf_dnsrbl]) + + case get_rblhost_ip(config[:nameserver]) do + {:ok, rblnsip} -> + :inet_res.lookup(query, :in, type, + nameservers: [{rblnsip, config[:port]}], + timeout: @query_timeout, + retry: @query_retries + ) + + _ -> + [] + end + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex index c95d35bb9..0c5b53def 100644 --- a/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/media_proxy_warming_policy.ex @@ -11,11 +11,6 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do require Logger - @adapter_options [ - pool: :media, - recv_timeout: 10_000 - ] - @impl true def history_awareness, do: :auto @@ -27,17 +22,14 @@ defmodule Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy do Logger.debug("Prefetching #{inspect(url)} as #{inspect(prefetch_url)}") - if Pleroma.Config.get(:env) == :test do - fetch(prefetch_url) - else - ConcurrentLimiter.limit(__MODULE__, fn -> - Task.start(fn -> fetch(prefetch_url) end) - end) - end + fetch(prefetch_url) end end - defp fetch(url), do: HTTP.get(url, [], @adapter_options) + defp fetch(url) do + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) + HTTP.get(url, [], http_client_opts) + end defp preload(%{"object" => %{"attachment" => attachments}} = _message) do Enum.each(attachments, fn diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex new file mode 100644 index 000000000..f7863039b --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -0,0 +1,265 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do + @moduledoc """ + Hide, delete, or mark sensitive NSFW content with artificial intelligence. + + Requires a NSFW API server, configured like so: + + config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF, + url: "http://127.0.0.1:5000/", + threshold: 0.7, + mark_sensitive: true, + unlist: false, + reject: false + + The NSFW API server must implement an HTTP endpoint like this: + + curl http://localhost:5000/?url=https://fedi.com/images/001.jpg + + Returning a response like this: + + {"score", 0.314} + + Where a score is 0-1, with `1` being definitely NSFW. + + A good API server is here: https://github.com/EugenCepoi/nsfw_api + You can run it with Docker with a one-liner: + + docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest + + Options: + + - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/" + - `threshold`: Lowest score to take action on. Default: `0.7` + - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true` + - `unlist`: Unlist all detected NSFW content? Default: `false` + - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` + """ + alias Pleroma.Config + alias Pleroma.Constants + alias Pleroma.HTTP + alias Pleroma.User + + require Logger + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @policy :mrf_nsfw_api + + def build_request_url(url) do + Config.get([@policy, :url]) + |> URI.parse() + |> fix_path() + |> Map.put(:query, "url=#{url}") + |> URI.to_string() + end + + def parse_url(url) do + request = build_request_url(url) + + with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do + Jason.decode(body) + else + error -> + Logger.warn(""" + [NsfwApiPolicy]: The API server failed. Skipping. + #{inspect(error)} + """) + + error + end + end + + def check_url_nsfw(url) when is_binary(url) do + threshold = Config.get([@policy, :threshold]) + + case parse_url(url) do + {:ok, %{"score" => score}} when score >= threshold -> + {:nsfw, %{url: url, score: score, threshold: threshold}} + + {:ok, %{"score" => score}} -> + {:sfw, %{url: url, score: score, threshold: threshold}} + + _ -> + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + end + + def check_url_nsfw(%{"href" => url}) when is_binary(url) do + check_url_nsfw(url) + end + + def check_url_nsfw(url) do + threshold = Config.get([@policy, :threshold]) + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + + def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do + if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do + {:sfw, attachment} + else + {:nsfw, attachment} + end + end + + def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do + case check_url_nsfw(url) do + {:sfw, _} -> {:sfw, attachment} + {:nsfw, _} -> {:nsfw, attachment} + end + end + + def check_attachment_nsfw(attachment), do: {:sfw, attachment} + + def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do + if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do + {:sfw, object} + else + {:nsfw, object} + end + end + + def check_object_nsfw(%{"object" => %{} = child_object} = object) do + case check_object_nsfw(child_object) do + {:sfw, _} -> {:sfw, object} + {:nsfw, _} -> {:nsfw, object} + end + end + + def check_object_nsfw(object), do: {:sfw, object} + + @impl true + def filter(object) do + with {:sfw, object} <- check_object_nsfw(object) do + {:ok, object} + else + {:nsfw, _data} -> handle_nsfw(object) + _ -> {:reject, "NSFW: Attachment rejected"} + end + end + + defp handle_nsfw(object) do + if Config.get([@policy, :reject]) do + {:reject, object} + else + {:ok, + object + |> maybe_unlist() + |> maybe_mark_sensitive()} + end + end + + defp maybe_unlist(object) do + if Config.get([@policy, :unlist]) do + unlist(object) + else + object + end + end + + defp maybe_mark_sensitive(object) do + if Config.get([@policy, :mark_sensitive]) do + mark_sensitive(object) + else + object + end + end + + def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + with %User{} = user <- User.get_cached_by_ap_id(actor) do + to = + [user.follower_address | to] + |> List.delete(Constants.as_public()) + |> Enum.uniq() + + cc = + [Constants.as_public() | cc] + |> List.delete(user.follower_address) + |> Enum.uniq() + + object + |> Map.put("to", to) + |> Map.put("cc", cc) + else + _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}" + end + end + + def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do + Map.put(object, "object", mark_sensitive(child_object)) + end + + def mark_sensitive(object) when is_map(object) do + tags = (object["tag"] || []) ++ ["nsfw"] + + object + |> Map.put("tag", tags) + |> Map.put("sensitive", true) + end + + # Hackney needs a trailing slash + defp fix_path(%URI{path: path} = uri) when is_binary(path) do + path = String.trim_trailing(path, "/") <> "/" + Map.put(uri, :path, path) + end + + defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/") + + @impl true + def describe do + options = %{ + threshold: Config.get([@policy, :threshold]), + mark_sensitive: Config.get([@policy, :mark_sensitive]), + unlist: Config.get([@policy, :unlist]), + reject: Config.get([@policy, :reject]) + } + + {:ok, %{@policy => options}} + end + + @impl true + def config_description do + %{ + key: @policy, + related_policy: to_string(__MODULE__), + label: "NSFW API Policy", + description: + "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.", + children: [ + %{ + key: :url, + type: :string, + description: "Base URL of the API server.", + suggestions: ["http://127.0.0.1:5000/"] + }, + %{ + key: :threshold, + type: :float, + description: "Lowest score to take action on. Between 0 and 1.", + suggestions: [0.7] + }, + %{ + key: :mark_sensitive, + type: :boolean, + description: "Mark sensitive all detected NSFW content?", + suggestions: [true] + }, + %{ + key: :unlist, + type: :boolean, + description: "Unlist sensitive all detected NSFW content?", + suggestions: [false] + }, + %{ + key: :reject, + type: :boolean, + description: "Reject sensitive all detected NSFW content (takes precedence)?", + suggestions: [false] + } + ] + } + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 72975f348..5ee9e7549 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string, default: "Link") field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:name, :string) + field(:summary, :string) field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do @@ -44,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do |> fix_url() struct - |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash]) |> cast_embed(:url, with: &url_changeset/2, required: true) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType]) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 7421b8ed8..60b4d5f1b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -21,7 +21,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Push alias Pleroma.Web.Streamer alias Pleroma.Workers.PollWorker @@ -125,7 +124,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do nil end - {:ok, notifications} = Notification.create_notifications(object, do_send: false) + {:ok, notifications} = Notification.create_notifications(object) meta = meta @@ -184,7 +183,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do liked_object = Object.get_by_ap_id(object.data["object"]) Utils.add_like_to_object(object, liked_object) - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -202,7 +205,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do - {:ok, notifications} = Notification.create_notifications(activity, do_send: false) + {:ok, notifications} = Notification.create_notifications(activity) {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) {:ok, _user} = ActivityPub.update_last_status_at_if_public(user, object) @@ -256,11 +259,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Utils.add_announce_to_object(object, announced_object) - if !User.internal?(user) do - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) - ap_streamer().stream_out(object) - end + if !User.internal?(user), do: ap_streamer().stream_out(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -281,7 +286,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do reacted_object = Object.get_by_ap_id(object.data["object"]) Utils.add_emoji_reaction_to_object(object, reacted_object) - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -585,10 +594,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do defp send_notifications(meta) do Keyword.get(meta, :notifications, []) - |> Enum.each(fn notification -> - Streamer.stream(["user", "user:notification"], notification) - Push.send(notification) - end) + |> Notification.send() meta end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 52cb64fc5..797e79dda 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -721,14 +721,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do #### Flag-related helpers @spec make_flag_data(map(), map()) :: map() - def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do + def make_flag_data( + %{actor: actor, context: context, content: content} = params, + additional + ) do %{ "type" => "Flag", "actor" => actor.ap_id, "content" => content, "object" => build_flag_object(params), "context" => context, - "state" => "open" + "state" => "open", + "rules" => Map.get(params, :rules, nil) } |> Map.merge(additional) end diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex new file mode 100644 index 000000000..43b2f209a --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex @@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + alias Pleroma.Rule + alias Pleroma.Web.Plugs.OAuthScopesPlug + + import Pleroma.Web.ControllerHelper, + only: [ + json_response: 3 + ] + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write"]} + when action in [:create, :update, :delete] + ) + + plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index) + + action_fallback(AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation + + def index(conn, _) do + rules = + Rule.query() + |> Repo.all() + + render(conn, "index.json", rules: rules) + end + + def create(%{body_params: params} = conn, _) do + rule = + params + |> Rule.create() + + render(conn, "show.json", rule: rule) + end + + def update(%{body_params: params} = conn, %{id: id}) do + rule = + params + |> Rule.update(id) + + render(conn, "show.json", rule: rule) + end + + def delete(conn, %{id: id}) do + with {:ok, _} <- Rule.delete(id) do + json(conn, %{}) + else + _ -> json_response(conn, :bad_request, "") + end + end +end diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index b761dbb22..b4b0be267 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -6,9 +6,11 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view alias Pleroma.HTML + alias Pleroma.Rule alias Pleroma.User alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.AdminAPI.RuleView alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.StatusView @@ -46,7 +48,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do as: :activity }), state: report.data["state"], - notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}) + notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}), + rules: rules(Map.get(report.data, "rules", nil)) } end @@ -71,4 +74,16 @@ defmodule Pleroma.Web.AdminAPI.ReportView do created_at: Utils.to_masto_date(inserted_at) } end + + defp rules(nil) do + [] + end + + defp rules(rule_ids) do + rules = + rule_ids + |> Rule.get() + + render(RuleView, "index.json", rules: rules) + end end diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex new file mode 100644 index 000000000..606443f05 --- /dev/null +++ b/lib/pleroma/web/admin_api/views/rule_view.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleView do + use Pleroma.Web, :view + + require Pleroma.Constants + + def render("index.json", %{rules: rules} = _opts) do + render_many(rules, __MODULE__, "show.json") + end + + def render("show.json", %{rule: rule} = _opts) do + %{ + id: to_string(rule.id), + priority: rule.priority, + text: rule.text, + hint: rule.hint + } + end +end diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 10d221571..314782818 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -97,6 +97,7 @@ defmodule Pleroma.Web.ApiSpec do "Frontend management", "Instance configuration", "Instance documents", + "Instance rule managment", "Invites", "MediaProxy cache", "OAuth application management", diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 36025e47a..85f02166f 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do alias Pleroma.Web.ApiSpec.Schemas.ActorType alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.List alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -513,6 +514,48 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end + def familiar_followers_operation do + %Operation{ + tags: ["Retrieve account information"], + summary: "Followers that you follow", + operationId: "AccountController.familiar_followers", + description: + "Obtain a list of all accounts that follow a given account, filtered for accounts you follow.", + security: [%{"oAuth" => ["read:follows"]}], + parameters: [ + Operation.parameter( + :id, + :query, + %Schema{ + oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}] + }, + "Account IDs", + example: "123" + ) + ], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: %Schema{ + title: "Account", + type: :object, + properties: %{ + id: FlakeID, + accounts: %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: Account, + example: [Account.schema().example] + } + } + } + }) + } + } + end + defp create_request do %Schema{ title: "AccountCreateRequest", diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index fbb6896a9..25a604beb 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -31,6 +31,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do "Filter by report state" ), Operation.parameter( + :rule_id, + :query, + %Schema{type: :string}, + "Filter by selected rule id" + ), + Operation.parameter( :limit, :query, %Schema{type: :integer}, @@ -169,6 +175,17 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do inserted_at: %Schema{type: :string, format: :"date-time"} } } + }, + rules: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + text: %Schema{type: :string}, + hint: %Schema{type: :string, nullable: true} + } + } } } } diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex new file mode 100644 index 000000000..c3a3ecc7c --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -0,0 +1,115 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Retrieve list of instance rules", + operationId: "AdminAPI.RuleController.index", + security: [%{"oAuth" => ["admin:read"]}], + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :array, + items: rule() + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Create new rule", + operationId: "AdminAPI.RuleController.create", + security: [%{"oAuth" => ["admin:write"]}], + parameters: admin_api_params(), + requestBody: request_body("Parameters", create_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def update_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Modify existing rule", + operationId: "AdminAPI.RuleController.update", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Delete rule", + operationId: "AdminAPI.RuleController.delete", + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + security: [%{"oAuth" => ["admin:write"]}], + responses: %{ + 200 => empty_object_response(), + 404 => Operation.response("Not Found", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + defp create_request do + %Schema{ + type: :object, + required: [:text], + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + end + + defp update_request do + %Schema{ + type: :object, + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + end + + defp rule do + %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string, nullable: true} + } + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 708b74b12..7d7a5ecc1 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -46,10 +46,30 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + def rules_operation do + %Operation{ + tags: ["Instance misc"], + summary: "Retrieve list of instance rules", + operationId: "InstanceController.rules", + responses: %{ + 200 => Operation.response("Array of domains", "application/json", array_of_rules()) + } + } + end + defp instance do %Schema{ type: :object, properties: %{ + accounts: %Schema{ + type: :object, + properties: %{ + max_featured_tags: %Schema{ + type: :integer, + description: "The maximum number of featured tags allowed for each account." + } + } + }, uri: %Schema{type: :string, description: "The domain name of the instance"}, title: %Schema{type: :string, description: "The title of the website"}, description: %Schema{ @@ -172,7 +192,8 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do "urls" => %{ "streaming_api" => "wss://lain.com" }, - "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)" + "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)", + "rules" => array_of_rules() } } end @@ -272,6 +293,19 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do type: :object, description: "Instance configuration", properties: %{ + accounts: %Schema{ + type: :object, + properties: %{ + max_featured_tags: %Schema{ + type: :integer, + description: "The maximum number of featured tags allowed for each account." + }, + max_pinned_statuses: %Schema{ + type: :integer, + description: "The maximum number of pinned statuses for each account." + } + } + }, urls: %Schema{ type: :object, properties: %{ @@ -285,6 +319,11 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do type: :object, description: "A map with poll limits for local statuses", properties: %{ + characters_reserved_per_url: %Schema{ + type: :integer, + description: + "Each URL in a status will be assumed to be exactly this many characters." + }, max_characters: %Schema{ type: :integer, description: "Posts character limit (CW/Subject included in the counter)" @@ -344,4 +383,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do example: ["pleroma.site", "lain.com", "bikeshed.party"] } end + + defp array_of_rules do + %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + } + end end diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 757429d12..a79eb8f74 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -202,7 +202,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "pleroma:report", "move", "follow_request", - "poll" + "poll", + "status" ], description: """ The type of event that resulted in the notification. @@ -216,6 +217,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do - `pleroma:emoji_reaction` - Someone reacted with emoji to your status - `pleroma:chat_mention` - Someone mentioned you in a chat message - `pleroma:report` - Someone was reported + - `status` - Someone you are subscribed to created a status """ } end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex index a994345db..0e2865191 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema - alias Pleroma.Web.ApiSpec.NotificationOperation alias Pleroma.Web.ApiSpec.Schemas.ApiError import Pleroma.Web.ApiSpec.Helpers @@ -35,12 +34,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do Operation.response( "A Notification or array of Notifications", "application/json", - %Schema{ - anyOf: [ - %Schema{type: :array, items: NotificationOperation.notification()}, - NotificationOperation.notification() - ] - } + %Schema{type: :string} ), 400 => Operation.response("Bad Request", "application/json", ApiError) } diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex index c74ac7d5f..f5f88974c 100644 --- a/lib/pleroma/web/api_spec/operations/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/report_operation.ex @@ -53,6 +53,12 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do default: false, description: "If the account is remote, should the report be forwarded to the remote admin?" + }, + rule_ids: %Schema{ + type: :array, + nullable: true, + items: %Schema{type: :string}, + description: "Array of rules" } }, required: [:account_id], @@ -60,7 +66,8 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do "account_id" => "123", "status_ids" => ["1337"], "comment" => "bad status!", - "forward" => "false" + "forward" => "false", + "rule_ids" => ["3"] } } end diff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex index 2871b5f99..4104ed25c 100644 --- a/lib/pleroma/web/api_spec/schemas/attachment.ex +++ b/lib/pleroma/web/api_spec/schemas/attachment.ex @@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do pleroma: %Schema{ type: :object, properties: %{ - mime_type: %Schema{type: :string, description: "mime type of the attachment"} + mime_type: %Schema{type: :string, description: "mime type of the attachment"}, + name: %Schema{ + type: :string, + description: "Name of the attachment, typically the filename" + } } } }, diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 27e82ecc8..34e480d73 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Formatter alias Pleroma.ModerationLog alias Pleroma.Object + alias Pleroma.Rule alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.UserRelationship @@ -568,14 +569,16 @@ defmodule Pleroma.Web.CommonAPI do def report(user, data) do with {:ok, account} <- get_reported_account(data.account_id), {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]), - {:ok, statuses} <- get_report_statuses(account, data) do + {:ok, statuses} <- get_report_statuses(account, data), + rules <- get_report_rules(Map.get(data, :rule_ids, nil)) do ActivityPub.flag(%{ context: Utils.generate_context_id(), actor: user, account: account, statuses: statuses, content: content_html, - forward: Map.get(data, :forward, false) + forward: Map.get(data, :forward, false), + rules: rules }) end end @@ -587,6 +590,15 @@ defmodule Pleroma.Web.CommonAPI do end end + defp get_report_rules(nil) do + nil + end + + defp get_report_rules(rule_ids) do + rule_ids + |> Enum.filter(&Rule.exists?/1) + end + def update_report_state(activity_ids, state) when is_list(activity_ids) do case Utils.update_report_state(activity_ids, state) do :ok -> {:ok, activity_ids} diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index bc46a8a36..8aa1e258d 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -129,8 +129,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft - defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do - %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)} + defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do + add_error(draft, dgettext("errors", "Cannot reply to a deleted status")) + end + + defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do + activity = Activity.get_by_id(id) + + params = + if is_nil(activity) do + # Deleted activities are returned as nil + Map.put(params, :in_reply_to_status_id, :deleted) + else + Map.put(params, :in_reply_to_status_id, activity) + end + + in_reply_to(%{draft | params: params}) end defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 2e2104904..fef907ace 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -38,6 +38,8 @@ defmodule Pleroma.Web.Endpoint do plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) + plug(Pleroma.Web.Plugs.LoggerMetadataPath) + plug(Pleroma.Web.Plugs.SetLocalePlug) plug(CORSPlug) plug(Pleroma.Web.Plugs.HTTPSecurityPlug) diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 1f2c3835a..4b30fd21d 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -44,7 +44,7 @@ defmodule Pleroma.Web.Federator do end def incoming_ap_doc(%{"type" => "Delete"} = params) do - ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3, queue: :slow) end def incoming_ap_doc(params) do diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 9226a2deb..47e6f0a64 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do %{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock] ) - plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships) + plug( + OAuthScopesPlug, + %{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers] + ) plug( OAuthScopesPlug, @@ -629,6 +632,35 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do ) end + @doc "GET /api/v1/accounts/familiar_followers" + def familiar_followers( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, + _id + ) do + users = + User.get_all_by_ids(List.wrap(id)) + |> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)}) + + conn + |> render("familiar_followers.json", + for: user, + users: users, + as: :user + ) + end + + defp get_familiar_followers(%{id: id} = user, %{id: id}) do + User.get_familiar_followers(user, user) + end + + defp get_familiar_followers(%{hide_followers: true}, _current_user) do + [] + end + + defp get_familiar_followers(user, current_user) do + User.get_familiar_followers(user, current_user) + end + @doc "GET /api/v1/identity_proofs" def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params) end diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 3e664903a..b97b0e476 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -25,4 +25,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) end + + @doc "GET /api/v1/instance/rules" + def rules(conn, _params) do + render(conn, "rules.json") + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index e305aea94..afd83b785 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -34,6 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do pleroma:emoji_reaction poll update + status } # GET /api/v1/notifications diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 267c3e3ed..6976ca6e5 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -193,6 +193,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render_many(targets, AccountView, "relationship.json", render_opts) end + def render("familiar_followers.json", %{users: users} = opts) do + opts = + opts + |> Map.merge(%{as: :user}) + |> Map.delete(:users) + + users + |> render_many(AccountView, "familiar_followers.json", opts) + end + + def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do + accounts = + accounts + |> render_many(AccountView, "show.json", opts) + |> Enum.filter(&Enum.any?/1) + + %{id: id, accounts: accounts} + end + defp do_render("show.json", %{user: user} = opts) do self = opts[:for] == user diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 210b46d2c..913684928 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -76,12 +76,26 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }) end + def render("rules.json", _) do + Pleroma.Rule.query() + |> Pleroma.Repo.all() + |> render_many(__MODULE__, "rule.json", as: :rule) + end + + def render("rule.json", %{rule: rule}) do + %{ + id: to_string(rule.id), + text: rule.text, + hint: rule.hint || "" + } + end + defp common_information(instance) do %{ - title: Keyword.get(instance, :name), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", languages: Keyword.get(instance, :languages, ["en"]), - rules: [] + rules: render(__MODULE__, "rules.json"), + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})" } end @@ -138,6 +152,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def federation do quarantined = Config.get([:instance, :quarantined_instances], []) + rejected = Config.get([:instance, :rejected_instances], []) if Config.get([:mrf, :transparency]) do {:ok, data} = MRF.describe() @@ -157,6 +172,12 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) |> Map.new() }) + |> Map.put( + :rejected_instances, + rejected + |> Enum.map(fn {instance, reason} -> {instance, %{"reason" => reason}} end) + |> Map.new() + ) else %{} end @@ -213,6 +234,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do defp configuration2 do configuration() + |> put_in([:accounts, :max_pinned_statuses], Config.get([:instance, :max_pinned_statuses], 0)) + |> put_in([:statuses, :characters_reserved_per_url], 0) |> Map.merge(%{ urls: %{ streaming: Pleroma.Web.Endpoint.websocket_url(), diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 2a51f3755..3f2478719 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -108,6 +108,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do "mention" -> put_status(response, activity, reading_user, status_render_opts) + "status" -> + put_status(response, activity, reading_user, status_render_opts) + "favourite" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c945290c1..0c16749a4 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -624,6 +624,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do to_string(attachment["id"] || hash_id) end + description = + if attachment["summary"] do + HTML.strip_tags(attachment["summary"]) + else + attachment["name"] + end + + name = if attachment["summary"], do: attachment["name"] + + pleroma = + %{mime_type: media_type} + |> Maps.put_if_present(:name, name) + %{ id: attachment_id, url: href, @@ -631,8 +644,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do preview_url: href_preview, text_url: href, type: type, - description: attachment["name"], - pleroma: %{mime_type: media_type}, + description: description, + pleroma: pleroma, blurhash: attachment["blurhash"] } |> Maps.put_if_present(:meta, meta) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index c11484ecb..0b446e0a6 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -54,9 +54,10 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do defp handle_preview(conn, url) do media_proxy_url = MediaProxy.url(url) + http_client_opts = Pleroma.Config.get([:media_proxy, :proxy_opts, :http], pool: :media) with {:ok, %{status: status} = head_response} when status in 200..299 <- - Pleroma.HTTP.request(:head, media_proxy_url, "", [], pool: :media) do + Pleroma.HTTP.request(:head, media_proxy_url, "", [], http_client_opts) do content_type = Tesla.get_header(head_response, "content-type") content_length = Tesla.get_header(head_response, "content-length") content_length = content_length && String.to_integer(content_length) diff --git a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex index f860eaf7e..435ccfabe 100644 --- a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex @@ -23,8 +23,9 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do } = conn, _ ) do - with {:ok, notification} <- Notification.read_one(user, notification_id) do - render(conn, "show.json", notification: notification, for: user) + with {:ok, _} <- Notification.read_one(user, notification_id) do + conn + |> json("ok") else {:error, message} -> conn @@ -38,11 +39,14 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do conn, _ ) do - notifications = - user - |> Notification.set_read_up_to(max_id) - |> Enum.take(80) - - render(conn, "index.json", notifications: notifications, for: user) + with {:ok, _} <- Notification.set_read_up_to(user, max_id) do + conn + |> json("ok") + else + {:error, message} -> + conn + |> put_status(:bad_request) + |> json(%{"error" => message}) + end end end diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index a27dcd0ab..38f6c511e 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -3,26 +3,27 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do - alias Pleroma.Config import Plug.Conn require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + def init(opts), do: opts def call(conn, _options) do - if Config.get([:http_security, :enabled]) do + if @config_impl.get([:http_security, :enabled]) do conn |> merge_resp_headers(headers()) - |> maybe_send_sts_header(Config.get([:http_security, :sts])) + |> maybe_send_sts_header(@config_impl.get([:http_security, :sts])) else conn end end def primary_frontend do - with %{"name" => frontend} <- Config.get([:frontends, :primary]), - available <- Config.get([:frontends, :available]), + with %{"name" => frontend} <- @config_impl.get([:frontends, :primary]), + available <- @config_impl.get([:frontends, :available]), %{} = primary_frontend <- Map.get(available, frontend) do {:ok, primary_frontend} end @@ -37,8 +38,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end def headers do - referrer_policy = Config.get([:http_security, :referrer_policy]) - report_uri = Config.get([:http_security, :report_uri]) + referrer_policy = @config_impl.get([:http_security, :referrer_policy]) + report_uri = @config_impl.get([:http_security, :report_uri]) custom_http_frontend_headers = custom_http_frontend_headers() headers = [ @@ -86,10 +87,10 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do @csp_start [Enum.join(static_csp_rules, ";") <> ";"] defp csp_string do - scheme = Config.get([Pleroma.Web.Endpoint, :url])[:scheme] + scheme = @config_impl.get([Pleroma.Web.Endpoint, :url])[:scheme] static_url = Pleroma.Web.Endpoint.static_url() websocket_url = Pleroma.Web.Endpoint.websocket_url() - report_uri = Config.get([:http_security, :report_uri]) + report_uri = @config_impl.get([:http_security, :report_uri]) img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" @@ -97,8 +98,8 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do # Strict multimedia CSP enforcement only when MediaProxy is enabled {img_src, media_src, connect_src} = - if Config.get([:media_proxy, :enabled]) && - !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do + if @config_impl.get([:media_proxy, :enabled]) && + !@config_impl.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do sources = build_csp_multimedia_source_list() { @@ -115,17 +116,21 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end connect_src = - if Config.get(:env) == :dev do + if @config_impl.get([:env]) == :dev do [connect_src, " http://localhost:3035/"] else connect_src end script_src = - if Config.get(:env) == :dev do - "script-src 'self' 'unsafe-eval'" + if @config_impl.get([:http_security, :allow_unsafe_eval]) do + if @config_impl.get([:env]) == :dev do + "script-src 'self' 'unsafe-eval'" + else + "script-src 'self' 'wasm-unsafe-eval'" + end else - "script-src 'self' 'wasm-unsafe-eval'" + "script-src 'self'" end report = if report_uri, do: ["report-uri ", report_uri, ";report-to csp-endpoint"] @@ -161,11 +166,11 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do defp build_csp_multimedia_source_list do media_proxy_whitelist = [:media_proxy, :whitelist] - |> Config.get() + |> @config_impl.get() |> build_csp_from_whitelist([]) - captcha_method = Config.get([Pleroma.Captcha, :method]) - captcha_endpoint = Config.get([captcha_method, :endpoint]) + captcha_method = @config_impl.get([Pleroma.Captcha, :method]) + captcha_endpoint = @config_impl.get([captcha_method, :endpoint]) base_endpoints = [ @@ -173,7 +178,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do [Pleroma.Upload, :base_url], [Pleroma.Uploaders.S3, :public_endpoint] ] - |> Enum.map(&Config.get/1) + |> Enum.map(&@config_impl.get/1) [captcha_endpoint | base_endpoints] |> Enum.map(&build_csp_param/1) @@ -200,7 +205,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do end def warn_if_disabled do - unless Config.get([:http_security, :enabled]) do + unless Pleroma.Config.get([:http_security, :enabled]) do Logger.warning(" .i;;;;i. iYcviii;vXY: @@ -245,8 +250,8 @@ your instance and your users via malicious posts: end defp maybe_send_sts_header(conn, true) do - max_age_sts = Config.get([:http_security, :sts_max_age]) - max_age_ct = Config.get([:http_security, :ct_max_age]) + max_age_sts = @config_impl.get([:http_security, :sts_max_age]) + max_age_ct = @config_impl.get([:http_security, :ct_max_age]) merge_resp_headers(conn, [ {"strict-transport-security", "max-age=#{max_age_sts}; includeSubDomains"}, diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index e814efc2c..6bf2dd432 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -3,10 +3,22 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do + alias Pleroma.Helpers.InetHelper + import Plug.Conn import Phoenix.Controller, only: [get_format: 1, text: 2] + + alias Pleroma.Web.ActivityPub.MRF + require Logger + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + @http_signatures_impl Application.compile_env( + :pleroma, + [__MODULE__, :http_signatures_impl], + HTTPSignatures + ) + def init(options) do options end @@ -19,7 +31,9 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do if get_format(conn) in ["json", "activity+json"] do conn |> maybe_assign_valid_signature() + |> maybe_assign_actor_id() |> maybe_require_signature() + |> maybe_filter_requests() else conn end @@ -33,7 +47,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do |> put_req_header("(request-target)", request_target) |> put_req_header("@request-target", request_target) - HTTPSignatures.validate_conn(conn) + @http_signatures_impl.validate_conn(conn) end defp validate_signature(conn) do @@ -83,20 +97,63 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do end end + defp maybe_assign_actor_id(%{assigns: %{valid_signature: true}} = conn) do + adapter = Application.get_env(:http_signatures, :adapter) + + {:ok, actor_id} = adapter.get_actor_id(conn) + + assign(conn, :actor_id, actor_id) + end + + defp maybe_assign_actor_id(conn), do: conn + defp has_signature_header?(conn) do conn |> get_req_header("signature") |> Enum.at(0, false) end defp maybe_require_signature(%{assigns: %{valid_signature: true}} = conn), do: conn - defp maybe_require_signature(conn) do - if Pleroma.Config.get([:activitypub, :authorized_fetch_mode], false) do + defp maybe_require_signature(%{remote_ip: remote_ip} = conn) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) do + exceptions = + @config_impl.get([:activitypub, :authorized_fetch_mode_exceptions], []) + |> Enum.map(&InetHelper.parse_cidr/1) + + if Enum.any?(exceptions, fn x -> InetCidr.contains?(x, remote_ip) end) do + conn + else + conn + |> put_status(:unauthorized) + |> text("Request not signed") + |> halt() + end + else conn - |> put_status(:unauthorized) - |> text("Request not signed") - |> halt() + end + end + + defp maybe_filter_requests(%{halted: true} = conn), do: conn + + defp maybe_filter_requests(conn) do + if @config_impl.get([:activitypub, :authorized_fetch_mode], false) and + conn.assigns[:actor_id] do + %{host: host} = URI.parse(conn.assigns.actor_id) + + if MRF.subdomain_match?(rejected_domains(), host) do + conn + |> put_status(:unauthorized) + |> halt() + else + conn + end else conn end end + + defp rejected_domains do + @config_impl.get([:instance, :rejected_instances]) + |> Pleroma.Web.ActivityPub.MRF.instance_list_from_tuples() + |> Pleroma.Web.ActivityPub.MRF.subdomains_regex() + end end diff --git a/lib/pleroma/web/plugs/logger_metadata_path.ex b/lib/pleroma/web/plugs/logger_metadata_path.ex new file mode 100644 index 000000000..a5553cfc8 --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_path.ex @@ -0,0 +1,12 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataPath do + def init(opts), do: opts + + def call(conn, _) do + Logger.metadata(path: conn.request_path) + conn + end +end diff --git a/lib/pleroma/web/plugs/logger_metadata_user.ex b/lib/pleroma/web/plugs/logger_metadata_user.ex new file mode 100644 index 000000000..6a5c0041d --- /dev/null +++ b/lib/pleroma/web/plugs/logger_metadata_user.ex @@ -0,0 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.Plugs.LoggerMetadataUser do + alias Pleroma.User + + def init(opts), do: opts + + def call(%{assigns: %{user: user = %User{}}} = conn, _) do + Logger.metadata(user: user.nickname) + conn + end + + def call(conn, _) do + conn + end +end diff --git a/lib/pleroma/web/plugs/remote_ip.ex b/lib/pleroma/web/plugs/remote_ip.ex index 9f733a96f..3a4bffb50 100644 --- a/lib/pleroma/web/plugs/remote_ip.ex +++ b/lib/pleroma/web/plugs/remote_ip.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.Plugs.RemoteIp do """ alias Pleroma.Config + alias Pleroma.Helpers.InetHelper import Plug.Conn @behaviour Plug @@ -30,19 +31,8 @@ defmodule Pleroma.Web.Plugs.RemoteIp do proxies = Config.get([__MODULE__, :proxies], []) |> Enum.concat(reserved) - |> Enum.map(&maybe_add_cidr/1) + |> Enum.map(&InetHelper.parse_cidr/1) {headers, proxies} end - - defp maybe_add_cidr(proxy) when is_binary(proxy) do - proxy = - cond do - "/" in String.codepoints(proxy) -> proxy - InetCidr.v4?(InetCidr.parse_address!(proxy)) -> proxy <> "/32" - InetCidr.v6?(InetCidr.parse_address!(proxy)) -> proxy <> "/128" - end - - InetCidr.parse_cidr!(proxy, true) - end end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 36f44d8e8..9e68d827b 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -192,6 +192,7 @@ defmodule Pleroma.Web.Push.Impl do def format_title(%{type: type}, mastodon_type) do case mastodon_type || type do "mention" -> "New Mention" + "status" -> "New Status" "follow" -> "New Follower" "follow_request" -> "New Follow Request" "reblog" -> "New Repeat" diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 119994458..ea41bd285 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp http_options do [ - pool: :media, + pool: :rich_media, max_body: Config.get([:rich_media, :max_body], 5_000_000) ] end diff --git a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex index 948c727e1..1172a120a 100644 --- a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do %URI{host: host, query: query} = URI.parse(image) is_binary(host) and String.contains?(host, "amazonaws.com") and - String.contains?(query, "X-Amz-Expires") + is_binary(query) and String.contains?(query, "X-Amz-Expires") end defp aws_signed_url?(_), do: nil diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 86d6da883..56c457e90 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -29,6 +29,7 @@ defmodule Pleroma.Web.Router do pipeline :browser do plug(:accepts, ["html"]) plug(:fetch_session) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :oauth do @@ -67,12 +68,14 @@ defmodule Pleroma.Web.Router do plug(:fetch_session) plug(:authenticate) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :no_auth_or_privacy_expectations_api do plug(:base_api) plug(:after_auth) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end # Pipeline for app-related endpoints (no user auth checks — app-bound tokens must be supported) @@ -83,12 +86,14 @@ defmodule Pleroma.Web.Router do pipeline :api do plug(:expect_public_instance_or_user_authentication) plug(:no_auth_or_privacy_expectations_api) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :authenticated_api do plug(:expect_user_authentication) plug(:no_auth_or_privacy_expectations_api) plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :admin_api do @@ -99,6 +104,7 @@ defmodule Pleroma.Web.Router do plug(Pleroma.Web.Plugs.EnsureAuthenticatedPlug) plug(Pleroma.Web.Plugs.UserIsStaffPlug) plug(Pleroma.Web.Plugs.IdempotencyPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :require_admin do @@ -179,6 +185,7 @@ defmodule Pleroma.Web.Router do plug(:browser) plug(:authenticate) plug(Pleroma.Web.Plugs.EnsureUserTokenAssignsPlug) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :well_known do @@ -193,6 +200,7 @@ defmodule Pleroma.Web.Router do pipeline :pleroma_api do plug(:accepts, ["html", "json"]) plug(OpenApiSpex.Plug.PutApiSpec, module: Pleroma.Web.ApiSpec) + plug(Pleroma.Web.Plugs.LoggerMetadataUser) end pipeline :mailbox_preview do @@ -292,6 +300,11 @@ defmodule Pleroma.Web.Router do post("/frontends/install", FrontendController, :install) post("/backups", AdminAPIController, :create_backup) + + get("/rules", RuleController, :index) + post("/rules", RuleController, :create) + patch("/rules/:id", RuleController, :update) + delete("/rules/:id", RuleController, :delete) end # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) @@ -633,6 +646,7 @@ defmodule Pleroma.Web.Router do patch("/accounts/update_credentials", AccountController, :update_credentials) get("/accounts/relationships", AccountController, :relationships) + get("/accounts/familiar_followers", AccountController, :familiar_followers) get("/accounts/:id/lists", AccountController, :lists) get("/accounts/:id/identity_proofs", AccountController, :identity_proofs) get("/endorsements", AccountController, :endorsements) @@ -764,6 +778,7 @@ defmodule Pleroma.Web.Router do get("/instance", InstanceController, :show) get("/instance/peers", InstanceController, :peers) + get("/instance/rules", InstanceController, :rules) get("/statuses", StatusController, :index) get("/statuses/:id", StatusController, :show) diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 26fb8af84..e653b3338 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do end end + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def find_lrdd_template(domain) do + @cachex.fetch!(:host_meta_cache, domain, fn _ -> + {:commit, fetch_lrdd_template(domain)} + end) + rescue + e -> {:error, "Cachex error: #{inspect(e)}"} + end + + defp fetch_lrdd_template(domain) do # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 meta_url = "https://#{domain}/.well-known/host-meta" @@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do end end - defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do + defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do case find_lrdd_template(domain) do {:ok, template} -> String.replace(template, "{uri}", encoded_account) @@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do end end + defp get_address_from_domain(domain, account) when is_binary(domain) do + encoded_account = URI.encode("acct:#{account}") + get_address_from_domain(domain, encoded_account) + end + defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain} @spec finger(String.t()) :: {:ok, map()} | {:error, any()} @@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do URI.parse(account).host end - encoded_account = URI.encode("acct:#{account}") - - with address when is_binary(address) <- get_address_from_domain(domain, encoded_account), + with address when is_binary(address) <- get_address_from_domain(domain, account), {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- HTTP.get( address, @@ -216,10 +228,28 @@ defmodule Pleroma.Web.WebFinger do _ -> {:error, {:content_type, nil}} end + |> case do + {:ok, data} -> validate_webfinger(address, data) + error -> error + end else error -> Logger.debug("Couldn't finger #{account}: #{inspect(error)}") error end end + + defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do + with [_name, acct_host] <- String.split(acct, "@"), + {_, url} <- {:address, get_address_from_domain(acct_host, subject)}, + %URI{host: request_host} <- URI.parse(request_url), + %URI{host: acct_host} <- URI.parse(url), + {_, true} <- {:hosts_match, acct_host == request_host} do + {:ok, data} + else + _ -> {:error, {:webfinger_invalid, request_url, data}} + end + end + + defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}} end diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 4c1764053..0b570b70b 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do alias Pleroma.Object alias Pleroma.Repo - use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" + use Pleroma.Workers.WorkerHelper, queue: "slow" @impl Oban.Worker def perform(%Job{ diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index a485ddb4b..54ac31a3c 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.BackupWorker do - use Oban.Worker, queue: :backup, max_attempts: 1 + use Oban.Worker, queue: :slow, max_attempts: 1 alias Oban.Job alias Pleroma.User.Backup diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 1c3e445aa..d2abb2d3b 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(_job) do diff --git a/lib/pleroma/workers/mailer_worker.ex b/lib/pleroma/workers/mailer_worker.ex index 940716558..652bf77e0 100644 --- a/lib/pleroma/workers/mailer_worker.ex +++ b/lib/pleroma/workers/mailer_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MailerWorker do - use Pleroma.Workers.WorkerHelper, queue: "mailer" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "email", "encoded_email" => encoded_email, "config" => config}}) do diff --git a/lib/pleroma/workers/mute_expire_worker.ex b/lib/pleroma/workers/mute_expire_worker.ex index 8ce458d48..8ad287a7f 100644 --- a/lib/pleroma/workers/mute_expire_worker.ex +++ b/lib/pleroma/workers/mute_expire_worker.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.MuteExpireWorker do - use Pleroma.Workers.WorkerHelper, queue: "mute_expire" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "unmute_user", "muter_id" => muter_id, "mutee_id" => mutee_id}}) do diff --git a/lib/pleroma/workers/poll_worker.ex b/lib/pleroma/workers/poll_worker.ex index 022d026f8..70df54193 100644 --- a/lib/pleroma/workers/poll_worker.ex +++ b/lib/pleroma/workers/poll_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.PollWorker do @moduledoc """ Generates notifications when a poll ends. """ - use Pleroma.Workers.WorkerHelper, queue: "poll_notifications" + use Pleroma.Workers.WorkerHelper, queue: "background" alias Pleroma.Activity alias Pleroma.Notification diff --git a/lib/pleroma/workers/purge_expired_activity.ex b/lib/pleroma/workers/purge_expired_activity.ex index e554684fe..a65593b6e 100644 --- a/lib/pleroma/workers/purge_expired_activity.ex +++ b/lib/pleroma/workers/purge_expired_activity.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do Worker which purges expired activity. """ - use Oban.Worker, queue: :activity_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :slow, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -59,7 +59,7 @@ defmodule Pleroma.Workers.PurgeExpiredActivity do def get_expiration(id) do from(j in Oban.Job, where: j.state == "scheduled", - where: j.queue == "activity_expiration", + where: j.queue == "slow", where: fragment("?->>'activity_id' = ?", j.args, ^id) ) |> Pleroma.Repo.one() diff --git a/lib/pleroma/workers/purge_expired_filter.ex b/lib/pleroma/workers/purge_expired_filter.ex index 9114aeb7f..1f6931e4c 100644 --- a/lib/pleroma/workers/purge_expired_filter.ex +++ b/lib/pleroma/workers/purge_expired_filter.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do Worker which purges expired filters """ - use Oban.Worker, queue: :filter_expiration, max_attempts: 1, unique: [period: :infinity] + use Oban.Worker, queue: :background, max_attempts: 1, unique: [period: :infinity] import Ecto.Query @@ -38,7 +38,7 @@ defmodule Pleroma.Workers.PurgeExpiredFilter do def get_expiration(id) do from(j in Job, where: j.state == "scheduled", - where: j.queue == "filter_expiration", + where: j.queue == "background", where: fragment("?->'filter_id' = ?", j.args, ^id) ) |> Repo.one() diff --git a/lib/pleroma/workers/purge_expired_token.ex b/lib/pleroma/workers/purge_expired_token.ex index 2ccd9e80b..1854bf561 100644 --- a/lib/pleroma/workers/purge_expired_token.ex +++ b/lib/pleroma/workers/purge_expired_token.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.PurgeExpiredToken do Worker which purges expired OAuth tokens """ - use Oban.Worker, queue: :token_expiration, max_attempts: 1 + use Oban.Worker, queue: :background, max_attempts: 1 @spec enqueue(%{token_id: integer(), valid_until: DateTime.t(), mod: module()}) :: {:ok, Oban.Job.t()} | {:error, Ecto.Changeset.t()} diff --git a/lib/pleroma/workers/remote_fetcher_worker.ex b/lib/pleroma/workers/remote_fetcher_worker.ex index c26418483..ed04c54b2 100644 --- a/lib/pleroma/workers/remote_fetcher_worker.ex +++ b/lib/pleroma/workers/remote_fetcher_worker.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Workers.RemoteFetcherWorker do alias Pleroma.Object.Fetcher - use Pleroma.Workers.WorkerHelper, queue: "remote_fetcher" + use Pleroma.Workers.WorkerHelper, queue: "background" @impl Oban.Worker def perform(%Job{args: %{"op" => "fetch_remote", "id" => id} = args}) do diff --git a/lib/pleroma/workers/rich_media_expiration_worker.ex b/lib/pleroma/workers/rich_media_expiration_worker.ex index d7ae497a7..0b74687cf 100644 --- a/lib/pleroma/workers/rich_media_expiration_worker.ex +++ b/lib/pleroma/workers/rich_media_expiration_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.RichMediaExpirationWorker do alias Pleroma.Web.RichMedia.Card use Oban.Worker, - queue: :rich_media_expiration + queue: :background @impl Oban.Worker def perform(%Job{args: %{"url" => url} = _args}) do diff --git a/lib/pleroma/workers/scheduled_activity_worker.ex b/lib/pleroma/workers/scheduled_activity_worker.ex index 4df84d00f..ab62686f4 100644 --- a/lib/pleroma/workers/scheduled_activity_worker.ex +++ b/lib/pleroma/workers/scheduled_activity_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.ScheduledActivityWorker do The worker to post scheduled activity. """ - use Pleroma.Workers.WorkerHelper, queue: "scheduled_activities" + use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing" alias Pleroma.Repo alias Pleroma.ScheduledActivity |