diff options
Diffstat (limited to 'lib')
61 files changed, 1857 insertions, 356 deletions
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index f2b234022..d266d1836 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -109,7 +109,8 @@ defmodule Pleroma.Application do streamer_registry() ++ background_migrators() ++ shout_child(shout_enabled?()) ++ - [Pleroma.Gopher.Server] + [Pleroma.Gopher.Server] ++ + [Pleroma.Search.Healthcheck] # See http://elixir-lang.org/docs/stable/elixir/Supervisor.html # for other strategies and supported options @@ -156,12 +157,14 @@ defmodule Pleroma.Application do build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), + build_cachex("failed_media_helper_url", default_ttl: :timer.minutes(15), limit: 2_500), build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000), build_cachex("chat_message_id_idempotency_key", expiration: chat_message_id_idempotency_key_expiration(), limit: 500_000 ), - build_cachex("rel_me", limit: 2500) + build_cachex("rel_me", limit: 2500), + build_cachex("host_meta", default_ttl: :timer.minutes(120), limit: 5000) ] end diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 819245481..8c0df64fc 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -28,6 +28,7 @@ defmodule Pleroma.ApplicationRequirements do |> check_welcome_message_config!() |> check_rum!() |> check_repo_pool_size!() + |> check_mrfs() |> handle_result() end @@ -234,4 +235,25 @@ defmodule Pleroma.ApplicationRequirements do true end end + + defp check_mrfs(:ok) do + mrfs = Config.get!([:mrf, :policies]) + + missing_mrfs = + Enum.reduce(mrfs, [], fn x, acc -> + if Code.ensure_compiled(x) do + acc + else + acc ++ [x] + end + end) + + if Enum.empty?(missing_mrfs) do + :ok + else + {:error, "The following MRF modules are configured but missing: #{inspect(missing_mrfs)}"} + end + end + + defp check_mrfs(result), do: result end diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index d814b4931..3a5e35301 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -19,7 +19,8 @@ defmodule Pleroma.Constants do "context_id", "deleted_activity_id", "pleroma_internal", - "generator" + "generator", + "rules" ] ) diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index 7864296fa..e44114d9d 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -12,6 +12,8 @@ defmodule Pleroma.Helpers.MediaHelper do require Logger + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + def missing_dependencies do Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> if Pleroma.Utils.command_available?(executable) do @@ -43,29 +45,40 @@ defmodule Pleroma.Helpers.MediaHelper do @spec video_framegrab(String.t()) :: {:ok, binary()} | {:error, any()} def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), + false <- @cachex.exists?(:failed_media_helper_cache, url), {:ok, env} <- HTTP.get(url, [], pool: :media), {:ok, pid} <- StringIO.open(env.body) do body_stream = IO.binstream(pid, 1) - result = - Exile.stream!( - [ - executable, - "-i", - "pipe:0", - "-vframes", - "1", - "-f", - "mjpeg", - "pipe:1" - ], - input: body_stream, - ignore_epipe: true, - stderr: :disable - ) - |> Enum.into(<<>>) + task = + Task.async(fn -> + Exile.stream!( + [ + executable, + "-i", + "pipe:0", + "-vframes", + "1", + "-f", + "mjpeg", + "pipe:1" + ], + input: body_stream, + ignore_epipe: true, + stderr: :disable + ) + |> Enum.into(<<>>) + end) + + case Task.yield(task, 5_000) do + nil -> + Task.shutdown(task) + @cachex.put(:failed_media_helper_cache, url, nil) + {:error, {:ffmpeg, :timeout}} - {:ok, result} + result -> + {:ok, result} + end else nil -> {:error, {:ffmpeg, :command_not_found}} {:error, _} = error -> error diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 84ff2f129..4de7cbb76 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -65,20 +65,16 @@ defmodule Pleroma.HTML do end end - @spec extract_first_external_url_from_object(Pleroma.Object.t()) :: - {:ok, String.t()} | {:error, :no_content} + @spec extract_first_external_url_from_object(Pleroma.Object.t()) :: String.t() | nil def extract_first_external_url_from_object(%{data: %{"content" => content}}) when is_binary(content) do - url = - content - |> Floki.parse_fragment!() - |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") - |> Enum.take(1) - |> Floki.attribute("href") - |> Enum.at(0) - - {:ok, url} + content + |> Floki.parse_fragment!() + |> Floki.find("a:not(.mention,.hashtag,.attachment,[rel~=\"tag\"])") + |> Enum.take(1) + |> Floki.attribute("href") + |> Enum.at(0) end - def extract_first_external_url_from_object(_), do: {:error, :no_content} + def extract_first_external_url_from_object(_), do: nil end diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 710b19866..4f714b25f 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -73,6 +73,7 @@ defmodule Pleroma.Notification do pleroma:report reblog poll + status } def changeset(%Notification{} = notification, attrs) do @@ -280,15 +281,10 @@ defmodule Pleroma.Notification do select: n.id ) - {:ok, %{ids: {_, notification_ids}}} = - Multi.new() - |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) - |> Marker.multi_set_last_read_id(user, "notifications") - |> Repo.transaction() - - for_user_query(user) - |> where([n], n.id in ^notification_ids) - |> Repo.all() + Multi.new() + |> Multi.update_all(:ids, query, set: [seen: true, updated_at: NaiveDateTime.utc_now()]) + |> Marker.multi_set_last_read_id(user, "notifications") + |> Repo.transaction() end @spec read_one(User.t(), String.t()) :: @@ -299,10 +295,6 @@ defmodule Pleroma.Notification do |> Multi.update(:update, changeset(notification, %{seen: true})) |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - |> case do - {:ok, %{update: notification}} -> {:ok, notification} - {:error, :update, changeset, _} -> {:error, changeset} - end end end @@ -361,37 +353,38 @@ defmodule Pleroma.Notification do end end - @spec create_notifications(Activity.t(), keyword()) :: {:ok, [Notification.t()] | []} - def create_notifications(activity, options \\ []) + @spec create_notifications(Activity.t()) :: {:ok, [Notification.t()] | []} + def create_notifications(activity) - def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity, options) do + def create_notifications(%Activity{data: %{"to" => _, "type" => "Create"}} = activity) do object = Object.normalize(activity, fetch: false) if object && object.data["type"] == "Answer" do {:ok, []} else - do_create_notifications(activity, options) + do_create_notifications(activity) end end - def create_notifications(%Activity{data: %{"type" => type}} = activity, options) + def create_notifications(%Activity{data: %{"type" => type}} = activity) when type in ["Follow", "Like", "Announce", "Move", "EmojiReact", "Flag", "Update"] do - do_create_notifications(activity, options) + do_create_notifications(activity) end - def create_notifications(_, _), do: {:ok, []} + def create_notifications(_), do: {:ok, []} - defp do_create_notifications(%Activity{} = activity, options) do - do_send = Keyword.get(options, :do_send, true) + defp do_create_notifications(%Activity{} = activity) do + enabled_receivers = get_notified_from_activity(activity) - {enabled_receivers, disabled_receivers} = get_notified_from_activity(activity) - potential_receivers = enabled_receivers ++ disabled_receivers + enabled_subscribers = get_notified_subscribers_from_activity(activity) notifications = - Enum.map(potential_receivers, fn user -> - do_send = do_send && user in enabled_receivers - create_notification(activity, user, do_send: do_send) - end) + (Enum.map(enabled_receivers, fn user -> + create_notification(activity, user) + end) ++ + Enum.map(enabled_subscribers -- enabled_receivers, fn user -> + create_notification(activity, user, type: "status") + end)) |> Enum.reject(&is_nil/1) {:ok, notifications} @@ -450,7 +443,6 @@ defmodule Pleroma.Notification do # TODO move to sql, too. def create_notification(%Activity{} = activity, %User{} = user, opts \\ []) do - do_send = Keyword.get(opts, :do_send, true) type = Keyword.get(opts, :type, type_from_activity(activity)) unless skip?(activity, user, opts) do @@ -465,11 +457,6 @@ defmodule Pleroma.Notification do |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - if do_send do - Streamer.stream(["user", "user:notification"], notification) - Push.send(notification) - end - notification end end @@ -527,13 +514,28 @@ defmodule Pleroma.Notification do |> exclude_relationship_restricted_ap_ids(activity) |> exclude_thread_muter_ap_ids(activity) - notification_enabled_users = - Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) + end + + def get_notified_from_activity(_, _local_only), do: [] + + def get_notified_subscribers_from_activity(activity, local_only \\ true) - {notification_enabled_users, potential_receivers -- notification_enabled_users} + def get_notified_subscribers_from_activity( + %Activity{data: %{"type" => "Create"}} = activity, + local_only + ) do + notification_enabled_ap_ids = + [] + |> Utils.maybe_notify_subscribers(activity) + + potential_receivers = + User.get_users_from_set(notification_enabled_ap_ids, local_only: local_only) + + Enum.filter(potential_receivers, fn u -> u.ap_id in notification_enabled_ap_ids end) end - def get_notified_from_activity(_, _local_only), do: {[], []} + def get_notified_subscribers_from_activity(_, _), do: [] # For some activities, only notify the author of the object def get_potential_receiver_ap_ids(%{data: %{"type" => type, "object" => object_id}}) @@ -576,7 +578,6 @@ defmodule Pleroma.Notification do [] |> Utils.maybe_notify_to_recipients(activity) |> Utils.maybe_notify_mentioned_recipients(activity) - |> Utils.maybe_notify_subscribers(activity) |> Utils.maybe_notify_followers(activity) |> Enum.uniq() end @@ -643,6 +644,7 @@ defmodule Pleroma.Notification do def skip?(%Activity{} = activity, %User{} = user, opts) do [ :self, + :internal, :invisible, :block_from_strangers, :recently_followed, @@ -662,6 +664,12 @@ defmodule Pleroma.Notification do end end + def skip?(:internal, %Activity{} = activity, _user, _opts) do + actor = activity.data["actor"] + user = User.get_cached_by_ap_id(actor) + User.internal?(user) + end + def skip?(:invisible, %Activity{} = activity, _user, _opts) do actor = activity.data["actor"] user = User.get_cached_by_ap_id(actor) @@ -748,4 +756,12 @@ defmodule Pleroma.Notification do ) |> Repo.update_all(set: [seen: true]) end + + @spec send(list(Notification.t())) :: :ok + def send(notifications) do + Enum.each(notifications, fn notification -> + Streamer.stream(["user", "user:notification"], notification) + Push.send(notification) + end) + end end diff --git a/lib/pleroma/rule.ex b/lib/pleroma/rule.ex new file mode 100644 index 000000000..3ba413214 --- /dev/null +++ b/lib/pleroma/rule.ex @@ -0,0 +1,68 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Rule do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.Repo + alias Pleroma.Rule + + schema "rules" do + field(:priority, :integer, default: 0) + field(:text, :string) + field(:hint, :string) + + timestamps() + end + + def changeset(%Rule{} = rule, params \\ %{}) do + rule + |> cast(params, [:priority, :text, :hint]) + |> validate_required([:text]) + end + + def query do + Rule + |> order_by(asc: :priority) + |> order_by(asc: :id) + end + + def get(ids) when is_list(ids) do + from(r in __MODULE__, where: r.id in ^ids) + |> Repo.all() + end + + def get(id), do: Repo.get(__MODULE__, id) + + def exists?(id) do + from(r in __MODULE__, where: r.id == ^id) + |> Repo.exists?() + end + + def create(params) do + {:ok, rule} = + %Rule{} + |> changeset(params) + |> Repo.insert() + + rule + end + + def update(params, id) do + {:ok, rule} = + get(id) + |> changeset(params) + |> Repo.update() + + rule + end + + def delete(id) do + get(id) + |> Repo.delete() + end +end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex index 3b266e59b..fd0218cb8 100644 --- a/lib/pleroma/search.ex +++ b/lib/pleroma/search.ex @@ -10,8 +10,12 @@ defmodule Pleroma.Search do end def search(query, options) do - search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) - + search_module = Pleroma.Config.get([Pleroma.Search, :module]) search_module.search(options[:for_user], query, options) end + + def healthcheck_endpoints do + search_module = Pleroma.Config.get([Pleroma.Search, :module]) + search_module.healthcheck_endpoints + end end diff --git a/lib/pleroma/search/database_search.ex b/lib/pleroma/search/database_search.ex index 31bfc7e33..11e99e7f1 100644 --- a/lib/pleroma/search/database_search.ex +++ b/lib/pleroma/search/database_search.ex @@ -48,6 +48,9 @@ defmodule Pleroma.Search.DatabaseSearch do @impl true def remove_from_index(_object), do: :ok + @impl true + def healthcheck_endpoints, do: nil + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end diff --git a/lib/pleroma/search/healthcheck.ex b/lib/pleroma/search/healthcheck.ex new file mode 100644 index 000000000..e562c8478 --- /dev/null +++ b/lib/pleroma/search/healthcheck.ex @@ -0,0 +1,86 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only +defmodule Pleroma.Search.Healthcheck do + @doc """ + Monitors health of search backend to control processing of events based on health and availability. + """ + use GenServer + require Logger + + @queue :search_indexing + @tick :timer.seconds(5) + @timeout :timer.seconds(2) + + def start_link(_) do + GenServer.start_link(__MODULE__, [], name: __MODULE__) + end + + @impl true + def init(_) do + state = %{healthy: false} + {:ok, state, {:continue, :start}} + end + + @impl true + def handle_continue(:start, state) do + tick() + {:noreply, state} + end + + @impl true + def handle_info(:check, state) do + urls = Pleroma.Search.healthcheck_endpoints() + + new_state = + if check(urls) do + Oban.resume_queue(queue: @queue) + Map.put(state, :healthy, true) + else + Oban.pause_queue(queue: @queue) + Map.put(state, :healthy, false) + end + + maybe_log_state_change(state, new_state) + + tick() + {:noreply, new_state} + end + + @impl true + def handle_call(:state, _from, state) do + {:reply, state, state, :hibernate} + end + + def state, do: GenServer.call(__MODULE__, :state) + + def check([]), do: true + + def check(urls) when is_list(urls) do + Enum.all?( + urls, + fn url -> + case Pleroma.HTTP.get(url, [], recv_timeout: @timeout) do + {:ok, %{status: 200}} -> true + _ -> false + end + end + ) + end + + def check(_), do: true + + defp tick do + Process.send_after(self(), :check, @tick) + end + + defp maybe_log_state_change(%{healthy: true}, %{healthy: false}) do + Logger.error("Pausing Oban queue #{@queue} due to search backend healthcheck failure") + end + + defp maybe_log_state_change(%{healthy: false}, %{healthy: true}) do + Logger.info("Resuming Oban queue #{@queue} due to search backend healthcheck pass") + end + + defp maybe_log_state_change(_, _), do: :ok +end diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex index 2bff663e8..08c2f3d86 100644 --- a/lib/pleroma/search/meilisearch.ex +++ b/lib/pleroma/search/meilisearch.ex @@ -178,4 +178,15 @@ defmodule Pleroma.Search.Meilisearch do def remove_from_index(object) do meili_delete("/indexes/objects/documents/#{object.id}") end + + @impl true + def healthcheck_endpoints do + endpoint = + Config.get([Pleroma.Search.Meilisearch, :url]) + |> URI.parse() + |> Map.put(:path, "/health") + |> URI.to_string() + + [endpoint] + end end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex index 68bc48cec..13c887bc2 100644 --- a/lib/pleroma/search/search_backend.ex +++ b/lib/pleroma/search/search_backend.ex @@ -21,4 +21,12 @@ defmodule Pleroma.Search.SearchBackend do from index. """ @callback remove_from_index(object :: Pleroma.Object.t()) :: :ok | {:error, any()} + + @doc """ + Healthcheck endpoints of search backend infrastructure to monitor for controlling + processing of jobs in the Oban queue. + + It is expected a 200 response is healthy and other responses are unhealthy. + """ + @callback healthcheck_endpoints :: list() | nil end diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index e6c484548..35c7c02a5 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -239,8 +239,12 @@ defmodule Pleroma.Upload do "" end - [base_url, path] - |> Path.join() + if String.contains?(base_url, Pleroma.Uploaders.IPFS.placeholder()) do + String.replace(base_url, Pleroma.Uploaders.IPFS.placeholder(), path) + else + [base_url, path] + |> Path.join() + end end defp url_from_spec(_upload, _base_url, {:url, url}), do: url @@ -277,6 +281,9 @@ defmodule Pleroma.Upload do Path.join([upload_base_url, bucket_with_namespace]) end + Pleroma.Uploaders.IPFS -> + @config_impl.get([Pleroma.Uploaders.IPFS, :get_gateway_url]) + _ -> public_endpoint || upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" end diff --git a/lib/pleroma/uploaders/ipfs.ex b/lib/pleroma/uploaders/ipfs.ex new file mode 100644 index 000000000..d171e4652 --- /dev/null +++ b/lib/pleroma/uploaders/ipfs.ex @@ -0,0 +1,77 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Uploaders.IPFS do + @behaviour Pleroma.Uploaders.Uploader + require Logger + + alias Tesla.Multipart + + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + + defp get_final_url(method) do + config = @config_impl.get([__MODULE__]) + post_base_url = Keyword.get(config, :post_gateway_url) + + Path.join([post_base_url, method]) + end + + def put_file_endpoint do + get_final_url("/api/v0/add") + end + + def delete_file_endpoint do + get_final_url("/api/v0/files/rm") + end + + @placeholder "{CID}" + def placeholder, do: @placeholder + + @impl true + def get_file(file) do + b_url = Pleroma.Upload.base_url() + + if String.contains?(b_url, @placeholder) do + {:ok, {:url, String.replace(b_url, @placeholder, URI.decode(file))}} + else + {:error, "IPFS Get URL doesn't contain 'cid' placeholder"} + end + end + + @impl true + def put_file(%Pleroma.Upload{} = upload) do + mp = + Multipart.new() + |> Multipart.add_content_type_param("charset=utf-8") + |> Multipart.add_file(upload.tempfile) + + case Pleroma.HTTP.post(put_file_endpoint(), mp, [], params: ["cid-version": "1"]) do + {:ok, ret} -> + case Jason.decode(ret.body) do + {:ok, ret} -> + if Map.has_key?(ret, "Hash") do + {:ok, {:file, ret["Hash"]}} + else + {:error, "JSON doesn't contain Hash key"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "JSON decode failed"} + end + + error -> + Logger.error("#{__MODULE__}: #{inspect(error)}") + {:error, "IPFS Gateway upload failed"} + end + end + + @impl true + def delete_file(file) do + case Pleroma.HTTP.post(delete_file_endpoint(), "", [], params: [arg: file]) do + {:ok, %{status: 204}} -> :ok + error -> {:error, inspect(error)} + end + end +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 778e20526..6d6aa98b5 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1404,6 +1404,40 @@ defmodule Pleroma.User do |> Repo.all() end + @spec get_familiar_followers_query(User.t(), User.t(), pos_integer() | nil) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user, nil) do + friends = + get_friends_query(current_user) + |> where([u], not u.hide_follows) + |> select([u], u.id) + + User.Query.build(%{is_active: true}) + |> where([u], u.id not in ^[user.id, current_user.id]) + |> join(:inner, [u], r in FollowingRelationship, + as: :followers_relationships, + on: r.following_id == ^user.id and r.follower_id == u.id + ) + |> where([followers_relationships: r], r.state == ^:follow_accept) + |> where([followers_relationships: r], r.follower_id in subquery(friends)) + end + + def get_familiar_followers_query(%User{} = user, %User{} = current_user, page) do + user + |> get_familiar_followers_query(current_user, nil) + |> User.Query.paginate(page, 20) + end + + @spec get_familiar_followers_query(User.t(), User.t()) :: Ecto.Query.t() + def get_familiar_followers_query(%User{} = user, %User{} = current_user), + do: get_familiar_followers_query(user, current_user, nil) + + @spec get_familiar_followers(User.t(), User.t(), pos_integer() | nil) :: {:ok, list(User.t())} + def get_familiar_followers(%User{} = user, %User{} = current_user, page \\ nil) do + user + |> get_familiar_followers_query(current_user, page) + |> Repo.all() + end + def increase_note_count(%User{} = user) do User |> where(id: ^user.id) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 2017c696d..5bb0fba6e 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -147,9 +147,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do # Splice in the child object if we have one. activity = Maps.put_if_present(activity, :object, object) - ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn -> - Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) - end) + Pleroma.Web.RichMedia.Card.get_by_activity(activity) # Add local posts to search index if local, do: Pleroma.Search.add_to_index(activity) @@ -177,7 +175,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do id: "pleroma:fakeid" } - Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + Pleroma.Web.RichMedia.Card.get_by_activity(activity) {:ok, activity} {:remote_limit_pass, _} -> @@ -202,7 +200,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end def notify_and_stream(activity) do - Notification.create_notifications(activity) + {:ok, notifications} = Notification.create_notifications(activity) + Notification.send(notifications) original_activity = case activity do @@ -980,8 +979,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_replies(query, %{exclude_replies: true}) do from( - [_activity, object] in query, - where: fragment("?->>'inReplyTo' is null", object.data) + [activity, object] in query, + where: + fragment("?->>'inReplyTo' is null or ?->>'type' = 'Announce'", object.data, activity.data) ) end @@ -1261,6 +1261,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_quote_url(query, _), do: query + defp restrict_rule(query, %{rule_id: rule_id}) do + from( + activity in query, + where: fragment("(?)->'rules' \\? (?)", activity.data, ^rule_id) + ) + end + + defp restrict_rule(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1423,6 +1432,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_instance(opts) |> restrict_announce_object_actor(opts) |> restrict_filtered(opts) + |> restrict_rule(opts) |> restrict_quote_url(opts) |> maybe_restrict_deactivated_users(opts) |> exclude_poll_votes(opts) diff --git a/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex new file mode 100644 index 000000000..531e75ce8 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/anti_mention_spam_policy.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.AntiMentionSpamPolicy do + alias Pleroma.Config + alias Pleroma.User + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + defp user_has_posted?(%User{} = u), do: u.note_count > 0 + + defp user_has_age?(%User{} = u) do + user_age_limit = Config.get([:mrf_antimentionspam, :user_age_limit], 30_000) + diff = NaiveDateTime.utc_now() |> NaiveDateTime.diff(u.inserted_at, :millisecond) + diff >= user_age_limit + end + + defp good_reputation?(%User{} = u) do + user_has_age?(u) and user_has_posted?(u) + end + + # copied from HellthreadPolicy + defp get_recipient_count(message) do + recipients = (message["to"] || []) ++ (message["cc"] || []) + + follower_collection = + User.get_cached_by_ap_id(message["actor"] || message["attributedTo"]).follower_address + + if Enum.member?(recipients, Pleroma.Constants.as_public()) do + recipients = + recipients + |> List.delete(Pleroma.Constants.as_public()) + |> List.delete(follower_collection) + + {:public, length(recipients)} + else + recipients = + recipients + |> List.delete(follower_collection) + + {:not_public, length(recipients)} + end + end + + defp object_has_recipients?(%{"object" => object} = activity) do + {_, object_count} = get_recipient_count(object) + {_, activity_count} = get_recipient_count(activity) + object_count + activity_count > 0 + end + + defp object_has_recipients?(object) do + {_, count} = get_recipient_count(object) + count > 0 + end + + @impl true + def filter(%{"type" => "Create", "actor" => actor} = activity) do + with {:ok, %User{local: false} = u} <- User.get_or_fetch_by_ap_id(actor), + {:has_mentions, true} <- {:has_mentions, object_has_recipients?(activity)}, + {:good_reputation, true} <- {:good_reputation, good_reputation?(u)} do + {:ok, activity} + else + {:ok, %User{local: true}} -> + {:ok, activity} + + {:has_mentions, false} -> + {:ok, activity} + + {:good_reputation, false} -> + {:reject, "[AntiMentionSpamPolicy] User rejected"} + + {:error, _} -> + {:reject, "[AntiMentionSpamPolicy] Failed to get or fetch user by ap_id"} + + e -> + {:reject, "[AntiMentionSpamPolicy] Unhandled error #{inspect(e)}"} + end + end + + # in all other cases, pass through + def filter(message), do: {:ok, message} + + @impl true + def describe, do: {:ok, %{}} +end diff --git a/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex new file mode 100644 index 000000000..9543cc545 --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/dnsrbl_policy.ex @@ -0,0 +1,142 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2024 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy do + @moduledoc """ + Dynamic activity filtering based on an RBL database + + This MRF makes queries to a custom DNS server which will + respond with values indicating the classification of the domain + the activity originated from. This method has been widely used + in the email anti-spam industry for very fast reputation checks. + + e.g., if the DNS response is 127.0.0.1 or empty, the domain is OK + Other values such as 127.0.0.2 may be used for specific classifications. + + Information for why the host is blocked can be stored in a corresponding TXT record. + + This method is fail-open so if the queries fail the activites are accepted. + + An example of software meant for this purpsoe is rbldnsd which can be found + at http://www.corpit.ru/mjt/rbldnsd.html or mirrored at + https://git.pleroma.social/feld/rbldnsd + + It is highly recommended that you run your own copy of rbldnsd and use an + external mechanism to sync/share the contents of the zone file. This is + important to keep the latency on the queries as low as possible and prevent + your DNS server from being attacked so it fails and content is permitted. + """ + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Config + + require Logger + + @query_retries 1 + @query_timeout 500 + + @impl true + def filter(%{"actor" => actor} = object) do + actor_info = URI.parse(actor) + + with {:ok, object} <- check_rbl(actor_info, object) do + {:ok, object} + else + _ -> {:reject, "[DNSRBLPolicy]"} + end + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe do + mrf_dnsrbl = + Config.get(:mrf_dnsrbl) + |> Enum.into(%{}) + + {:ok, %{mrf_dnsrbl: mrf_dnsrbl}} + end + + @impl true + def config_description do + %{ + key: :mrf_dnsrbl, + related_policy: "Pleroma.Web.ActivityPub.MRF.DNSRBLPolicy", + label: "MRF DNSRBL", + description: "DNS RealTime Blackhole Policy", + children: [ + %{ + key: :nameserver, + type: {:string}, + description: "DNSRBL Nameserver to Query (IP or hostame)", + suggestions: ["127.0.0.1"] + }, + %{ + key: :port, + type: {:string}, + description: "Nameserver port", + suggestions: ["53"] + }, + %{ + key: :zone, + type: {:string}, + description: "Root zone for querying", + suggestions: ["bl.pleroma.com"] + } + ] + } + end + + defp check_rbl(%{host: actor_host}, object) do + with false <- match?(^actor_host, Pleroma.Web.Endpoint.host()), + zone when not is_nil(zone) <- Keyword.get(Config.get([:mrf_dnsrbl]), :zone) do + query = + Enum.join([actor_host, zone], ".") + |> String.to_charlist() + + rbl_response = rblquery(query) + + if Enum.empty?(rbl_response) do + {:ok, object} + else + Task.start(fn -> + reason = rblquery(query, :txt) || "undefined" + + Logger.warning( + "DNSRBL Rejected activity from #{actor_host} for reason: #{inspect(reason)}" + ) + end) + + :error + end + else + _ -> {:ok, object} + end + end + + defp get_rblhost_ip(rblhost) do + case rblhost |> String.to_charlist() |> :inet_parse.address() do + {:ok, _} -> rblhost |> String.to_charlist() |> :inet_parse.address() + _ -> {:ok, rblhost |> String.to_charlist() |> :inet_res.lookup(:in, :a) |> Enum.random()} + end + end + + defp rblquery(query, type \\ :a) do + config = Config.get([:mrf_dnsrbl]) + + case get_rblhost_ip(config[:nameserver]) do + {:ok, rblnsip} -> + :inet_res.lookup(query, :in, type, + nameservers: [{rblnsip, config[:port]}], + timeout: @query_timeout, + retry: @query_retries + ) + + _ -> + [] + end + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex new file mode 100644 index 000000000..f7863039b --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/nsfw_api_policy.ex @@ -0,0 +1,265 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.NsfwApiPolicy do + @moduledoc """ + Hide, delete, or mark sensitive NSFW content with artificial intelligence. + + Requires a NSFW API server, configured like so: + + config :pleroma, Pleroma.Web.ActivityPub.MRF.NsfwMRF, + url: "http://127.0.0.1:5000/", + threshold: 0.7, + mark_sensitive: true, + unlist: false, + reject: false + + The NSFW API server must implement an HTTP endpoint like this: + + curl http://localhost:5000/?url=https://fedi.com/images/001.jpg + + Returning a response like this: + + {"score", 0.314} + + Where a score is 0-1, with `1` being definitely NSFW. + + A good API server is here: https://github.com/EugenCepoi/nsfw_api + You can run it with Docker with a one-liner: + + docker run -it -p 127.0.0.1:5000:5000/tcp --env PORT=5000 eugencepoi/nsfw_api:latest + + Options: + + - `url`: Base URL of the API server. Default: "http://127.0.0.1:5000/" + - `threshold`: Lowest score to take action on. Default: `0.7` + - `mark_sensitive`: Mark sensitive all detected NSFW content? Default: `true` + - `unlist`: Unlist all detected NSFW content? Default: `false` + - `reject`: Reject all detected NSFW content (takes precedence)? Default: `false` + """ + alias Pleroma.Config + alias Pleroma.Constants + alias Pleroma.HTTP + alias Pleroma.User + + require Logger + require Pleroma.Constants + + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + @policy :mrf_nsfw_api + + def build_request_url(url) do + Config.get([@policy, :url]) + |> URI.parse() + |> fix_path() + |> Map.put(:query, "url=#{url}") + |> URI.to_string() + end + + def parse_url(url) do + request = build_request_url(url) + + with {:ok, %Tesla.Env{body: body}} <- HTTP.get(request) do + Jason.decode(body) + else + error -> + Logger.warn(""" + [NsfwApiPolicy]: The API server failed. Skipping. + #{inspect(error)} + """) + + error + end + end + + def check_url_nsfw(url) when is_binary(url) do + threshold = Config.get([@policy, :threshold]) + + case parse_url(url) do + {:ok, %{"score" => score}} when score >= threshold -> + {:nsfw, %{url: url, score: score, threshold: threshold}} + + {:ok, %{"score" => score}} -> + {:sfw, %{url: url, score: score, threshold: threshold}} + + _ -> + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + end + + def check_url_nsfw(%{"href" => url}) when is_binary(url) do + check_url_nsfw(url) + end + + def check_url_nsfw(url) do + threshold = Config.get([@policy, :threshold]) + {:sfw, %{url: url, score: nil, threshold: threshold}} + end + + def check_attachment_nsfw(%{"url" => urls} = attachment) when is_list(urls) do + if Enum.all?(urls, &match?({:sfw, _}, check_url_nsfw(&1))) do + {:sfw, attachment} + else + {:nsfw, attachment} + end + end + + def check_attachment_nsfw(%{"url" => url} = attachment) when is_binary(url) do + case check_url_nsfw(url) do + {:sfw, _} -> {:sfw, attachment} + {:nsfw, _} -> {:nsfw, attachment} + end + end + + def check_attachment_nsfw(attachment), do: {:sfw, attachment} + + def check_object_nsfw(%{"attachment" => attachments} = object) when is_list(attachments) do + if Enum.all?(attachments, &match?({:sfw, _}, check_attachment_nsfw(&1))) do + {:sfw, object} + else + {:nsfw, object} + end + end + + def check_object_nsfw(%{"object" => %{} = child_object} = object) do + case check_object_nsfw(child_object) do + {:sfw, _} -> {:sfw, object} + {:nsfw, _} -> {:nsfw, object} + end + end + + def check_object_nsfw(object), do: {:sfw, object} + + @impl true + def filter(object) do + with {:sfw, object} <- check_object_nsfw(object) do + {:ok, object} + else + {:nsfw, _data} -> handle_nsfw(object) + _ -> {:reject, "NSFW: Attachment rejected"} + end + end + + defp handle_nsfw(object) do + if Config.get([@policy, :reject]) do + {:reject, object} + else + {:ok, + object + |> maybe_unlist() + |> maybe_mark_sensitive()} + end + end + + defp maybe_unlist(object) do + if Config.get([@policy, :unlist]) do + unlist(object) + else + object + end + end + + defp maybe_mark_sensitive(object) do + if Config.get([@policy, :mark_sensitive]) do + mark_sensitive(object) + else + object + end + end + + def unlist(%{"to" => to, "cc" => cc, "actor" => actor} = object) do + with %User{} = user <- User.get_cached_by_ap_id(actor) do + to = + [user.follower_address | to] + |> List.delete(Constants.as_public()) + |> Enum.uniq() + + cc = + [Constants.as_public() | cc] + |> List.delete(user.follower_address) + |> Enum.uniq() + + object + |> Map.put("to", to) + |> Map.put("cc", cc) + else + _ -> raise "[NsfwApiPolicy]: Could not find user #{actor}" + end + end + + def mark_sensitive(%{"object" => child_object} = object) when is_map(child_object) do + Map.put(object, "object", mark_sensitive(child_object)) + end + + def mark_sensitive(object) when is_map(object) do + tags = (object["tag"] || []) ++ ["nsfw"] + + object + |> Map.put("tag", tags) + |> Map.put("sensitive", true) + end + + # Hackney needs a trailing slash + defp fix_path(%URI{path: path} = uri) when is_binary(path) do + path = String.trim_trailing(path, "/") <> "/" + Map.put(uri, :path, path) + end + + defp fix_path(%URI{path: nil} = uri), do: Map.put(uri, :path, "/") + + @impl true + def describe do + options = %{ + threshold: Config.get([@policy, :threshold]), + mark_sensitive: Config.get([@policy, :mark_sensitive]), + unlist: Config.get([@policy, :unlist]), + reject: Config.get([@policy, :reject]) + } + + {:ok, %{@policy => options}} + end + + @impl true + def config_description do + %{ + key: @policy, + related_policy: to_string(__MODULE__), + label: "NSFW API Policy", + description: + "Hide, delete, or mark sensitive NSFW content with artificial intelligence. Requires running an external API server.", + children: [ + %{ + key: :url, + type: :string, + description: "Base URL of the API server.", + suggestions: ["http://127.0.0.1:5000/"] + }, + %{ + key: :threshold, + type: :float, + description: "Lowest score to take action on. Between 0 and 1.", + suggestions: [0.7] + }, + %{ + key: :mark_sensitive, + type: :boolean, + description: "Mark sensitive all detected NSFW content?", + suggestions: [true] + }, + %{ + key: :unlist, + type: :boolean, + description: "Unlist sensitive all detected NSFW content?", + suggestions: [false] + }, + %{ + key: :reject, + type: :boolean, + description: "Reject sensitive all detected NSFW content (takes precedence)?", + suggestions: [false] + } + ] + } + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index 72975f348..5ee9e7549 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:type, :string, default: "Link") field(:mediaType, ObjectValidators.MIME, default: "application/octet-stream") field(:name, :string) + field(:summary, :string) field(:blurhash, :string) embeds_many :url, UrlObjectValidator, primary_key: false do @@ -44,7 +45,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do |> fix_url() struct - |> cast(data, [:id, :type, :mediaType, :name, :blurhash]) + |> cast(data, [:id, :type, :mediaType, :name, :summary, :blurhash]) |> cast_embed(:url, with: &url_changeset/2, required: true) |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) |> validate_required([:type, :mediaType]) diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 5cb8a9700..60b4d5f1b 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -21,7 +21,6 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils - alias Pleroma.Web.Push alias Pleroma.Web.Streamer alias Pleroma.Workers.PollWorker @@ -125,7 +124,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do nil end - {:ok, notifications} = Notification.create_notifications(object, do_send: false) + {:ok, notifications} = Notification.create_notifications(object) meta = meta @@ -184,7 +183,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do liked_object = Object.get_by_ap_id(object.data["object"]) Utils.add_like_to_object(object, liked_object) - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -202,7 +205,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do - {:ok, notifications} = Notification.create_notifications(activity, do_send: false) + {:ok, notifications} = Notification.create_notifications(activity) {:ok, _user} = ActivityPub.increase_note_count_if_public(user, object) {:ok, _user} = ActivityPub.update_last_status_at_if_public(user, object) @@ -227,9 +230,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end end - ConcurrentLimiter.limit(Pleroma.Web.RichMedia.Helpers, fn -> - Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) - end) + Pleroma.Web.RichMedia.Card.get_by_activity(activity) Pleroma.Search.add_to_index(Map.put(activity, :object, object)) @@ -258,11 +259,13 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Utils.add_announce_to_object(object, announced_object) - if !User.internal?(user) do - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) - ap_streamer().stream_out(object) - end + if !User.internal?(user), do: ap_streamer().stream_out(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -283,7 +286,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do reacted_object = Object.get_by_ap_id(object.data["object"]) Utils.add_emoji_reaction_to_object(object, reacted_object) - Notification.create_notifications(object) + {:ok, notifications} = Notification.create_notifications(object) + + meta = + meta + |> add_notifications(notifications) {:ok, object, meta} end @@ -587,10 +594,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do defp send_notifications(meta) do Keyword.get(meta, :notifications, []) - |> Enum.each(fn notification -> - Streamer.stream(["user", "user:notification"], notification) - Push.send(notification) - end) + |> Notification.send() meta end diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 52cb64fc5..797e79dda 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -721,14 +721,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do #### Flag-related helpers @spec make_flag_data(map(), map()) :: map() - def make_flag_data(%{actor: actor, context: context, content: content} = params, additional) do + def make_flag_data( + %{actor: actor, context: context, content: content} = params, + additional + ) do %{ "type" => "Flag", "actor" => actor.ap_id, "content" => content, "object" => build_flag_object(params), "context" => context, - "state" => "open" + "state" => "open", + "rules" => Map.get(params, :rules, nil) } |> Map.merge(additional) end diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index 24ee683ae..937e4fd67 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -67,8 +67,13 @@ defmodule Pleroma.Web.ActivityPub.UserView do def render("user.json", %{user: %User{nickname: nil} = user}), do: render("service.json", %{user: user}) - def render("user.json", %{user: %User{nickname: "internal." <> _} = user}), - do: render("service.json", %{user: user}) |> Map.put("preferredUsername", user.nickname) + def render("user.json", %{user: %User{nickname: "internal." <> _} = user}) do + render("service.json", %{user: user}) + |> Map.merge(%{ + "preferredUsername" => user.nickname, + "webfinger" => "acct:#{User.full_nickname(user)}" + }) + end def render("user.json", %{user: user}) do {:ok, _, public_key} = Keys.keys_from_pem(user.keys) @@ -121,7 +126,8 @@ defmodule Pleroma.Web.ActivityPub.UserView do "discoverable" => user.is_discoverable, "capabilities" => capabilities, "alsoKnownAs" => user.also_known_as, - "vcard:bday" => birthday + "vcard:bday" => birthday, + "webfinger" => "acct:#{User.full_nickname(user)}" } |> Map.merge(maybe_make_image(&User.avatar_url/2, "icon", user)) |> Map.merge(maybe_make_image(&User.banner_url/2, "image", user)) diff --git a/lib/pleroma/web/admin_api/controllers/rule_controller.ex b/lib/pleroma/web/admin_api/controllers/rule_controller.ex new file mode 100644 index 000000000..43b2f209a --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/rule_controller.ex @@ -0,0 +1,62 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleController do + use Pleroma.Web, :controller + + alias Pleroma.Repo + alias Pleroma.Rule + alias Pleroma.Web.Plugs.OAuthScopesPlug + + import Pleroma.Web.ControllerHelper, + only: [ + json_response: 3 + ] + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["admin:write"]} + when action in [:create, :update, :delete] + ) + + plug(OAuthScopesPlug, %{scopes: ["admin:read"]} when action == :index) + + action_fallback(AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.Admin.RuleOperation + + def index(conn, _) do + rules = + Rule.query() + |> Repo.all() + + render(conn, "index.json", rules: rules) + end + + def create(%{body_params: params} = conn, _) do + rule = + params + |> Rule.create() + + render(conn, "show.json", rule: rule) + end + + def update(%{body_params: params} = conn, %{id: id}) do + rule = + params + |> Rule.update(id) + + render(conn, "show.json", rule: rule) + end + + def delete(conn, %{id: id}) do + with {:ok, _} <- Rule.delete(id) do + json(conn, %{}) + else + _ -> json_response(conn, :bad_request, "") + end + end +end diff --git a/lib/pleroma/web/admin_api/views/report_view.ex b/lib/pleroma/web/admin_api/views/report_view.ex index b761dbb22..b4b0be267 100644 --- a/lib/pleroma/web/admin_api/views/report_view.ex +++ b/lib/pleroma/web/admin_api/views/report_view.ex @@ -6,9 +6,11 @@ defmodule Pleroma.Web.AdminAPI.ReportView do use Pleroma.Web, :view alias Pleroma.HTML + alias Pleroma.Rule alias Pleroma.User alias Pleroma.Web.AdminAPI alias Pleroma.Web.AdminAPI.Report + alias Pleroma.Web.AdminAPI.RuleView alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.StatusView @@ -46,7 +48,8 @@ defmodule Pleroma.Web.AdminAPI.ReportView do as: :activity }), state: report.data["state"], - notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}) + notes: render(__MODULE__, "index_notes.json", %{notes: report.report_notes}), + rules: rules(Map.get(report.data, "rules", nil)) } end @@ -71,4 +74,16 @@ defmodule Pleroma.Web.AdminAPI.ReportView do created_at: Utils.to_masto_date(inserted_at) } end + + defp rules(nil) do + [] + end + + defp rules(rule_ids) do + rules = + rule_ids + |> Rule.get() + + render(RuleView, "index.json", rules: rules) + end end diff --git a/lib/pleroma/web/admin_api/views/rule_view.ex b/lib/pleroma/web/admin_api/views/rule_view.ex new file mode 100644 index 000000000..606443f05 --- /dev/null +++ b/lib/pleroma/web/admin_api/views/rule_view.ex @@ -0,0 +1,22 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.RuleView do + use Pleroma.Web, :view + + require Pleroma.Constants + + def render("index.json", %{rules: rules} = _opts) do + render_many(rules, __MODULE__, "show.json") + end + + def render("show.json", %{rule: rule} = _opts) do + %{ + id: to_string(rule.id), + priority: rule.priority, + text: rule.text, + hint: rule.hint + } + end +end diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 10d221571..314782818 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -97,6 +97,7 @@ defmodule Pleroma.Web.ApiSpec do "Frontend management", "Instance configuration", "Instance documents", + "Instance rule managment", "Invites", "MediaProxy cache", "OAuth application management", diff --git a/lib/pleroma/web/api_spec/operations/account_operation.ex b/lib/pleroma/web/api_spec/operations/account_operation.ex index 36025e47a..85f02166f 100644 --- a/lib/pleroma/web/api_spec/operations/account_operation.ex +++ b/lib/pleroma/web/api_spec/operations/account_operation.ex @@ -11,6 +11,7 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do alias Pleroma.Web.ApiSpec.Schemas.ActorType alias Pleroma.Web.ApiSpec.Schemas.ApiError alias Pleroma.Web.ApiSpec.Schemas.BooleanLike + alias Pleroma.Web.ApiSpec.Schemas.FlakeID alias Pleroma.Web.ApiSpec.Schemas.List alias Pleroma.Web.ApiSpec.Schemas.Status alias Pleroma.Web.ApiSpec.Schemas.VisibilityScope @@ -513,6 +514,48 @@ defmodule Pleroma.Web.ApiSpec.AccountOperation do } end + def familiar_followers_operation do + %Operation{ + tags: ["Retrieve account information"], + summary: "Followers that you follow", + operationId: "AccountController.familiar_followers", + description: + "Obtain a list of all accounts that follow a given account, filtered for accounts you follow.", + security: [%{"oAuth" => ["read:follows"]}], + parameters: [ + Operation.parameter( + :id, + :query, + %Schema{ + oneOf: [%Schema{type: :array, items: %Schema{type: :string}}, %Schema{type: :string}] + }, + "Account IDs", + example: "123" + ) + ], + responses: %{ + 200 => + Operation.response("Accounts", "application/json", %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: %Schema{ + title: "Account", + type: :object, + properties: %{ + id: FlakeID, + accounts: %Schema{ + title: "ArrayOfAccounts", + type: :array, + items: Account, + example: [Account.schema().example] + } + } + } + }) + } + } + end + defp create_request do %Schema{ title: "AccountCreateRequest", diff --git a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex index fbb6896a9..25a604beb 100644 --- a/lib/pleroma/web/api_spec/operations/admin/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/admin/report_operation.ex @@ -31,6 +31,12 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do "Filter by report state" ), Operation.parameter( + :rule_id, + :query, + %Schema{type: :string}, + "Filter by selected rule id" + ), + Operation.parameter( :limit, :query, %Schema{type: :integer}, @@ -169,6 +175,17 @@ defmodule Pleroma.Web.ApiSpec.Admin.ReportOperation do inserted_at: %Schema{type: :string, format: :"date-time"} } } + }, + rules: %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + text: %Schema{type: :string}, + hint: %Schema{type: :string, nullable: true} + } + } } } } diff --git a/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex new file mode 100644 index 000000000..c3a3ecc7c --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/rule_operation.ex @@ -0,0 +1,115 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.RuleOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Retrieve list of instance rules", + operationId: "AdminAPI.RuleController.index", + security: [%{"oAuth" => ["admin:read"]}], + responses: %{ + 200 => + Operation.response("Response", "application/json", %Schema{ + type: :array, + items: rule() + }), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def create_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Create new rule", + operationId: "AdminAPI.RuleController.create", + security: [%{"oAuth" => ["admin:write"]}], + parameters: admin_api_params(), + requestBody: request_body("Parameters", create_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def update_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Modify existing rule", + operationId: "AdminAPI.RuleController.update", + security: [%{"oAuth" => ["admin:write"]}], + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + requestBody: request_body("Parameters", update_request(), required: true), + responses: %{ + 200 => Operation.response("Response", "application/json", rule()), + 400 => Operation.response("Bad Request", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + def delete_operation do + %Operation{ + tags: ["Instance rule managment"], + summary: "Delete rule", + operationId: "AdminAPI.RuleController.delete", + parameters: [Operation.parameter(:id, :path, :string, "Rule ID")], + security: [%{"oAuth" => ["admin:write"]}], + responses: %{ + 200 => empty_object_response(), + 404 => Operation.response("Not Found", "application/json", ApiError), + 403 => Operation.response("Forbidden", "application/json", ApiError) + } + } + end + + defp create_request do + %Schema{ + type: :object, + required: [:text], + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + end + + defp update_request do + %Schema{ + type: :object, + properties: %{ + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + end + + defp rule do + %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + priority: %Schema{type: :integer}, + text: %Schema{type: :string}, + hint: %Schema{type: :string, nullable: true} + } + } + end +end diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index 708b74b12..7d7a5ecc1 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -46,10 +46,30 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + def rules_operation do + %Operation{ + tags: ["Instance misc"], + summary: "Retrieve list of instance rules", + operationId: "InstanceController.rules", + responses: %{ + 200 => Operation.response("Array of domains", "application/json", array_of_rules()) + } + } + end + defp instance do %Schema{ type: :object, properties: %{ + accounts: %Schema{ + type: :object, + properties: %{ + max_featured_tags: %Schema{ + type: :integer, + description: "The maximum number of featured tags allowed for each account." + } + } + }, uri: %Schema{type: :string, description: "The domain name of the instance"}, title: %Schema{type: :string, description: "The title of the website"}, description: %Schema{ @@ -172,7 +192,8 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do "urls" => %{ "streaming_api" => "wss://lain.com" }, - "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)" + "version" => "2.7.2 (compatible; Pleroma 2.0.50-536-g25eec6d7-develop)", + "rules" => array_of_rules() } } end @@ -272,6 +293,19 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do type: :object, description: "Instance configuration", properties: %{ + accounts: %Schema{ + type: :object, + properties: %{ + max_featured_tags: %Schema{ + type: :integer, + description: "The maximum number of featured tags allowed for each account." + }, + max_pinned_statuses: %Schema{ + type: :integer, + description: "The maximum number of pinned statuses for each account." + } + } + }, urls: %Schema{ type: :object, properties: %{ @@ -285,6 +319,11 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do type: :object, description: "A map with poll limits for local statuses", properties: %{ + characters_reserved_per_url: %Schema{ + type: :integer, + description: + "Each URL in a status will be assumed to be exactly this many characters." + }, max_characters: %Schema{ type: :integer, description: "Posts character limit (CW/Subject included in the counter)" @@ -344,4 +383,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do example: ["pleroma.site", "lain.com", "bikeshed.party"] } end + + defp array_of_rules do + %Schema{ + type: :array, + items: %Schema{ + type: :object, + properties: %{ + id: %Schema{type: :string}, + text: %Schema{type: :string}, + hint: %Schema{type: :string} + } + } + } + end end diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index 757429d12..a79eb8f74 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -202,7 +202,8 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "pleroma:report", "move", "follow_request", - "poll" + "poll", + "status" ], description: """ The type of event that resulted in the notification. @@ -216,6 +217,7 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do - `pleroma:emoji_reaction` - Someone reacted with emoji to your status - `pleroma:chat_mention` - Someone mentioned you in a chat message - `pleroma:report` - Someone was reported + - `status` - Someone you are subscribed to created a status """ } end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex index a994345db..0e2865191 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_notification_operation.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do alias OpenApiSpex.Operation alias OpenApiSpex.Schema - alias Pleroma.Web.ApiSpec.NotificationOperation alias Pleroma.Web.ApiSpec.Schemas.ApiError import Pleroma.Web.ApiSpec.Helpers @@ -35,12 +34,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaNotificationOperation do Operation.response( "A Notification or array of Notifications", "application/json", - %Schema{ - anyOf: [ - %Schema{type: :array, items: NotificationOperation.notification()}, - NotificationOperation.notification() - ] - } + %Schema{type: :string} ), 400 => Operation.response("Bad Request", "application/json", ApiError) } diff --git a/lib/pleroma/web/api_spec/operations/report_operation.ex b/lib/pleroma/web/api_spec/operations/report_operation.ex index c74ac7d5f..f5f88974c 100644 --- a/lib/pleroma/web/api_spec/operations/report_operation.ex +++ b/lib/pleroma/web/api_spec/operations/report_operation.ex @@ -53,6 +53,12 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do default: false, description: "If the account is remote, should the report be forwarded to the remote admin?" + }, + rule_ids: %Schema{ + type: :array, + nullable: true, + items: %Schema{type: :string}, + description: "Array of rules" } }, required: [:account_id], @@ -60,7 +66,8 @@ defmodule Pleroma.Web.ApiSpec.ReportOperation do "account_id" => "123", "status_ids" => ["1337"], "comment" => "bad status!", - "forward" => "false" + "forward" => "false", + "rule_ids" => ["3"] } } end diff --git a/lib/pleroma/web/api_spec/schemas/attachment.ex b/lib/pleroma/web/api_spec/schemas/attachment.ex index 2871b5f99..4104ed25c 100644 --- a/lib/pleroma/web/api_spec/schemas/attachment.ex +++ b/lib/pleroma/web/api_spec/schemas/attachment.ex @@ -50,7 +50,11 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Attachment do pleroma: %Schema{ type: :object, properties: %{ - mime_type: %Schema{type: :string, description: "mime type of the attachment"} + mime_type: %Schema{type: :string, description: "mime type of the attachment"}, + name: %Schema{ + type: :string, + description: "Name of the attachment, typically the filename" + } } } }, diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index a4052803b..6e537b5da 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -58,6 +58,10 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do format: :uri, description: "Preview thumbnail" }, + image_description: %Schema{ + type: :string, + description: "Alternate text that describes what is in the thumbnail" + }, title: %Schema{type: :string, description: "Title of linked resource"}, description: %Schema{type: :string, description: "Description of preview"} } diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 27e82ecc8..34e480d73 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Formatter alias Pleroma.ModerationLog alias Pleroma.Object + alias Pleroma.Rule alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.UserRelationship @@ -568,14 +569,16 @@ defmodule Pleroma.Web.CommonAPI do def report(user, data) do with {:ok, account} <- get_reported_account(data.account_id), {:ok, {content_html, _, _}} <- make_report_content_html(data[:comment]), - {:ok, statuses} <- get_report_statuses(account, data) do + {:ok, statuses} <- get_report_statuses(account, data), + rules <- get_report_rules(Map.get(data, :rule_ids, nil)) do ActivityPub.flag(%{ context: Utils.generate_context_id(), actor: user, account: account, statuses: statuses, content: content_html, - forward: Map.get(data, :forward, false) + forward: Map.get(data, :forward, false), + rules: rules }) end end @@ -587,6 +590,15 @@ defmodule Pleroma.Web.CommonAPI do end end + defp get_report_rules(nil) do + nil + end + + defp get_report_rules(rule_ids) do + rule_ids + |> Enum.filter(&Rule.exists?/1) + end + def update_report_state(activity_ids, state) when is_list(activity_ids) do case Utils.update_report_state(activity_ids, state) do :ok -> {:ok, activity_ids} diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index bc46a8a36..8aa1e258d 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -129,8 +129,22 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(%{params: %{in_reply_to_status_id: ""}} = draft), do: draft - defp in_reply_to(%{params: %{in_reply_to_status_id: id}} = draft) when is_binary(id) do - %__MODULE__{draft | in_reply_to: Activity.get_by_id(id)} + defp in_reply_to(%{params: %{in_reply_to_status_id: :deleted}} = draft) do + add_error(draft, dgettext("errors", "Cannot reply to a deleted status")) + end + + defp in_reply_to(%{params: %{in_reply_to_status_id: id} = params} = draft) when is_binary(id) do + activity = Activity.get_by_id(id) + + params = + if is_nil(activity) do + # Deleted activities are returned as nil + Map.put(params, :in_reply_to_status_id, :deleted) + else + Map.put(params, :in_reply_to_status_id, activity) + end + + in_reply_to(%{draft | params: params}) end defp in_reply_to(%{params: %{in_reply_to_status_id: %Activity{} = in_reply_to}} = draft) do diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 9226a2deb..47e6f0a64 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -72,7 +72,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do %{scopes: ["follow", "write:blocks"]} when action in [:block, :unblock] ) - plug(OAuthScopesPlug, %{scopes: ["read:follows"]} when action == :relationships) + plug( + OAuthScopesPlug, + %{scopes: ["read:follows"]} when action in [:relationships, :familiar_followers] + ) plug( OAuthScopesPlug, @@ -629,6 +632,35 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do ) end + @doc "GET /api/v1/accounts/familiar_followers" + def familiar_followers( + %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, + _id + ) do + users = + User.get_all_by_ids(List.wrap(id)) + |> Enum.map(&%{id: &1.id, accounts: get_familiar_followers(&1, user)}) + + conn + |> render("familiar_followers.json", + for: user, + users: users, + as: :user + ) + end + + defp get_familiar_followers(%{id: id} = user, %{id: id}) do + User.get_familiar_followers(user, user) + end + + defp get_familiar_followers(%{hide_followers: true}, _current_user) do + [] + end + + defp get_familiar_followers(user, current_user) do + User.get_familiar_followers(user, current_user) + end + @doc "GET /api/v1/identity_proofs" def identity_proofs(conn, params), do: MastodonAPIController.empty_array(conn, params) end diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 3e664903a..b97b0e476 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -25,4 +25,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) end + + @doc "GET /api/v1/instance/rules" + def rules(conn, _params) do + render(conn, "rules.json") + end end diff --git a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex index e305aea94..afd83b785 100644 --- a/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/notification_controller.ex @@ -34,6 +34,7 @@ defmodule Pleroma.Web.MastodonAPI.NotificationController do pleroma:emoji_reaction poll update + status } # GET /api/v1/notifications diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 4f6de8a00..83e1bee54 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -38,7 +38,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do when action in [ :index, :show, - :card, :context, :show_history, :show_source @@ -473,21 +472,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do end end - @doc "GET /api/v1/statuses/:id/card" - @deprecated "https://github.com/tootsuite/mastodon/pull/11213" - def card( - %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: status_id}}}} = conn, - _ - ) do - with %Activity{} = activity <- Activity.get_by_id(status_id), - true <- Visibility.visible_for_user?(activity, user) do - data = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - render(conn, "card.json", data) - else - _ -> render_error(conn, :not_found, "Record not found") - end - end - @doc "GET /api/v1/statuses/:id/favourited_by" def favourited_by( %{assigns: %{user: user}, private: %{open_api_spex: %{params: %{id: id}}}} = conn, diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 267c3e3ed..6976ca6e5 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -193,6 +193,25 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do render_many(targets, AccountView, "relationship.json", render_opts) end + def render("familiar_followers.json", %{users: users} = opts) do + opts = + opts + |> Map.merge(%{as: :user}) + |> Map.delete(:users) + + users + |> render_many(AccountView, "familiar_followers.json", opts) + end + + def render("familiar_followers.json", %{user: %{id: id, accounts: accounts}} = opts) do + accounts = + accounts + |> render_many(AccountView, "show.json", opts) + |> Enum.filter(&Enum.any?/1) + + %{id: id, accounts: accounts} + end + defp do_render("show.json", %{user: user} = opts) do self = opts[:for] == user diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index 210b46d2c..99fc6d0c3 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -76,12 +76,26 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do }) end + def render("rules.json", _) do + Pleroma.Rule.query() + |> Pleroma.Repo.all() + |> render_many(__MODULE__, "rule.json", as: :rule) + end + + def render("rule.json", %{rule: rule}) do + %{ + id: to_string(rule.id), + text: rule.text, + hint: rule.hint || "" + } + end + defp common_information(instance) do %{ - title: Keyword.get(instance, :name), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", languages: Keyword.get(instance, :languages, ["en"]), - rules: [] + rules: render(__MODULE__, "rules.json"), + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})" } end @@ -213,6 +227,8 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do defp configuration2 do configuration() + |> put_in([:accounts, :max_pinned_statuses], Config.get([:instance, :max_pinned_statuses], 0)) + |> put_in([:statuses, :characters_reserved_per_url], 0) |> Map.merge(%{ urls: %{ streaming: Pleroma.Web.Endpoint.websocket_url(), diff --git a/lib/pleroma/web/mastodon_api/views/notification_view.ex b/lib/pleroma/web/mastodon_api/views/notification_view.ex index 2a51f3755..3f2478719 100644 --- a/lib/pleroma/web/mastodon_api/views/notification_view.ex +++ b/lib/pleroma/web/mastodon_api/views/notification_view.ex @@ -108,6 +108,9 @@ defmodule Pleroma.Web.MastodonAPI.NotificationView do "mention" -> put_status(response, activity, reading_user, status_render_opts) + "status" -> + put_status(response, activity, reading_user, status_render_opts) + "favourite" -> put_status(response, parent_activity_fn.(), reading_user, status_render_opts) diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e464f60dc..0c16749a4 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -21,6 +21,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.MediaProxy alias Pleroma.Web.PleromaAPI.EmojiReactionController + alias Pleroma.Web.RichMedia.Card import Pleroma.Web.ActivityPub.Visibility, only: [get_visibility: 1, visible_for_user?: 2] @@ -29,9 +30,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # pagination is restricted to 40 activities at a time defp fetch_rich_media_for_activities(activities) do Enum.each(activities, fn activity -> - spawn(fn -> - Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) - end) + spawn(fn -> Card.get_by_activity(activity) end) end) end @@ -113,9 +112,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # To do: check AdminAPIControllerTest on the reasons behind nil activities in the list activities = Enum.filter(opts.activities, & &1) - # Start fetching rich media before doing anything else, so that later calls to get the cards - # only block for timeout in the worst case, as opposed to - # length(activities_with_links) * timeout + # Start prefetching rich media before doing anything else fetch_rich_media_for_activities(activities) replied_to_activities = get_replied_to_activities(activities) quoted_activities = get_quoted_activities(activities) @@ -364,7 +361,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do summary = object.data["summary"] || "" - card = render("card.json", Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity)) + card = + case Card.get_by_activity(activity) do + %Card{} = result -> render("card.json", result) + _ -> nil + end url = if user.local do @@ -567,15 +568,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do } end - def render("card.json", %{rich_media: rich_media, page_url: page_url}) do - page_url_data = URI.parse(page_url) - - page_url_data = - if is_binary(rich_media["url"]) do - URI.merge(page_url_data, URI.parse(rich_media["url"])) - else - page_url_data - end + def render("card.json", %Card{fields: rich_media}) do + page_url_data = URI.parse(rich_media["url"]) page_url = page_url_data |> to_string @@ -589,6 +583,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do provider_url: page_url_data.scheme <> "://" <> page_url_data.host, url: page_url, image: image_url, + image_description: rich_media["image:alt"] || "", title: rich_media["title"] || "", description: rich_media["description"] || "", pleroma: %{ @@ -629,6 +624,19 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do to_string(attachment["id"] || hash_id) end + description = + if attachment["summary"] do + HTML.strip_tags(attachment["summary"]) + else + attachment["name"] + end + + name = if attachment["summary"], do: attachment["name"] + + pleroma = + %{mime_type: media_type} + |> Maps.put_if_present(:name, name) + %{ id: attachment_id, url: href, @@ -636,8 +644,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do preview_url: href_preview, text_url: href, type: type, - description: attachment["name"], - pleroma: %{mime_type: media_type}, + description: description, + pleroma: pleroma, blurhash: attachment["blurhash"] } |> Maps.put_if_present(:meta, meta) diff --git a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex index f860eaf7e..435ccfabe 100644 --- a/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/notification_controller.ex @@ -23,8 +23,9 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do } = conn, _ ) do - with {:ok, notification} <- Notification.read_one(user, notification_id) do - render(conn, "show.json", notification: notification, for: user) + with {:ok, _} <- Notification.read_one(user, notification_id) do + conn + |> json("ok") else {:error, message} -> conn @@ -38,11 +39,14 @@ defmodule Pleroma.Web.PleromaAPI.NotificationController do conn, _ ) do - notifications = - user - |> Notification.set_read_up_to(max_id) - |> Enum.take(80) - - render(conn, "index.json", notifications: notifications, for: user) + with {:ok, _} <- Notification.set_read_up_to(user, max_id) do + conn + |> json("ok") + else + {:error, message} -> + conn + |> put_status(:bad_request) + |> json(%{"error" => message}) + end end end diff --git a/lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex b/lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex index fc6ad59d0..12decb816 100644 --- a/lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex +++ b/lib/pleroma/web/pleroma_api/views/bookmark_folder_view.ex @@ -13,10 +13,8 @@ defmodule Pleroma.Web.PleromaAPI.BookmarkFolderView do %{ id: folder.id |> to_string(), name: folder.name, - emoji: get_emoji(folder.emoji), - source: %{ - emoji: folder.emoji - } + emoji: folder.emoji, + emoji_url: get_emoji_url(folder.emoji) } end @@ -24,18 +22,18 @@ defmodule Pleroma.Web.PleromaAPI.BookmarkFolderView do render_many(folders, __MODULE__, "show.json", Map.delete(opts, :folders)) end - defp get_emoji(nil) do + defp get_emoji_url(nil) do nil end - defp get_emoji(emoji) do + defp get_emoji_url(emoji) do if Emoji.unicode?(emoji) do - emoji + nil else emoji = Emoji.get(emoji) if emoji != nil do - Endpoint.url() |> URI.merge(emoji.relative_url) |> to_string() + Endpoint.url() |> URI.merge(emoji.file) |> to_string() else nil end diff --git a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex index 241bf0010..a1c88d075 100644 --- a/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex +++ b/lib/pleroma/web/pleroma_api/views/chat/message_reference_view.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do alias Pleroma.User alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.RichMedia.Card @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @@ -23,6 +24,12 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do } } ) do + card = + case Card.get_by_object(object) do + %Card{} = card_data -> StatusView.render("card.json", card_data) + _ -> nil + end + %{ id: id |> to_string(), content: chat_message["content"], @@ -34,11 +41,7 @@ defmodule Pleroma.Web.PleromaAPI.Chat.MessageReferenceView do chat_message["attachment"] && StatusView.render("attachment.json", attachment: chat_message["attachment"]), unread: unread, - card: - StatusView.render( - "card.json", - Pleroma.Web.RichMedia.Helpers.fetch_data_for_object(object) - ) + card: card } |> put_idempotency_key() end diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 36f44d8e8..9e68d827b 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -192,6 +192,7 @@ defmodule Pleroma.Web.Push.Impl do def format_title(%{type: type}, mastodon_type) do case mastodon_type || type do "mention" -> "New Mention" + "status" -> "New Status" "follow" -> "New Follower" "follow_request" -> "New Follow Request" "reblog" -> "New Repeat" diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex new file mode 100644 index 000000000..4ec50e132 --- /dev/null +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -0,0 +1,101 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.RichMedia.Backfill do + alias Pleroma.Web.RichMedia.Card + alias Pleroma.Web.RichMedia.Parser + alias Pleroma.Web.RichMedia.Parser.TTL + alias Pleroma.Workers.RichMediaExpirationWorker + + require Logger + + @backfiller Pleroma.Config.get([__MODULE__, :provider], Pleroma.Web.RichMedia.Backfill.Task) + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @max_attempts 3 + @retry 5_000 + + def start(%{url: url} = args) when is_binary(url) do + url_hash = Card.url_to_hash(url) + + args = + args + |> Map.put(:attempt, 1) + |> Map.put(:url_hash, url_hash) + + @backfiller.run(args) + end + + def run(%{url: url, url_hash: url_hash, attempt: attempt} = args) + when attempt <= @max_attempts do + case Parser.parse(url) do + {:ok, fields} -> + {:ok, card} = Card.create(url, fields) + + maybe_schedule_expiration(url, fields) + + if Map.has_key?(args, :activity_id) do + stream_update(args) + end + + warm_cache(url_hash, card) + + {:error, {:invalid_metadata, fields}} -> + Logger.debug("Rich media incomplete or invalid metadata for #{url}: #{inspect(fields)}") + negative_cache(url_hash) + + {:error, :body_too_large} -> + Logger.error("Rich media error for #{url}: :body_too_large") + negative_cache(url_hash) + + {:error, {:content_type, type}} -> + Logger.debug("Rich media error for #{url}: :content_type is #{type}") + negative_cache(url_hash) + + e -> + Logger.debug("Rich media error for #{url}: #{inspect(e)}") + + :timer.sleep(@retry * attempt) + + run(%{args | attempt: attempt + 1}) + end + end + + def run(%{url: url, url_hash: url_hash}) do + Logger.debug("Rich media failure for #{url}") + + negative_cache(url_hash, :timer.minutes(15)) + end + + defp maybe_schedule_expiration(url, fields) do + case TTL.process(fields, url) do + {:ok, ttl} when is_number(ttl) -> + timestamp = DateTime.from_unix!(ttl) + + RichMediaExpirationWorker.new(%{"url" => url}, scheduled_at: timestamp) + |> Oban.insert() + + _ -> + :ok + end + end + + defp stream_update(%{activity_id: activity_id}) do + Pleroma.Activity.get_by_id(activity_id) + |> Pleroma.Activity.normalize() + |> Pleroma.Web.ActivityPub.ActivityPub.stream_out() + end + + defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val) + defp negative_cache(key, ttl \\ nil), do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl) +end + +defmodule Pleroma.Web.RichMedia.Backfill.Task do + alias Pleroma.Web.RichMedia.Backfill + + def run(args) do + Task.Supervisor.start_child(Pleroma.TaskSupervisor, Backfill, :run, [args], + name: {:global, {:rich_media, args.url_hash}} + ) + end +end diff --git a/lib/pleroma/web/rich_media/card.ex b/lib/pleroma/web/rich_media/card.ex new file mode 100644 index 000000000..36a1ae44a --- /dev/null +++ b/lib/pleroma/web/rich_media/card.ex @@ -0,0 +1,157 @@ +defmodule Pleroma.Web.RichMedia.Card do + use Ecto.Schema + import Ecto.Changeset + import Ecto.Query + + alias Pleroma.Activity + alias Pleroma.HTML + alias Pleroma.Object + alias Pleroma.Repo + alias Pleroma.Web.RichMedia.Backfill + alias Pleroma.Web.RichMedia.Parser + + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + + @type t :: %__MODULE__{} + + schema "rich_media_card" do + field(:url_hash, :binary) + field(:fields, :map) + + timestamps() + end + + @doc false + def changeset(card, attrs) do + card + |> cast(attrs, [:url_hash, :fields]) + |> validate_required([:url_hash, :fields]) + |> unique_constraint(:url_hash) + end + + @spec create(String.t(), map()) :: {:ok, t()} + def create(url, fields) do + url_hash = url_to_hash(url) + + fields = Map.put_new(fields, "url", url) + + %__MODULE__{} + |> changeset(%{url_hash: url_hash, fields: fields}) + |> Repo.insert(on_conflict: {:replace, [:fields]}, conflict_target: :url_hash) + end + + @spec delete(String.t()) :: {:ok, Ecto.Schema.t()} | {:error, Ecto.Changeset.t()} | :ok + def delete(url) do + url_hash = url_to_hash(url) + @cachex.del(:rich_media_cache, url_hash) + + case get_by_url(url) do + %__MODULE__{} = card -> Repo.delete(card) + nil -> :ok + end + end + + @spec get_by_url(String.t() | nil) :: t() | nil | :error + def get_by_url(url) when is_binary(url) do + if @config_impl.get([:rich_media, :enabled]) do + url_hash = url_to_hash(url) + + @cachex.fetch!(:rich_media_cache, url_hash, fn _ -> + result = + __MODULE__ + |> where(url_hash: ^url_hash) + |> Repo.one() + + case result do + %__MODULE__{} = card -> {:commit, card} + _ -> {:ignore, nil} + end + end) + else + :error + end + end + + def get_by_url(nil), do: nil + + @spec get_or_backfill_by_url(String.t(), map()) :: t() | nil + def get_or_backfill_by_url(url, backfill_opts \\ %{}) do + case get_by_url(url) do + %__MODULE__{} = card -> + card + + nil -> + backfill_opts = Map.put(backfill_opts, :url, url) + + Backfill.start(backfill_opts) + + nil + + :error -> + nil + end + end + + @spec get_by_object(Object.t()) :: t() | nil | :error + def get_by_object(object) do + case HTML.extract_first_external_url_from_object(object) do + nil -> nil + url -> get_or_backfill_by_url(url) + end + end + + @spec get_by_activity(Activity.t()) :: t() | nil | :error + # Fake/Draft activity + def get_by_activity(%Activity{id: "pleroma:fakeid"} = activity) do + with %Object{} = object <- Object.normalize(activity, fetch: false), + url when not is_nil(url) <- HTML.extract_first_external_url_from_object(object) do + case get_by_url(url) do + # Cache hit + %__MODULE__{} = card -> + card + + # Cache miss, but fetch for rendering the Draft + _ -> + with {:ok, fields} <- Parser.parse(url), + {:ok, card} <- create(url, fields) do + card + else + _ -> nil + end + end + else + _ -> + nil + end + end + + def get_by_activity(activity) do + with %Object{} = object <- Object.normalize(activity, fetch: false), + {_, nil} <- {:cached, get_cached_url(object, activity.id)} do + nil + else + {:cached, url} -> + get_or_backfill_by_url(url, %{activity_id: activity.id}) + + _ -> + :error + end + end + + @spec url_to_hash(String.t()) :: String.t() + def url_to_hash(url) do + :crypto.hash(:sha256, url) |> Base.encode16(case: :lower) + end + + defp get_cached_url(object, activity_id) do + key = "URL|#{activity_id}" + + @cachex.fetch!(:scrubber_cache, key, fn _ -> + url = HTML.extract_first_external_url_from_object(object) + Activity.HTML.add_cache_key_for(activity_id, key) + + {:commit, url} + end) + end +end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index a711dc436..119994458 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -3,65 +3,13 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.RichMedia.Helpers do - alias Pleroma.Activity - alias Pleroma.HTML - alias Pleroma.Object - alias Pleroma.Web.RichMedia.Parser - - @cachex Pleroma.Config.get([:cachex, :provider], Cachex) - - @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) - - @options [ - pool: :media, - max_body: 2_000_000, - recv_timeout: 2_000 - ] - - def fetch_data_for_object(object) do - with true <- @config_impl.get([:rich_media, :enabled]), - {:ok, page_url} <- - HTML.extract_first_external_url_from_object(object), - {:ok, rich_media} <- Parser.parse(page_url) do - %{page_url: page_url, rich_media: rich_media} - else - _ -> %{} - end - end - - def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do - with true <- @config_impl.get([:rich_media, :enabled]), - %Object{} = object <- Object.normalize(activity, fetch: false) do - if object.data["fake"] do - fetch_data_for_object(object) - else - key = "URL|#{activity.id}" - - @cachex.fetch!(:scrubber_cache, key, fn _ -> - result = fetch_data_for_object(object) - - cond do - match?(%{page_url: _, rich_media: _}, result) -> - Activity.HTML.add_cache_key_for(activity.id, key) - {:commit, result} - - true -> - {:ignore, %{}} - end - end) - end - else - _ -> %{} - end - end - - def fetch_data_for_activity(_), do: %{} + alias Pleroma.Config def rich_media_get(url) do headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] head_check = - case Pleroma.HTTP.head(url, headers, @options) do + case Pleroma.HTTP.head(url, headers, http_options()) do # If the HEAD request didn't reach the server for whatever reason, # we assume the GET that comes right after won't either {:error, _} = e -> @@ -76,7 +24,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do :ok end - with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options) + with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, http_options()) end defp check_content_type(headers) do @@ -92,12 +40,13 @@ defmodule Pleroma.Web.RichMedia.Helpers do end end - @max_body @options[:max_body] defp check_content_length(headers) do + max_body = Keyword.get(http_options(), :max_body) + case List.keyfind(headers, "content-length", 0) do {_, maybe_content_length} -> case Integer.parse(maybe_content_length) do - {content_length, ""} when content_length <= @max_body -> :ok + {content_length, ""} when content_length <= max_body -> :ok {_, ""} -> {:error, :body_too_large} _ -> :ok end @@ -106,4 +55,11 @@ defmodule Pleroma.Web.RichMedia.Helpers do :ok end end + + defp http_options do + [ + pool: :media, + max_body: Config.get([:rich_media, :max_body], 5_000_000) + ] + end end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index a73fbc4b9..37cf29029 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -5,134 +5,28 @@ defmodule Pleroma.Web.RichMedia.Parser do require Logger - @cachex Pleroma.Config.get([:cachex, :provider], Cachex) @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) defp parsers do Pleroma.Config.get([:rich_media, :parsers]) end - def parse(nil), do: {:error, "No URL provided"} + def parse(nil), do: nil @spec parse(String.t()) :: {:ok, map()} | {:error, any()} def parse(url) do with :ok <- validate_page_url(url), - {:ok, data} <- get_cached_or_parse(url), - {:ok, _} <- set_ttl_based_on_image(data, url) do + {:ok, data} <- parse_url(url) do + data = Map.put(data, "url", url) {:ok, data} end end - defp get_cached_or_parse(url) do - case @cachex.fetch(:rich_media_cache, url, fn -> - case parse_url(url) do - {:ok, _} = res -> - {:commit, res} - - {:error, reason} = e -> - # Unfortunately we have to log errors here, instead of doing that - # along with ttl setting at the bottom. Otherwise we can get log spam - # if more than one process was waiting for the rich media card - # while it was generated. Ideally we would set ttl here as well, - # so we don't override it number_of_waiters_on_generation - # times, but one, obviously, can't set ttl for not-yet-created entry - # and Cachex doesn't support returning ttl from the fetch callback. - log_error(url, reason) - {:commit, e} - end - end) do - {action, res} when action in [:commit, :ok] -> - case res do - {:ok, _data} = res -> - res - - {:error, reason} = e -> - if action == :commit, do: set_error_ttl(url, reason) - e - end - - {:error, e} -> - {:error, {:cachex_error, e}} - end - end - - defp set_error_ttl(_url, :body_too_large), do: :ok - defp set_error_ttl(_url, {:content_type, _}), do: :ok - - # The TTL is not set for the errors above, since they are unlikely to change - # with time - - defp set_error_ttl(url, _reason) do - ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) - @cachex.expire(:rich_media_cache, url, ttl) - :ok - end - - defp log_error(url, {:invalid_metadata, data}) do - Logger.debug(fn -> "Incomplete or invalid metadata for #{url}: #{inspect(data)}" end) - end - - defp log_error(url, reason) do - Logger.warning(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) - end - - @doc """ - Set the rich media cache based on the expiration time of image. - - Adopt behaviour `Pleroma.Web.RichMedia.Parser.TTL` - - ## Example - - defmodule MyModule do - @behaviour Pleroma.Web.RichMedia.Parser.TTL - def ttl(data, url) do - image_url = Map.get(data, :image) - # do some parsing in the url and get the ttl of the image - # and return ttl is unix time - parse_ttl_from_url(image_url) - end - end - - Define the module in the config - - config :pleroma, :rich_media, - ttl_setters: [MyModule] - """ - @spec set_ttl_based_on_image(map(), String.t()) :: - {:ok, integer() | :noop} | {:error, :no_key} - def set_ttl_based_on_image(data, url) do - case get_ttl_from_image(data, url) do - ttl when is_number(ttl) -> - ttl = ttl * 1000 - - case @cachex.expire_at(:rich_media_cache, url, ttl) do - {:ok, true} -> {:ok, ttl} - {:ok, false} -> {:error, :no_key} - end - - _ -> - {:ok, :noop} - end - end - - defp get_ttl_from_image(data, url) do - [:rich_media, :ttl_setters] - |> Pleroma.Config.get() - |> Enum.reduce({:ok, nil}, fn - module, {:ok, _ttl} -> - module.ttl(data, url) - - _, error -> - error - end) - end - - def parse_url(url) do + defp parse_url(url) do with {:ok, %Tesla.Env{body: html}} <- Pleroma.Web.RichMedia.Helpers.rich_media_get(url), {:ok, html} <- Floki.parse_document(html) do html |> maybe_parse() - |> Map.put("url", url) |> clean_parsed_data() |> check_parsed_data() end diff --git a/lib/pleroma/web/rich_media/parser/ttl.ex b/lib/pleroma/web/rich_media/parser/ttl.ex index b51298bd8..7e56375ff 100644 --- a/lib/pleroma/web/rich_media/parser/ttl.ex +++ b/lib/pleroma/web/rich_media/parser/ttl.ex @@ -4,4 +4,17 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL do @callback ttl(map(), String.t()) :: integer() | nil + + @spec process(map(), String.t()) :: {:ok, integer() | nil} + def process(data, url) do + [:rich_media, :ttl_setters] + |> Pleroma.Config.get() + |> Enum.reduce_while({:ok, nil}, fn + module, acc -> + case module.ttl(data, url) do + ttl when is_number(ttl) -> {:halt, {:ok, ttl}} + _ -> {:cont, acc} + end + end) + end end diff --git a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex index a0d567c42..1172a120a 100644 --- a/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex +++ b/lib/pleroma/web/rich_media/parser/ttl/aws_signed_url.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do @impl true def ttl(data, _url) do - image = Map.get(data, :image) + image = Map.get(data, "image") if aws_signed_url?(image) do image @@ -15,14 +15,15 @@ defmodule Pleroma.Web.RichMedia.Parser.TTL.AwsSignedUrl do |> format_query_params() |> get_expiration_timestamp() else - {:error, "Not aws signed url #{inspect(image)}"} + nil end end defp aws_signed_url?(image) when is_binary(image) and image != "" do %URI{host: host, query: query} = URI.parse(image) - String.contains?(host, "amazonaws.com") and String.contains?(query, "X-Amz-Expires") + is_binary(host) and String.contains?(host, "amazonaws.com") and + is_binary(query) and String.contains?(query, "X-Amz-Expires") end defp aws_signed_url?(_), do: nil diff --git a/lib/pleroma/web/rich_media/parser/ttl/opengraph.ex b/lib/pleroma/web/rich_media/parser/ttl/opengraph.ex new file mode 100644 index 000000000..b06889669 --- /dev/null +++ b/lib/pleroma/web/rich_media/parser/ttl/opengraph.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.RichMedia.Parser.TTL.Opengraph do + @behaviour Pleroma.Web.RichMedia.Parser.TTL + + @impl true + def ttl(%{"ttl" => ttl_string}, _url) when is_binary(ttl_string) do + try do + ttl = String.to_integer(ttl_string) + now = DateTime.utc_now() |> DateTime.to_unix() + now + ttl + rescue + _ -> nil + end + end + + def ttl(_, _), do: nil +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index f0414cc35..56c457e90 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -300,6 +300,11 @@ defmodule Pleroma.Web.Router do post("/frontends/install", FrontendController, :install) post("/backups", AdminAPIController, :create_backup) + + get("/rules", RuleController, :index) + post("/rules", RuleController, :create) + patch("/rules/:id", RuleController, :update) + delete("/rules/:id", RuleController, :delete) end # AdminAPI: admins and mods (staff) can perform these actions (if privileged by role) @@ -641,6 +646,7 @@ defmodule Pleroma.Web.Router do patch("/accounts/update_credentials", AccountController, :update_credentials) get("/accounts/relationships", AccountController, :relationships) + get("/accounts/familiar_followers", AccountController, :familiar_followers) get("/accounts/:id/lists", AccountController, :lists) get("/accounts/:id/identity_proofs", AccountController, :identity_proofs) get("/endorsements", AccountController, :endorsements) @@ -772,11 +778,11 @@ defmodule Pleroma.Web.Router do get("/instance", InstanceController, :show) get("/instance/peers", InstanceController, :peers) + get("/instance/rules", InstanceController, :rules) get("/statuses", StatusController, :index) get("/statuses/:id", StatusController, :show) get("/statuses/:id/context", StatusController, :context) - get("/statuses/:id/card", StatusController, :card) get("/statuses/:id/favourited_by", StatusController, :favourited_by) get("/statuses/:id/reblogged_by", StatusController, :reblogged_by) get("/statuses/:id/history", StatusController, :show_history) diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index 26fb8af84..e653b3338 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -155,7 +155,16 @@ defmodule Pleroma.Web.WebFinger do end end + @cachex Pleroma.Config.get([:cachex, :provider], Cachex) def find_lrdd_template(domain) do + @cachex.fetch!(:host_meta_cache, domain, fn _ -> + {:commit, fetch_lrdd_template(domain)} + end) + rescue + e -> {:error, "Cachex error: #{inspect(e)}"} + end + + defp fetch_lrdd_template(domain) do # WebFinger is restricted to HTTPS - https://tools.ietf.org/html/rfc7033#section-9.1 meta_url = "https://#{domain}/.well-known/host-meta" @@ -168,7 +177,7 @@ defmodule Pleroma.Web.WebFinger do end end - defp get_address_from_domain(domain, encoded_account) when is_binary(domain) do + defp get_address_from_domain(domain, "acct:" <> _ = encoded_account) when is_binary(domain) do case find_lrdd_template(domain) do {:ok, template} -> String.replace(template, "{uri}", encoded_account) @@ -178,6 +187,11 @@ defmodule Pleroma.Web.WebFinger do end end + defp get_address_from_domain(domain, account) when is_binary(domain) do + encoded_account = URI.encode("acct:#{account}") + get_address_from_domain(domain, encoded_account) + end + defp get_address_from_domain(_, _), do: {:error, :webfinger_no_domain} @spec finger(String.t()) :: {:ok, map()} | {:error, any()} @@ -192,9 +206,7 @@ defmodule Pleroma.Web.WebFinger do URI.parse(account).host end - encoded_account = URI.encode("acct:#{account}") - - with address when is_binary(address) <- get_address_from_domain(domain, encoded_account), + with address when is_binary(address) <- get_address_from_domain(domain, account), {:ok, %{status: status, body: body, headers: headers}} when status in 200..299 <- HTTP.get( address, @@ -216,10 +228,28 @@ defmodule Pleroma.Web.WebFinger do _ -> {:error, {:content_type, nil}} end + |> case do + {:ok, data} -> validate_webfinger(address, data) + error -> error + end else error -> Logger.debug("Couldn't finger #{account}: #{inspect(error)}") error end end + + defp validate_webfinger(request_url, %{"subject" => "acct:" <> acct = subject} = data) do + with [_name, acct_host] <- String.split(acct, "@"), + {_, url} <- {:address, get_address_from_domain(acct_host, subject)}, + %URI{host: request_host} <- URI.parse(request_url), + %URI{host: acct_host} <- URI.parse(url), + {_, true} <- {:hosts_match, acct_host == request_host} do + {:ok, data} + else + _ -> {:error, {:webfinger_invalid, request_url, data}} + end + end + + defp validate_webfinger(url, data), do: {:error, {:webfinger_invalid, url, data}} end diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index 1dddd8d2e..8b2052c23 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -52,7 +52,8 @@ defmodule Pleroma.Workers.ReceiverWorker do {:error, {:reject, reason}} -> {:cancel, reason} {:signature, false} -> {:cancel, :invalid_signature} {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason} - e -> e + {:error, _} = e -> e + e -> {:error, e} end end end diff --git a/lib/pleroma/workers/rich_media_expiration_worker.ex b/lib/pleroma/workers/rich_media_expiration_worker.ex new file mode 100644 index 000000000..d7ae497a7 --- /dev/null +++ b/lib/pleroma/workers/rich_media_expiration_worker.ex @@ -0,0 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.RichMediaExpirationWorker do + alias Pleroma.Web.RichMedia.Card + + use Oban.Worker, + queue: :rich_media_expiration + + @impl Oban.Worker + def perform(%Job{args: %{"url" => url} = _args}) do + Card.delete(url) + end +end |