diff options
Diffstat (limited to 'lib')
99 files changed, 2487 insertions, 536 deletions
diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex deleted file mode 100644 index f32492169..000000000 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ /dev/null @@ -1,113 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Benchmark do - import Mix.Pleroma - use Mix.Task - - def run(["search"]) do - start_pleroma() - - Benchee.run(%{ - "search" => fn -> - Pleroma.Activity.search(nil, "cofe") - end - }) - end - - def run(["tag"]) do - start_pleroma() - - Benchee.run(%{ - "tag" => fn -> - %{"type" => "Create", "tag" => "cofe"} - |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() - end - }) - end - - def run(["render_timeline", nickname | _] = args) do - start_pleroma() - user = Pleroma.User.get_by_nickname(nickname) - - activities = - %{} - |> Map.put("type", ["Create", "Announce"]) - |> Map.put("blocking_user", user) - |> Map.put("muting_user", user) - |> Map.put("user", user) - |> Map.put("limit", 4096) - |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() - |> Enum.reverse() - - inputs = %{ - "1 activity" => Enum.take_random(activities, 1), - "10 activities" => Enum.take_random(activities, 10), - "20 activities" => Enum.take_random(activities, 20), - "40 activities" => Enum.take_random(activities, 40), - "80 activities" => Enum.take_random(activities, 80) - } - - inputs = - if Enum.at(args, 2) == "extended" do - Map.merge(inputs, %{ - "200 activities" => Enum.take_random(activities, 200), - "500 activities" => Enum.take_random(activities, 500), - "2000 activities" => Enum.take_random(activities, 2000), - "4096 activities" => Enum.take_random(activities, 4096) - }) - else - inputs - end - - Benchee.run( - %{ - "Standart rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity - }) - end - }, - inputs: inputs - ) - end - - def run(["adapters"]) do - start_pleroma() - - :ok = - Pleroma.Gun.Conn.open( - "https://httpbin.org/stream-bytes/1500", - :gun_connections - ) - - Process.sleep(1_500) - - Benchee.run( - %{ - "Without conn and without pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - pool: :no_pool, - receive_conn: false - ) - end, - "Without conn and with pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false) - end, - "With reused conn and without pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool) - end, - "With reused conn and with pool" => fn -> - {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") - end - }, - parallel: 10 - ) - end -end diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..8379a0c25 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -0,0 +1,145 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Meilisearch do + require Pleroma.Constants + + import Mix.Pleroma + import Ecto.Query + + import Pleroma.Search.Meilisearch, + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1] + + def run(["index"]) do + start_pleroma() + Pleroma.HTML.compile_scrubbers() + + meili_version = + ( + {:ok, result} = meili_get("/version") + + result["pkgVersion"] + ) + + # The ranking rule syntax was changed but nothing about that is mentioned in the changelog + if not Version.match?(meili_version, ">= 0.25.0") do + raise "Meilisearch <0.24.0 not supported" + end + + {:ok, _} = + meili_post( + "/indexes/objects/settings/ranking-rules", + [ + "published:desc", + "words", + "exactness", + "proximity", + "typo", + "attribute", + "sort" + ] + ) + + {:ok, _} = + meili_post( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) + + IO.puts("Created indices. Starting to insert posts.") + + chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size]) + + Pleroma.Repo.transaction( + fn -> + query = + from(Pleroma.Object, + # Only index public and unlisted posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or + fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())), + order_by: [desc: fragment("data->'published'")] + ) + + count = query |> Pleroma.Repo.aggregate(:count, :data) + IO.puts("Entries to index: #{count}") + + Pleroma.Repo.stream( + query, + timeout: :infinity + ) + |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) + |> Stream.filter(fn o -> not is_nil(o) end) + |> Stream.chunk_every(chunk_size) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) + + # Reset to the beginning of the line and rewrite it + IO.write("\r") + IO.write("Indexed #{new_acc} entries") + + {[objects], new_acc} + end) + |> Stream.each(fn objects -> + result = + meili_put( + "/indexes/objects/documents", + objects + ) + + with {:ok, res} <- result do + if not Map.has_key?(res, "uid") do + IO.puts("\nFailed to index: #{inspect(result)}") + end + else + e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}") + end + end) + |> Stream.run() + end, + timeout: :infinity + ) + + IO.write("\n") + end + + def run(["clear"]) do + start_pleroma() + + meili_delete("/indexes/objects/documents") + end + + def run(["show-keys", master_key]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, "/keys"), + [{"Authorization", "Bearer #{master_key}"}] + ) + + decoded = Jason.decode!(result.body) + + if decoded["results"] do + Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} -> + IO.puts("#{desc}: #{key}") + end) + else + IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}") + end + end + + def run(["stats"]) do + start_pleroma() + + {:ok, result} = meili_get("/indexes/objects/stats") + IO.puts("Number of entries: #{result["numberOfDocuments"]}") + IO.puts("Indexing? #{result["isIndexing"]}") + end +end diff --git a/lib/phoenix/transports/web_socket/raw.ex b/lib/phoenix/transports/web_socket/raw.ex index 8cf9c32a2..cf4fda79f 100644 --- a/lib/phoenix/transports/web_socket/raw.ex +++ b/lib/phoenix/transports/web_socket/raw.ex @@ -26,7 +26,6 @@ defmodule Phoenix.Transports.WebSocket.Raw do conn |> fetch_query_params |> Transport.transport_log(opts[:transport_log]) - |> Transport.force_ssl(handler, endpoint, opts) |> Transport.check_origin(handler, endpoint, opts) case conn do diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 3556aaf9e..8a512dc57 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -368,7 +368,7 @@ defmodule Pleroma.Activity do ) end - defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e68a3c57e..8fa6f3fae 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -54,7 +54,6 @@ defmodule Pleroma.Application do Config.DeprecationWarnings.warn() Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.ApplicationRequirements.verify!() - setup_instrumenters() load_custom_modules() Pleroma.Docs.JSON.compile() limiters_setup() @@ -91,6 +90,7 @@ defmodule Pleroma.Application do # Define workers and child supervisors to be supervised children = [ + Pleroma.PromEx, Pleroma.Repo, Config.TransferTask, Pleroma.Emoji, @@ -138,7 +138,7 @@ defmodule Pleroma.Application do num else e -> - Logger.warn( + Logger.warning( "Could not get the postgres version: #{inspect(e)}.\nSetting the default value of 9.6" ) @@ -170,29 +170,6 @@ defmodule Pleroma.Application do end end - defp setup_instrumenters do - require Prometheus.Registry - - if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do - :ok = - :telemetry.attach( - "prometheus-ecto", - [:pleroma, :repo, :query], - &Pleroma.Repo.Instrumenter.handle_event/4, - %{} - ) - - Pleroma.Repo.Instrumenter.setup() - end - - Pleroma.Web.Endpoint.MetricsExporter.setup() - Pleroma.Web.Endpoint.PipelineInstrumenter.setup() - - # Note: disabled until prometheus-phx is integrated into prometheus-phoenix: - # Pleroma.Web.Endpoint.Instrumenter.setup() - PrometheusPhx.setup() - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), @@ -322,7 +299,11 @@ defmodule Pleroma.Application do def limiters_setup do config = Config.get(ConcurrentLimiter, []) - [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy] + [ + Pleroma.Web.RichMedia.Helpers, + Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, + Pleroma.Search + ] |> Enum.each(fn module -> mod_config = Keyword.get(config, module, []) diff --git a/lib/pleroma/application_requirements.ex b/lib/pleroma/application_requirements.ex index 44b1c1705..1dbfea3e2 100644 --- a/lib/pleroma/application_requirements.ex +++ b/lib/pleroma/application_requirements.ex @@ -34,7 +34,7 @@ defmodule Pleroma.ApplicationRequirements do defp check_welcome_message_config!(:ok) do if Pleroma.Config.get([:welcome, :email, :enabled], false) and not Pleroma.Emails.Mailer.enabled?() do - Logger.warn(""" + Logger.warning(""" To send welcome emails, you need to enable the mailer. Welcome emails will NOT be sent with the current config. @@ -53,7 +53,7 @@ defmodule Pleroma.ApplicationRequirements do def check_confirmation_accounts!(:ok) do if Pleroma.Config.get([:instance, :account_activation_required]) && not Pleroma.Emails.Mailer.enabled?() do - Logger.warn(""" + Logger.warning(""" Account activation is required, but the mailer is disabled. Users will NOT be able to confirm their accounts with this config. Either disable account activation or enable the mailer. @@ -168,8 +168,6 @@ defmodule Pleroma.ApplicationRequirements do check_filter(Pleroma.Upload.Filter.Exiftool.ReadDescription, "exiftool"), check_filter(Pleroma.Upload.Filter.Mogrify, "mogrify"), check_filter(Pleroma.Upload.Filter.Mogrifun, "mogrify"), - check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "mogrify"), - check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "convert"), check_filter(Pleroma.Upload.Filter.AnalyzeMetadata, "ffprobe") ] diff --git a/lib/pleroma/config/deprecation_warnings.ex b/lib/pleroma/config/deprecation_warnings.ex index b53b15d95..1cd3241ea 100644 --- a/lib/pleroma/config/deprecation_warnings.ex +++ b/lib/pleroma/config/deprecation_warnings.ex @@ -24,7 +24,7 @@ defmodule Pleroma.Config.DeprecationWarnings do filters = Config.get([Pleroma.Upload]) |> Keyword.get(:filters, []) if Pleroma.Upload.Filter.Exiftool in filters do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using Exiftool as a filter instead of Exiftool.StripLocation. This should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -63,7 +63,7 @@ defmodule Pleroma.Config.DeprecationWarnings do |> Enum.any?(fn {_, v} -> Enum.any?(v, &is_binary/1) end) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the SimplePolicy configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -121,7 +121,7 @@ defmodule Pleroma.Config.DeprecationWarnings do has_strings = Config.get([:instance, :quarantined_instances]) |> Enum.any?(&is_binary/1) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the quarantined_instances configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -158,7 +158,7 @@ defmodule Pleroma.Config.DeprecationWarnings do has_strings = Config.get([:mrf, :transparency_exclusions]) |> Enum.any?(&is_binary/1) if has_strings do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using strings in the transparency_exclusions configuration instead of tuples. They should work for now, but you are advised to change to the new configuration to prevent possible issues later: @@ -193,7 +193,7 @@ defmodule Pleroma.Config.DeprecationWarnings do def check_hellthread_threshold do if Config.get([:mrf_hellthread, :threshold]) do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! You are using the old configuration mechanism for the hellthread filter. Please check config.md. """) @@ -274,7 +274,7 @@ defmodule Pleroma.Config.DeprecationWarnings do if warning == "" do :ok else - Logger.warn(warning_preface <> warning) + Logger.warning(warning_preface <> warning) :error end end @@ -284,7 +284,7 @@ defmodule Pleroma.Config.DeprecationWarnings do whitelist = Config.get([:media_proxy, :whitelist]) if Enum.any?(whitelist, &(not String.starts_with?(&1, "http"))) do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using old format (only domain) for MediaProxy whitelist option. Setting should work for now, but you are advised to change format to scheme with port to prevent possible issues later. """) @@ -299,7 +299,7 @@ defmodule Pleroma.Config.DeprecationWarnings do pool_config = Config.get(:connections_pool) if timeout = pool_config[:await_up_timeout] do - Logger.warn(""" + Logger.warning(""" !!!DEPRECATION WARNING!!! Your config is using old setting `config :pleroma, :connections_pool, await_up_timeout`. Please change to `config :pleroma, :connections_pool, connect_timeout` to ensure compatibility with future releases. """) @@ -331,7 +331,7 @@ defmodule Pleroma.Config.DeprecationWarnings do "\n* `:timeout` options in #{pool_name} pool is now `:recv_timeout`" end) - Logger.warn(Enum.join([warning_preface | pool_warnings])) + Logger.warning(Enum.join([warning_preface | pool_warnings])) Config.put(:pools, updated_config) :error diff --git a/lib/pleroma/config/getting.ex b/lib/pleroma/config/getting.ex index f9b66bba6..ec93fd02a 100644 --- a/lib/pleroma/config/getting.ex +++ b/lib/pleroma/config/getting.ex @@ -5,4 +5,11 @@ defmodule Pleroma.Config.Getting do @callback get(any()) :: any() @callback get(any(), any()) :: any() + + def get(key), do: get(key, nil) + def get(key, default), do: impl().get(key, default) + + def impl do + Application.get_env(:pleroma, :config_impl, Pleroma.Config) + end end diff --git a/lib/pleroma/config/oban.ex b/lib/pleroma/config/oban.ex index 483d2bb79..836f0c1a7 100644 --- a/lib/pleroma/config/oban.ex +++ b/lib/pleroma/config/oban.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Config.Oban do You are using old workers in Oban crontab settings, which were removed. Please, remove setting from crontab in your config file (prod.secret.exs): #{inspect(setting)} """ - |> Logger.warn() + |> Logger.warning() List.delete(acc, setting) else diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 44a984019..91885347f 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -55,8 +55,7 @@ defmodule Pleroma.Config.TransferTask do started_applications = Application.started_applications() - # TODO: some problem with prometheus after restart! - reject = [nil, :prometheus, :postgrex] + reject = [nil, :postgrex] reject = if restart_pleroma? do @@ -145,7 +144,7 @@ defmodule Pleroma.Config.TransferTask do error_msg = "updating env causes error, group: #{inspect(group)}, key: #{inspect(key)}, value: #{inspect(value)} error: #{inspect(error)}" - Logger.warn(error_msg) + Logger.warning(error_msg) nil end @@ -179,12 +178,12 @@ defmodule Pleroma.Config.TransferTask do :ok = Application.start(app) else nil -> - Logger.warn("#{app} is not started.") + Logger.warning("#{app} is not started.") error -> error |> inspect() - |> Logger.warn() + |> Logger.warning() end end diff --git a/lib/pleroma/constants.ex b/lib/pleroma/constants.ex index c2e577b49..d27d92278 100644 --- a/lib/pleroma/constants.ex +++ b/lib/pleroma/constants.ex @@ -85,4 +85,19 @@ defmodule Pleroma.Constants do ) const(upload_object_types, do: ["Document", "Image"]) + + const(activity_json_canonical_mime_type, + do: "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"" + ) + + const(activity_json_mime_types, + do: [ + "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", + "application/activity+json" + ] + ) + + const(public_streams, + do: ["public", "public:local", "public:media", "public:local:media"] + ) end diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 6508f1947..456a8fd54 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Docs.Generator do # This shouldn't be needed as all modules are expected to have module_info/1, # but in test enviroments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). + Code.ensure_loaded(module) + if function_exported?(module, :module_info, 1) do module.module_info(:attributes) |> Keyword.get_values(:behaviour) diff --git a/lib/pleroma/emoji/loader.ex b/lib/pleroma/emoji/loader.ex index 97d4b8f70..eb6f6816b 100644 --- a/lib/pleroma/emoji/loader.ex +++ b/lib/pleroma/emoji/loader.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Emoji.Loader do Logger.info("Found emoji packs: #{Enum.join(packs, ", ")}") if not Enum.empty?(files) do - Logger.warn( + Logger.warning( "Found files in the emoji folder. These will be ignored, please move them to a subdirectory\nFound files: #{Enum.join(files, ", ")}" ) end diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index a46c3e381..11d5af2fb 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -124,7 +124,7 @@ defmodule Pleroma.Formatter do end def markdown_to_html(text) do - Earmark.as_html!(text, %Earmark.Options{compact_output: true}) + Earmark.as_html!(text, %Earmark.Options{compact_output: true, smartypants: false}) end def html_escape({text, mentions, hashtags}, type) do diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 7c5785def..804cd11c7 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening proxied connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) @@ -90,7 +90,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening socks proxied connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) @@ -106,7 +106,7 @@ defmodule Pleroma.Gun.Conn do {:ok, conn, protocol} else error -> - Logger.warn( + Logger.warning( "Opening connection to #{compose_uri_log(uri)} failed with error #{inspect(error)}" ) diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index 24c845fcd..07dfea55b 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -8,11 +8,12 @@ defmodule Pleroma.Helpers.MediaHelper do """ alias Pleroma.HTTP + alias Vix.Vips.Operation require Logger def missing_dependencies do - Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> + Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> if Pleroma.Utils.command_available?(executable) do acc else @@ -22,54 +23,22 @@ defmodule Pleroma.Helpers.MediaHelper do end def image_resize(url, options) do - with executable when is_binary(executable) <- System.find_executable("convert"), - {:ok, args} <- prepare_image_resize_args(options), - {:ok, env} <- HTTP.get(url, [], pool: :media), - {:ok, fifo_path} <- mkfifo() do - args = List.flatten([fifo_path, args]) - run_fifo(fifo_path, env, executable, args) + with {:ok, env} <- HTTP.get(url, [], pool: :media), + {:ok, resized} <- + Operation.thumbnail_buffer(env.body, options.max_width, + height: options.max_height, + size: :VIPS_SIZE_DOWN + ) do + if options[:format] == "png" do + Operation.pngsave_buffer(resized, Q: options[:quality]) + else + Operation.jpegsave_buffer(resized, Q: options[:quality], interlace: true) + end else - nil -> {:error, {:convert, :command_not_found}} {:error, _} = error -> error end end - defp prepare_image_resize_args( - %{max_width: max_width, max_height: max_height, format: "png"} = options - ) do - quality = options[:quality] || 85 - resize = Enum.join([max_width, "x", max_height, ">"]) - - args = [ - "-resize", - resize, - "-quality", - to_string(quality), - "png:-" - ] - - {:ok, args} - end - - defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do - quality = options[:quality] || 85 - resize = Enum.join([max_width, "x", max_height, ">"]) - - args = [ - "-interlace", - "Plane", - "-resize", - resize, - "-quality", - to_string(quality), - "jpg:-" - ] - - {:ok, args} - end - - defp prepare_image_resize_args(_), do: {:error, :missing_options} - # Note: video thumbnail is intentionally not resized (always has original dimensions) def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), diff --git a/lib/pleroma/http/adapter_helper.ex b/lib/pleroma/http/adapter_helper.ex index 252a6aba5..e9bb2023a 100644 --- a/lib/pleroma/http/adapter_helper.ex +++ b/lib/pleroma/http/adapter_helper.ex @@ -70,15 +70,15 @@ defmodule Pleroma.HTTP.AdapterHelper do {:ok, parse_host(host), port} else {_, _} -> - Logger.warn("Parsing port failed #{inspect(proxy)}") + Logger.warning("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} :error -> - Logger.warn("Parsing port failed #{inspect(proxy)}") + Logger.warning("Parsing port failed #{inspect(proxy)}") {:error, :invalid_proxy_port} _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") + Logger.warning("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end @@ -88,7 +88,7 @@ defmodule Pleroma.HTTP.AdapterHelper do {:ok, type, parse_host(host), port} else _ -> - Logger.warn("Parsing proxy failed #{inspect(proxy)}") + Logger.warning("Parsing proxy failed #{inspect(proxy)}") {:error, :invalid_proxy} end end diff --git a/lib/pleroma/http/web_push.ex b/lib/pleroma/http/web_push.ex index ca399b6c8..888079c1e 100644 --- a/lib/pleroma/http/web_push.ex +++ b/lib/pleroma/http/web_push.ex @@ -6,7 +6,11 @@ defmodule Pleroma.HTTP.WebPush do @moduledoc false def post(url, payload, headers, options \\ []) do - list_headers = Map.to_list(headers) + list_headers = + headers + |> Map.to_list() + |> Kernel.++([{"content-type", "octet-stream"}]) + Pleroma.HTTP.post(url, payload, list_headers, options) end end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 9756c66dc..c497a4fb7 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -97,13 +97,9 @@ defmodule Pleroma.Instances.Instance do def reachable?(url_or_host) when is_binary(url_or_host), do: true def set_reachable(url_or_host) when is_binary(url_or_host) do - with host <- host(url_or_host), - %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do - {:ok, _instance} = - existing_record - |> changeset(%{unreachable_since: nil}) - |> Repo.update() - end + %Instance{host: host(url_or_host)} + |> changeset(%{unreachable_since: nil}) + |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host) end def set_reachable(_), do: {:error, nil} @@ -177,7 +173,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") + Logger.warning("Instance.get_or_update_favicon(\"#{host}\") error: #{inspect(e)}") nil end @@ -205,7 +201,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn( + Logger.warning( "Instance.scrape_favicon(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" ) @@ -288,7 +284,7 @@ defmodule Pleroma.Instances.Instance do end rescue e -> - Logger.warn( + Logger.warning( "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" ) diff --git a/lib/pleroma/maintenance.ex b/lib/pleroma/maintenance.ex index eb5a6ef42..1e39b03e6 100644 --- a/lib/pleroma/maintenance.ex +++ b/lib/pleroma/maintenance.ex @@ -20,7 +20,7 @@ defmodule Pleroma.Maintenance do "full" -> Logger.info("Running VACUUM FULL.") - Logger.warn( + Logger.warning( "Re-packing your entire database may take a while and will consume extra disk space during the process." ) diff --git a/lib/pleroma/migrators/support/base_migrator.ex b/lib/pleroma/migrators/support/base_migrator.ex index 3bcd59fd0..ce88caac7 100644 --- a/lib/pleroma/migrators/support/base_migrator.ex +++ b/lib/pleroma/migrators/support/base_migrator.ex @@ -73,7 +73,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do data_migration.state == :manual or data_migration.name in manual_migrations -> message = "Data migration is in manual execution or manual fix mode." update_status(:manual, message) - Logger.warn("#{__MODULE__}: #{message}") + Logger.warning("#{__MODULE__}: #{message}") data_migration.state == :complete -> on_complete(data_migration) @@ -109,7 +109,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do Putting data migration to manual fix mode. Try running `#{__MODULE__}.retry_failed/0`. """ - Logger.warn("#{__MODULE__}: #{message}") + Logger.warning("#{__MODULE__}: #{message}") update_status(:manual, message) on_complete(data_migration()) @@ -125,7 +125,7 @@ defmodule Pleroma.Migrators.Support.BaseMigrator do defp on_complete(data_migration) do if data_migration.feature_lock || feature_state() == :disabled do - Logger.warn( + Logger.warning( "#{__MODULE__}: migration complete but feature is locked; consider enabling." ) diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index aa137d250..fa5baf1a4 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -328,6 +328,52 @@ defmodule Pleroma.Object do end end + def increase_quotes_count(ap_id) do + Object + |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id))) + |> update([o], + set: [ + data: + fragment( + """ + safe_jsonb_set(?, '{quotesCount}', + (coalesce((?->>'quotesCount')::int, 0) + 1)::varchar::jsonb, true) + """, + o.data, + o.data + ) + ] + ) + |> Repo.update_all([]) + |> case do + {1, [object]} -> set_cache(object) + _ -> {:error, "Not found"} + end + end + + def decrease_quotes_count(ap_id) do + Object + |> where([o], fragment("?->>'id' = ?::text", o.data, ^to_string(ap_id))) + |> update([o], + set: [ + data: + fragment( + """ + safe_jsonb_set(?, '{quotesCount}', + (greatest(0, (?->>'quotesCount')::int - 1))::varchar::jsonb, true) + """, + o.data, + o.data + ) + ] + ) + |> Repo.update_all([]) + |> case do + {1, [object]} -> set_cache(object) + _ -> {:error, "Not found"} + end + end + def increase_vote_count(ap_id, name, actor) do with %Object{} = object <- Object.normalize(ap_id, fetch: false), "Question" <- object.data["type"] do diff --git a/lib/pleroma/prom_ex.ex b/lib/pleroma/prom_ex.ex new file mode 100644 index 000000000..6608708b7 --- /dev/null +++ b/lib/pleroma/prom_ex.ex @@ -0,0 +1,49 @@ +defmodule Pleroma.PromEx do + use PromEx, otp_app: :pleroma + + alias PromEx.Plugins + + @impl true + def plugins do + [ + # PromEx built in plugins + Plugins.Application, + Plugins.Beam, + {Plugins.Phoenix, router: Pleroma.Web.Router, endpoint: Pleroma.Web.Endpoint}, + Plugins.Ecto, + Plugins.Oban + # Plugins.PhoenixLiveView, + # Plugins.Absinthe, + # Plugins.Broadway, + + # Add your own PromEx metrics plugins + # Pleroma.Users.PromExPlugin + ] + end + + @impl true + def dashboard_assigns do + [ + datasource_id: Pleroma.Config.get([Pleroma.PromEx, :datasource]), + default_selected_interval: "30s" + ] + end + + @impl true + def dashboards do + [ + # PromEx built in Grafana dashboards + {:prom_ex, "application.json"}, + {:prom_ex, "beam.json"}, + {:prom_ex, "phoenix.json"}, + {:prom_ex, "ecto.json"}, + {:prom_ex, "oban.json"} + # {:prom_ex, "phoenix_live_view.json"}, + # {:prom_ex, "absinthe.json"}, + # {:prom_ex, "broadway.json"}, + + # Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"} + # {:pleroma, "/grafana_dashboards/user_metrics.json"} + ] + end +end diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 515b0c1ff..a50a59b3b 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -11,8 +11,6 @@ defmodule Pleroma.Repo do import Ecto.Query require Logger - defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter) - @doc """ Dynamically loads the repository url from the DATABASE_URL environment variable. diff --git a/lib/pleroma/reverse_proxy.ex b/lib/pleroma/reverse_proxy.ex index 2248c2713..880940d07 100644 --- a/lib/pleroma/reverse_proxy.ex +++ b/lib/pleroma/reverse_proxy.ex @@ -192,7 +192,7 @@ defmodule Pleroma.ReverseProxy do halt(conn) {:error, error, conn} -> - Logger.warn( + Logger.warning( "#{__MODULE__} request to #{url} failed while reading/chunking: #{inspect(error)}" ) diff --git a/lib/pleroma/scheduled_activity.ex b/lib/pleroma/scheduled_activity.ex index a7be58512..63c6cb45b 100644 --- a/lib/pleroma/scheduled_activity.ex +++ b/lib/pleroma/scheduled_activity.ex @@ -6,7 +6,6 @@ defmodule Pleroma.ScheduledActivity do use Ecto.Schema alias Ecto.Multi - alias Pleroma.Config alias Pleroma.Repo alias Pleroma.ScheduledActivity alias Pleroma.User @@ -20,6 +19,8 @@ defmodule Pleroma.ScheduledActivity do @min_offset :timer.minutes(5) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + schema "scheduled_activities" do belongs_to(:user, User, type: FlakeId.Ecto.CompatType) field(:scheduled_at, :naive_datetime) @@ -40,7 +41,11 @@ defmodule Pleroma.ScheduledActivity do %{changes: %{params: %{"media_ids" => media_ids} = params}} = changeset ) when is_list(media_ids) do - media_attachments = Utils.attachments_from_ids(%{media_ids: media_ids}) + media_attachments = + Utils.attachments_from_ids( + %{media_ids: media_ids}, + User.get_cached_by_id(changeset.data.user_id) + ) params = params @@ -83,7 +88,7 @@ defmodule Pleroma.ScheduledActivity do |> where([sa], type(sa.scheduled_at, :date) == type(^scheduled_at, :date)) |> select([sa], count(sa.id)) |> Repo.one() - |> Kernel.>=(Config.get([ScheduledActivity, :daily_user_limit])) + |> Kernel.>=(@config_impl.get([ScheduledActivity, :daily_user_limit])) end def exceeds_total_user_limit?(user_id) do @@ -91,7 +96,7 @@ defmodule Pleroma.ScheduledActivity do |> where(user_id: ^user_id) |> select([sa], count(sa.id)) |> Repo.one() - |> Kernel.>=(Config.get([ScheduledActivity, :total_user_limit])) + |> Kernel.>=(@config_impl.get([ScheduledActivity, :total_user_limit])) end def far_enough?(scheduled_at) when is_binary(scheduled_at) do @@ -119,7 +124,7 @@ defmodule Pleroma.ScheduledActivity do def create(%User{} = user, attrs) do Multi.new() |> Multi.insert(:scheduled_activity, new(user, attrs)) - |> maybe_add_jobs(Config.get([ScheduledActivity, :enabled])) + |> maybe_add_jobs(@config_impl.get([ScheduledActivity, :enabled])) |> Repo.transaction() |> transaction_response end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex new file mode 100644 index 000000000..3b266e59b --- /dev/null +++ b/lib/pleroma/search.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.Search do + alias Pleroma.Workers.SearchIndexingWorker + + def add_to_index(%Pleroma.Activity{id: activity_id}) do + SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id}) + end + + def remove_from_index(%Pleroma.Object{id: object_id}) do + SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id}) + end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end +end diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/search/database_search.ex index 0b9b24aa4..c6311e0c7 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/search/database_search.ex @@ -1,9 +1,10 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Activity.Search do +defmodule Pleroma.Search.DatabaseSearch do alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object.Fetcher alias Pleroma.Pagination alias Pleroma.User @@ -13,8 +14,11 @@ defmodule Pleroma.Activity.Search do import Ecto.Query + @behaviour Pleroma.Search.SearchBackend + + @impl true def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + index_type = if Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) @@ -45,6 +49,12 @@ defmodule Pleroma.Activity.Search do end end + @impl true + def add_to_index(_activity), do: :ok + + @impl true + def remove_from_index(_object), do: :ok + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end @@ -136,8 +146,8 @@ defmodule Pleroma.Activity.Search do ) end - defp maybe_restrict_local(q, user) do - limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) + def maybe_restrict_local(q, user) do + limit = Config.get([:instance, :limit_to_local_content], :unauthenticated) case {limit, user} do {:all, _} -> restrict_local(q) @@ -149,7 +159,7 @@ defmodule Pleroma.Activity.Search do defp restrict_local(q), do: where(q, local: true) - defp maybe_fetch(activities, user, search_query) do + def maybe_fetch(activities, user, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..2bff663e8 --- /dev/null +++ b/lib/pleroma/search/meilisearch.ex @@ -0,0 +1,181 @@ +defmodule Pleroma.Search.Meilisearch do + require Logger + require Pleroma.Constants + + alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config + + import Pleroma.Search.DatabaseSearch + import Ecto.Query + + @behaviour Pleroma.Search.SearchBackend + + defp meili_headers do + private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) + + [{"Content-Type", "application/json"}] ++ + if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}] + end + + def meili_get(path) do + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.get( + Path.join(endpoint, path), + meili_headers() + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_post(path, params) do + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.post( + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers() + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_put(path, params) do + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.request( + :put, + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers(), + [] + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_delete(path) do + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + with {:ok, _} <- + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + [] + ) do + :ok + else + _ -> {:error, "Could not remove from index"} + end + end + + @impl true + def search(user, query, options \\ []) do + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + res = + meili_post( + "/indexes/objects/search", + %{q: query, offset: offset, limit: limit} + ) + + with {:ok, result} <- res do + hits = result["hits"] |> Enum.map(& &1["ap"]) + + try do + hits + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> maybe_fetch(user, query) + |> order_by([object: obj], desc: obj.data["published"]) + |> Pleroma.Repo.all() + rescue + _ -> maybe_fetch([], user, query) + end + end + end + + def object_to_search_data(object) do + # Only index public or unlisted Notes + if not is_nil(object) and object.data["type"] == "Note" and + not is_nil(object.data["content"]) and + (Pleroma.Constants.as_public() in object.data["to"] or + Pleroma.Constants.as_public() in object.data["cc"]) and + object.data["content"] not in ["", "."] do + data = object.data + + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + + content = + with {:ok, scrubbed} <- + FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing), + trimmed <- String.trim(scrubbed) do + trimmed + end + + # Make sure we have a non-empty string + if content != "" do + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + end + end + end + + @impl true + def add_to_index(activity) do + maybe_search_data = object_to_search_data(activity.object) + + if activity.data["type"] == "Create" and maybe_search_data do + result = + meili_put( + "/indexes/objects/documents", + [maybe_search_data] + ) + + with {:ok, %{"status" => "enqueued"}} <- result do + # Added successfully + :ok + else + _ -> + # There was an error, report it + Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") + {:error, result} + end + else + # The post isn't something we can search, that's ok + :ok + end + end + + @impl true + def remove_from_index(object) do + meili_delete("/indexes/objects/documents/#{object.id}") + end +end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex new file mode 100644 index 000000000..a42e2f5f6 --- /dev/null +++ b/lib/pleroma/search/search_backend.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Search.SearchBackend do + @doc """ + Search statuses with a query, restricting to only those the user should have access to. + """ + @callback search(user :: Pleroma.User.t(), query :: String.t(), options :: [any()]) :: [ + Pleroma.Activity.t() + ] + + @doc """ + Add the object associated with the activity to the search index. + + The whole activity is passed, to allow filtering on things such as scope. + """ + @callback add_to_index(activity :: Pleroma.Activity.t()) :: :ok | {:error, any()} + + @doc """ + Remove the object from the index. + + Just the object, as opposed to the whole activity, is passed, since the object + is what contains the actual content and there is no need for fitlering when removing + from index. + """ + @callback remove_from_index(object :: Pleroma.Object.t()) :: {:ok, any()} | {:error, any()} +end diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 384c70fbc..92d395394 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Telemetry.Logger do %{key: key}, _ ) do - Logger.warn(fn -> + Logger.warning(fn -> "Pool worker for #{key}: Client #{inspect(client_pid)} died before releasing the connection with #{inspect(reason)}" end) end diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex index 4aee9326f..bedd7889a 100644 --- a/lib/pleroma/upload.ex +++ b/lib/pleroma/upload.ex @@ -34,7 +34,6 @@ defmodule Pleroma.Upload do """ alias Ecto.UUID - alias Pleroma.Config alias Pleroma.Maps alias Pleroma.Web.ActivityPub.Utils require Logger @@ -76,6 +75,8 @@ defmodule Pleroma.Upload do :path ] + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + defp get_description(upload) do case {upload.description, Pleroma.Config.get([Pleroma.Upload, :default_description])} do {description, _} when is_binary(description) -> description @@ -244,18 +245,18 @@ defmodule Pleroma.Upload do defp url_from_spec(_upload, _base_url, {:url, url}), do: url def base_url do - uploader = Config.get([Pleroma.Upload, :uploader]) - upload_base_url = Config.get([Pleroma.Upload, :base_url]) - public_endpoint = Config.get([uploader, :public_endpoint]) + uploader = @config_impl.get([Pleroma.Upload, :uploader]) + upload_base_url = @config_impl.get([Pleroma.Upload, :base_url]) + public_endpoint = @config_impl.get([uploader, :public_endpoint]) case uploader do Pleroma.Uploaders.Local -> upload_base_url || Pleroma.Web.Endpoint.url() <> "/media/" Pleroma.Uploaders.S3 -> - bucket = Config.get([Pleroma.Uploaders.S3, :bucket]) - truncated_namespace = Config.get([Pleroma.Uploaders.S3, :truncated_namespace]) - namespace = Config.get([Pleroma.Uploaders.S3, :bucket_namespace]) + bucket = @config_impl.get([Pleroma.Uploaders.S3, :bucket]) + truncated_namespace = @config_impl.get([Pleroma.Uploaders.S3, :truncated_namespace]) + namespace = @config_impl.get([Pleroma.Uploaders.S3, :bucket_namespace]) bucket_with_namespace = cond do diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 9a76a998b..7ee643277 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -8,27 +8,28 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do """ require Logger + alias Vix.Vips.Image + alias Vix.Vips.Operation + @behaviour Pleroma.Upload.Filter @spec filter(Pleroma.Upload.t()) :: {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do try do - image = - file - |> Mogrify.open() - |> Mogrify.verbose() + {:ok, image} = Image.new_from_file(file) + {width, height} = {Image.width(image), Image.height(image)} upload = upload - |> Map.put(:width, image.width) - |> Map.put(:height, image.height) - |> Map.put(:blurhash, get_blurhash(file)) + |> Map.put(:width, width) + |> Map.put(:height, height) + |> Map.put(:blurhash, get_blurhash(image)) {:ok, :filtered, upload} rescue e in ErlangError -> - Logger.warn("#{__MODULE__}: #{inspect(e)}") + Logger.warning("#{__MODULE__}: #{inspect(e)}") {:ok, :noop} end end @@ -45,7 +46,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {:ok, :filtered, upload} rescue e in ErlangError -> - Logger.warn("#{__MODULE__}: #{inspect(e)}") + Logger.warning("#{__MODULE__}: #{inspect(e)}") {:ok, :noop} end end @@ -53,7 +54,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do def filter(_), do: {:ok, :noop} defp get_blurhash(file) do - with {:ok, blurhash} <- :eblurhash.magick(file) do + with {:ok, blurhash} <- vips_blurhash(file) do blurhash else _ -> nil @@ -77,7 +78,28 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do %{width: width, height: height} else nil -> {:error, {:ffprobe, :command_not_found}} - {:error, _} = error -> error + error -> {:error, error} + end + end + + defp vips_blurhash(%Vix.Vips.Image{} = image) do + with {:ok, resized_image} <- Operation.thumbnail_image(image, 100), + {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, + max <- max(height, width), + {x, y} <- {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} do + {:ok, rgb} = + if Image.has_alpha?(resized_image) do + # remove alpha channel + resized_image + |> Operation.extract_band!(0, n: 3) + |> Image.write_to_binary() + else + Image.write_to_binary(resized_image) + end + + Blurhash.encode(rgb, width, height, x, y) + else + _ -> nil end end end diff --git a/lib/pleroma/upload/filter/exiftool/read_description.ex b/lib/pleroma/upload/filter/exiftool/read_description.ex index 543b22031..8c1ed82f8 100644 --- a/lib/pleroma/upload/filter/exiftool/read_description.ex +++ b/lib/pleroma/upload/filter/exiftool/read_description.ex @@ -10,8 +10,6 @@ defmodule Pleroma.Upload.Filter.Exiftool.ReadDescription do """ @behaviour Pleroma.Upload.Filter - @spec filter(Pleroma.Upload.t()) :: {:ok, any()} | {:error, String.t()} - def filter(%Pleroma.Upload{description: description}) when is_binary(description), do: {:ok, :noop} diff --git a/lib/pleroma/uploaders/s3.ex b/lib/pleroma/uploaders/s3.ex index 19287c532..7b32bd8a5 100644 --- a/lib/pleroma/uploaders/s3.ex +++ b/lib/pleroma/uploaders/s3.ex @@ -6,7 +6,8 @@ defmodule Pleroma.Uploaders.S3 do @behaviour Pleroma.Uploaders.Uploader require Logger - alias Pleroma.Config + @ex_aws_impl Application.compile_env(:pleroma, [__MODULE__, :ex_aws_impl], ExAws) + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) # The file name is re-encoded with S3's constraints here to comply with previous # links with less strict filenames @@ -22,7 +23,7 @@ defmodule Pleroma.Uploaders.S3 do @impl true def put_file(%Pleroma.Upload{} = upload) do - config = Config.get([__MODULE__]) + config = @config_impl.get([__MODULE__]) bucket = Keyword.get(config, :bucket) streaming = Keyword.get(config, :streaming_enabled) @@ -56,7 +57,7 @@ defmodule Pleroma.Uploaders.S3 do ]) end - case ExAws.request(op) do + case @ex_aws_impl.request(op) do {:ok, _} -> {:ok, {:file, s3_name}} @@ -69,9 +70,9 @@ defmodule Pleroma.Uploaders.S3 do @impl true def delete_file(file) do [__MODULE__, :bucket] - |> Config.get() + |> @config_impl.get() |> ExAws.S3.delete_object(file) - |> ExAws.request() + |> @ex_aws_impl.request() |> case do {:ok, %{status_code: 204}} -> :ok error -> {:error, inspect(error)} @@ -83,3 +84,7 @@ defmodule Pleroma.Uploaders.S3 do String.replace(name, @regex, "-") end end + +defmodule Pleroma.Uploaders.S3.ExAwsAPI do + @callback request(op :: ExAws.Operation.t()) :: {:ok, ExAws.Operation.t()} | {:error, term()} +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index ce125d608..5f98935b3 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1560,7 +1560,7 @@ defmodule Pleroma.User do unmute(muter, mutee) else {who, result} = error -> - Logger.warn( + Logger.warning( "User.unmute/2 failed. #{who}: #{result}, muter_id: #{muter_id}, mutee_id: #{mutee_id}" ) @@ -2136,7 +2136,7 @@ defmodule Pleroma.User do def public_key(_), do: {:error, "key not found"} def get_public_key_for_ap_id(ap_id) do - with {:ok, %User{} = user} <- get_or_fetch_by_ap_id(ap_id), + with %User{} = user <- get_cached_by_ap_id(ap_id), {:ok, public_key} <- public_key(user) do {:ok, public_key} else @@ -2681,6 +2681,8 @@ defmodule Pleroma.User do |> update_and_set_cache() end + def update_last_active_at(user), do: user + def active_user_count(days \\ 30) do active_after = Timex.shift(NaiveDateTime.utc_now(), days: -days) diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 447fca2a1..74e0ec073 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -35,6 +35,8 @@ defmodule Pleroma.User.Backup do timestamps() end + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + def create(user, admin_id \\ nil) do with :ok <- validate_limit(user, admin_id), {:ok, backup} <- user |> new() |> Repo.insert() do @@ -124,7 +126,10 @@ defmodule Pleroma.User.Backup do |> Repo.update() end - def process(%__MODULE__{} = backup) do + def process( + %__MODULE__{} = backup, + processor_module \\ __MODULE__.Processor + ) do set_state(backup, :running, 0) current_pid = self() @@ -132,7 +137,7 @@ defmodule Pleroma.User.Backup do task = Task.Supervisor.async_nolink( Pleroma.TaskSupervisor, - __MODULE__, + processor_module, :do_process, [backup, current_pid] ) @@ -140,25 +145,8 @@ defmodule Pleroma.User.Backup do wait_backup(backup, backup.processed_number, task) end - def do_process(backup, current_pid) do - with {:ok, zip_file} <- export(backup, current_pid), - {:ok, %{size: size}} <- File.stat(zip_file), - {:ok, _upload} <- upload(backup, zip_file) do - backup - |> cast( - %{ - file_size: size, - processed: true, - state: :complete - }, - [:file_size, :processed, :state] - ) - |> Repo.update() - end - end - defp wait_backup(backup, current_processed, task) do - wait_time = Pleroma.Config.get([__MODULE__, :process_wait_time]) + wait_time = @config_impl.get([__MODULE__, :process_wait_time]) receive do {:progress, new_processed} -> @@ -305,7 +293,7 @@ defmodule Pleroma.User.Backup do acc + 1 else {:error, e} -> - Logger.warn( + Logger.warning( "Error processing backup item: #{inspect(e)}\n The item is: #{inspect(i)}" ) @@ -365,3 +353,35 @@ defmodule Pleroma.User.Backup do ) end end + +defmodule Pleroma.User.Backup.ProcessorAPI do + @callback do_process(%Pleroma.User.Backup{}, pid()) :: + {:ok, %Pleroma.User.Backup{}} | {:error, any()} +end + +defmodule Pleroma.User.Backup.Processor do + @behaviour Pleroma.User.Backup.ProcessorAPI + + alias Pleroma.Repo + alias Pleroma.User.Backup + + import Ecto.Changeset + + @impl true + def do_process(backup, current_pid) do + with {:ok, zip_file} <- Backup.export(backup, current_pid), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- Backup.upload(backup, zip_file) do + backup + |> cast( + %{ + file_size: size, + processed: true, + state: :complete + }, + [:file_size, :processed, :state] + ) + |> Repo.update() + end + end +end diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index aee41b0fe..7a8b176cd 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -136,7 +136,7 @@ defmodule Pleroma.Web do namespace: Pleroma.Web # Import convenience functions from controllers - import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1] + import Phoenix.Controller, only: [get_csrf_token: 0, view_module: 1] import Pleroma.Web.ErrorHelpers import Pleroma.Web.Gettext diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 3979d418e..32d1a1037 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -96,6 +96,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop + defp increase_quotes_count_if_quote(%{ + "object" => %{"quoteUrl" => quote_ap_id} = object, + "type" => "Create" + }) do + if is_public?(object) do + Object.increase_quotes_count(quote_ap_id) + end + end + + defp increase_quotes_count_if_quote(_create_data), do: :noop + @object_types ~w[ChatMessage Question Answer Audio Video Image Event Article Note Page] @impl true def persist(%{"type" => type} = object, meta) when type in @object_types do @@ -140,6 +151,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + # Add local posts to search index + if local, do: Pleroma.Search.add_to_index(activity) + {:ok, activity} else %Activity{} = activity -> @@ -299,6 +313,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do with {:ok, activity} <- insert(create_data, local, fake), {:fake, false, activity} <- {:fake, fake, activity}, _ <- increase_replies_count_if_reply(create_data), + _ <- increase_quotes_count_if_quote(create_data), {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity}, {:ok, _actor} <- increase_note_count_if_public(actor, activity), {:ok, _actor} <- update_last_status_at_if_public(actor, activity), @@ -1237,6 +1252,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_unauthenticated(query, _), do: query + defp restrict_quote_url(query, %{quote_url: quote_url}) do + from([_activity, object] in query, + where: fragment("(?)->'quoteUrl' = ?", object.data, ^quote_url) + ) + end + + defp restrict_quote_url(query, _), do: query + defp exclude_poll_votes(query, %{include_poll_votes: true}), do: query defp exclude_poll_votes(query, _) do @@ -1399,6 +1422,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> restrict_instance(opts) |> restrict_announce_object_actor(opts) |> restrict_filtered(opts) + |> restrict_quote_url(opts) |> maybe_restrict_deactivated_users(opts) |> exclude_poll_votes(opts) |> exclude_chat_messages(opts) diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex index 1357c379c..e38a94966 100644 --- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex +++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex @@ -273,12 +273,17 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do end def inbox(%{assigns: %{valid_signature: true}} = conn, %{"nickname" => nickname} = params) do - with %User{} = recipient <- User.get_cached_by_nickname(nickname), - {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]), + with %User{is_active: true} = recipient <- User.get_cached_by_nickname(nickname), + {:ok, %User{is_active: true} = actor} <- User.get_or_fetch_by_ap_id(params["actor"]), true <- Utils.recipient_in_message(recipient, actor, params), params <- Utils.maybe_splice_recipient(recipient.ap_id, params) do Federator.incoming_ap_doc(params) json(conn, "ok") + else + _ -> + conn + |> put_status(:bad_request) + |> json("Invalid request.") end end @@ -287,10 +292,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do json(conn, "ok") end - def inbox(%{assigns: %{valid_signature: false}} = conn, _params) do - conn - |> put_status(:bad_request) - |> json("Invalid HTTP Signature") + def inbox(%{assigns: %{valid_signature: false}, req_headers: req_headers} = conn, params) do + Federator.incoming_ap_doc(%{req_headers: req_headers, params: params}) + json(conn, "ok") end # POST /relay/inbox -or- POST /internal/fetch/inbox @@ -476,7 +480,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do |> json(message) e -> - Logger.warn(fn -> "AP C2S: #{inspect(e)}" end) + Logger.warning(fn -> "AP C2S: #{inspect(e)}" end) conn |> put_status(:bad_request) diff --git a/lib/pleroma/web/activity_pub/builder.ex b/lib/pleroma/web/activity_pub/builder.ex index 8eab3a241..eb0bb0e33 100644 --- a/lib/pleroma/web/activity_pub/builder.ex +++ b/lib/pleroma/web/activity_pub/builder.ex @@ -217,6 +217,7 @@ defmodule Pleroma.Web.ActivityPub.Builder do "tag" => Keyword.values(draft.tags) |> Enum.uniq() } |> add_in_reply_to(draft.in_reply_to) + |> add_quote(draft.quote_post) |> Map.merge(draft.extra) {:ok, data, []} @@ -232,6 +233,16 @@ defmodule Pleroma.Web.ActivityPub.Builder do end end + defp add_quote(object, nil), do: object + + defp add_quote(object, quote_post) do + with %Object{} = quote_object <- Object.normalize(quote_post, fetch: false) do + Map.put(object, "quoteUrl", quote_object.data["id"]) + else + _ -> object + end + end + def chat_message(actor, recipient, content, opts \\ []) do basic = %{ "id" => Utils.generate_object_id(), diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index ff9f84497..7f6dce925 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -54,6 +54,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do @required_description_keys [:key, :related_policy] def filter_one(policy, message) do + Code.ensure_loaded(policy) + should_plug_history? = if function_exported?(policy, :history_awareness, 0) do policy.history_awareness() @@ -188,6 +190,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do def config_descriptions(policies) do Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc -> + Code.ensure_loaded(policy) + if function_exported?(policy, :config_description, 0) do description = @default_description @@ -199,7 +203,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do if Enum.all?(@required_description_keys, &Map.has_key?(description, &1)) do [description | acc] else - Logger.warn( + Logger.warning( "#{policy} config description doesn't have one or all required keys #{inspect(@required_description_keys)}" ) diff --git a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex index 5b6adbb4b..5a4a97626 100644 --- a/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/follow_bot_policy.ex @@ -19,7 +19,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.FollowBotPolicy do try_follow(follower, message) else nil -> - Logger.warn( + Logger.warning( "#{__MODULE__} skipped because of missing `:mrf_follow_bot, :follower_nickname` configuration, the :follower_nickname account does not exist, or the account is not correctly configured as a bot." ) diff --git a/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex new file mode 100644 index 000000000..171b22c5e --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/inline_quote_policy.ex @@ -0,0 +1,78 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy do + @moduledoc "Force a quote line into the message content." + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + defp build_inline_quote(template, url) do + quote_line = String.replace(template, "{url}", "<a href=\"#{url}\">#{url}</a>") + + "<span class=\"quote-inline\"><br/><br/>#{quote_line}</span>" + end + + defp has_inline_quote?(content, quote_url) do + cond do + # Does the quote URL exist in the content? + content =~ quote_url -> true + # Does the content already have a .quote-inline span? + content =~ "<span class=\"quote-inline\">" -> true + # No inline quote found + true -> false + end + end + + defp filter_object(%{"quoteUrl" => quote_url} = object) do + content = object["content"] || "" + + if has_inline_quote?(content, quote_url) do + object + else + template = Pleroma.Config.get([:mrf_inline_quote, :template]) + + content = + if String.ends_with?(content, "</p>"), + do: + String.trim_trailing(content, "</p>") <> + build_inline_quote(template, quote_url) <> "</p>", + else: content <> build_inline_quote(template, quote_url) + + Map.put(object, "content", content) + end + end + + @impl true + def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do + {:ok, Map.put(activity, "object", filter_object(object))} + end + + @impl true + def filter(object), do: {:ok, object} + + @impl true + def describe, do: {:ok, %{}} + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def history_awareness, do: :auto + + @impl true + def config_description do + %{ + key: :mrf_inline_quote, + related_policy: "Pleroma.Web.ActivityPub.MRF.InlineQuotePolicy", + label: "MRF Inline Quote Policy", + type: :group, + description: "Force quote url to appear in post content.", + children: [ + %{ + key: :template, + type: :string, + description: + "The template to append to the post. `{url}` will be replaced with the actual link to the quoted post.", + suggestions: ["<bdi>RT:</bdi> {url}"] + } + ] + } + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex new file mode 100644 index 000000000..f1c573d1b --- /dev/null +++ b/lib/pleroma/web/activity_pub/mrf/quote_to_link_tag_policy.ex @@ -0,0 +1,49 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.MRF.QuoteToLinkTagPolicy do + @moduledoc "Force a Link tag for posts quoting another post. (may break outgoing federation of quote posts with older Pleroma versions)" + @behaviour Pleroma.Web.ActivityPub.MRF.Policy + + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + + require Pleroma.Constants + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def filter(%{"object" => %{"quoteUrl" => _} = object} = activity) do + {:ok, Map.put(activity, "object", filter_object(object))} + end + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def filter(object), do: {:ok, object} + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def describe, do: {:ok, %{}} + + @impl Pleroma.Web.ActivityPub.MRF.Policy + def history_awareness, do: :auto + + defp filter_object(%{"quoteUrl" => quote_url} = object) do + tags = object["tag"] || [] + + if Enum.any?(tags, fn tag -> + CommonFixes.is_object_link_tag(tag) and tag["href"] == quote_url + end) do + object + else + object + |> Map.put( + "tag", + tags ++ + [ + %{ + "type" => "Link", + "mediaType" => Pleroma.Constants.activity_json_canonical_mime_type(), + "href" => quote_url + } + ] + ) + end + end +end diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index f66c379b5..28c2cf3b3 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do shortcode e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") + Logger.warning("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") nil end else @@ -53,7 +53,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do end else e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") + Logger.warning("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") nil end end diff --git a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex index 0c7aa769b..417f04312 100644 --- a/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_page_validator.ex @@ -84,6 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNotePageValidator do |> fix_tag() |> fix_replies() |> fix_attachments() + |> CommonFixes.fix_quote_url() |> Transmogrifier.fix_emoji() |> Transmogrifier.fix_content_map() |> CommonFixes.maybe_add_language(meta) diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex index 79ff76104..65ac6bb93 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/audio_image_video_validator.ex @@ -99,6 +99,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioImageVideoValidator do data |> CommonFixes.fix_actor() |> CommonFixes.fix_object_defaults() + |> CommonFixes.fix_quote_url() |> Transmogrifier.fix_emoji() |> fix_url() |> fix_content() diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 4a385633a..22cf0cc05 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do end end - # All objects except Answer and CHatMessage + # All objects except Answer and ChatMessage defmacro object_fields do quote bind_quoted: binding() do field(:content, :string) @@ -58,8 +58,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) + field(:quotes_count, :integer, default: 0) field(:language, ObjectValidators.LanguageCode) field(:inReplyTo, ObjectValidators.ObjectID) + field(:quoteUrl, ObjectValidators.ObjectID) field(:url, ObjectValidators.BareUri) field(:likes, {:array, ObjectValidators.ObjectID}, default: []) diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index fa581eba4..218342136 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -15,6 +15,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] + require Pleroma.Constants + def cast_and_filter_recipients(message, field, follower_collection, field_fallback \\ []) do {:ok, data} = ObjectValidators.Recipients.cast(message[field] || field_fallback) @@ -82,6 +84,50 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do Map.put(data, "to", to) end + def fix_quote_url(%{"quoteUrl" => _quote_url} = data), do: data + + # Fedibird + # https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac + def fix_quote_url(%{"quoteUri" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + # Old Fedibird (bug) + # https://github.com/fedibird/mastodon/issues/9 + def fix_quote_url(%{"quoteURL" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + # Misskey fallback + def fix_quote_url(%{"_misskey_quote" => quote_url} = data) do + Map.put(data, "quoteUrl", quote_url) + end + + def fix_quote_url(%{"tag" => [_ | _] = tags} = data) do + tag = Enum.find(tags, &is_object_link_tag/1) + + if not is_nil(tag) do + data + |> Map.put("quoteUrl", tag["href"]) + else + data + end + end + + def fix_quote_url(data), do: data + + # https://codeberg.org/fediverse/fep/src/branch/main/fep/e232/fep-e232.md + def is_object_link_tag(%{ + "type" => "Link", + "mediaType" => media_type, + "href" => href + }) + when media_type in Pleroma.Constants.activity_json_mime_types() and is_binary(href) do + true + end + + def is_object_link_tag(_), do: false + def maybe_add_language(object, meta \\ []) do language = [ diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index ce3305142..621085e6c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -62,6 +62,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do data |> CommonFixes.fix_actor() |> CommonFixes.fix_object_defaults() + |> CommonFixes.fix_quote_url() |> Transmogrifier.fix_emoji() |> fix_closed() end diff --git a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex index cfd510c19..47cf7b415 100644 --- a/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/tag_validator.ex @@ -9,15 +9,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do import Ecto.Changeset + require Pleroma.Constants + @primary_key false embedded_schema do # Common field(:type, :string) field(:name, :string) - # Mention, Hashtag + # Mention, Hashtag, Link field(:href, ObjectValidators.Uri) + # Link + field(:mediaType, :string) + # Emoji embeds_one :icon, IconObjectValidator, primary_key: false do field(:type, :string) @@ -68,6 +73,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.TagValidator do |> validate_required([:type, :name, :icon]) end + def changeset(struct, %{"type" => "Link"} = data) do + struct + |> cast(data, [:type, :name, :mediaType, :href]) + |> validate_inclusion(:mediaType, Pleroma.Constants.activity_json_mime_types()) + |> validate_required([:type, :href, :mediaType]) + end + def changeset(struct, %{"type" => _} = data) do struct |> cast(data, []) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index af6aa0781..a580994b1 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -118,7 +118,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do end end - @spec recipients(User.t(), Activity.t()) :: list(User.t()) | [] + @spec recipients(User.t(), Activity.t()) :: [[User.t()]] defp recipients(actor, activity) do followers = if actor.follower_address in activity.recipients do @@ -138,7 +138,10 @@ defmodule Pleroma.Web.ActivityPub.Publisher do [] end - Pleroma.Web.Federator.Publisher.remote_users(actor, activity) ++ followers ++ fetchers + mentioned = Pleroma.Web.Federator.Publisher.remote_users(actor, activity) + non_mentioned = (followers ++ fetchers) -- mentioned + + [mentioned, non_mentioned] end defp get_cc_ap_ids(ap_id, recipients) do @@ -195,34 +198,39 @@ defmodule Pleroma.Web.ActivityPub.Publisher do public = is_public?(activity) {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) - recipients = recipients(actor, activity) + [priority_recipients, recipients] = recipients(actor, activity) inboxes = - recipients - |> Enum.map(fn actor -> actor.inbox end) - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - |> Instances.filter_reachable() + [priority_recipients, recipients] + |> Enum.map(fn recipients -> + recipients + |> Enum.map(fn actor -> actor.inbox end) + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + |> Instances.filter_reachable() + end) Repo.checkout(fn -> - Enum.each(inboxes, fn {inbox, unreachable_since} -> - %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) - - # Get all the recipients on the same host and add them to cc. Otherwise, a remote - # instance would only accept a first message for the first recipient and ignore the rest. - cc = get_cc_ap_ids(ap_id, recipients) - - json = - data - |> Map.put("cc", cc) - |> Jason.encode!() - - Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ - inbox: inbox, - json: json, - actor_id: actor.id, - id: activity.data["id"], - unreachable_since: unreachable_since - }) + Enum.each(inboxes, fn inboxes -> + Enum.each(inboxes, fn {inbox, unreachable_since} -> + %User{ap_id: ap_id} = Enum.find(recipients, fn actor -> actor.inbox == inbox end) + + # Get all the recipients on the same host and add them to cc. Otherwise, a remote + # instance would only accept a first message for the first recipient and ignore the rest. + cc = get_cc_ap_ids(ap_id, recipients) + + json = + data + |> Map.put("cc", cc) + |> Jason.encode!() + + Pleroma.Web.Federator.Publisher.enqueue_one(__MODULE__, %{ + inbox: inbox, + json: json, + actor_id: actor.id, + id: activity.data["id"], + unreachable_since: unreachable_since + }) + end) end) end) end @@ -239,25 +247,36 @@ defmodule Pleroma.Web.ActivityPub.Publisher do {:ok, data} = Transmogrifier.prepare_outgoing(activity.data) json = Jason.encode!(data) - recipients(actor, activity) - |> Enum.map(fn %User{} = user -> - determine_inbox(activity, user) - end) - |> Enum.uniq() - |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) - |> Instances.filter_reachable() - |> Enum.each(fn {inbox, unreachable_since} -> - Pleroma.Web.Federator.Publisher.enqueue_one( - __MODULE__, - %{ - inbox: inbox, - json: json, - actor_id: actor.id, - id: activity.data["id"], - unreachable_since: unreachable_since - } - ) + [priority_inboxes, inboxes] = + recipients(actor, activity) + |> Enum.map(fn recipients -> + recipients + |> Enum.map(fn actor -> actor.inbox end) + |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) + end) + + inboxes = inboxes -- priority_inboxes + + [{priority_inboxes, 0}, {inboxes, 1}] + |> Enum.each(fn {inboxes, priority} -> + inboxes + |> Instances.filter_reachable() + |> Enum.each(fn {inbox, unreachable_since} -> + Pleroma.Web.Federator.Publisher.enqueue_one( + __MODULE__, + %{ + inbox: inbox, + json: json, + actor_id: actor.id, + id: activity.data["id"], + unreachable_since: unreachable_since + }, + priority: priority + ) + end) end) + + :ok end def gather_webfinger_links(%User{} = user) do diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 098c177c7..10f268f05 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -197,6 +197,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Increase replies count # - Set up ActivityExpiration # - Set up notifications + # - Index incoming posts for search (if needed) @impl true def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), @@ -209,6 +210,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.increase_replies_count(in_reply_to) end + if quote_url = object.data["quoteUrl"] do + Object.increase_quotes_count(quote_url) + end + reply_depth = (meta[:depth] || 0) + 1 # FIXME: Force inReplyTo to replies @@ -226,6 +231,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + meta = meta |> add_notifications(notifications) @@ -285,6 +292,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Reduce the user note count # - Reduce the reply count # - Stream out the activity + # - Removes posts from search index (if needed) @impl true def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do deleted_object = @@ -305,6 +313,10 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Object.decrease_replies_count(in_reply_to) end + if quote_url = deleted_object.data["quoteUrl"] do + Object.decrease_quotes_count(quote_url) + end + MessageReference.delete_for_object(deleted_object) ap_streamer().stream_out(object) @@ -323,6 +335,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end if result == :ok do + # Only remove from index when deleting actual objects, not users or anything else + with %Pleroma.Object{} <- deleted_object do + Pleroma.Search.remove_from_index(deleted_object) + end + {:ok, object, meta} else {:error, result} diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index a60e98c28..705faea16 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -157,7 +157,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> Map.drop(["conversation", "inReplyToAtomUri"]) else e -> - Logger.warn("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") + Logger.warning("Couldn't fetch #{inspect(in_reply_to_id)}, error: #{inspect(e)}") object end else @@ -167,6 +167,27 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_in_reply_to(object, _options), do: object + def fix_quote_url_and_maybe_fetch(object, options \\ []) do + quote_url = + case Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes.fix_quote_url(object) do + %{"quoteUrl" => quote_url} -> quote_url + _ -> nil + end + + with {:quoting?, true} <- {:quoting?, not is_nil(quote_url)}, + {:ok, quoted_object} <- get_obj_helper(quote_url, options), + %Activity{} <- Activity.get_create_by_object_ap_id(quoted_object.data["id"]) do + Map.put(object, "quoteUrl", quoted_object.data["id"]) + else + {:quoting?, _} -> + object + + e -> + Logger.warning("Couldn't fetch #{inspect(quote_url)}, error: #{inspect(e)}") + object + end + end + defp prepare_in_reply_to(in_reply_to) do cond do is_bitstring(in_reply_to) -> @@ -457,6 +478,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> strip_internal_fields() |> fix_type(fetch_options) |> fix_in_reply_to(fetch_options) + |> fix_quote_url_and_maybe_fetch(fetch_options) data = Map.put(data, "object", object) options = Keyword.put(options, :local, false) @@ -632,6 +654,16 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def set_reply_to_uri(obj), do: obj @doc """ + Fedibird compatibility + https://github.com/fedibird/mastodon/commit/dbd7ae6cf58a92ec67c512296b4daaea0d01e6ac + """ + def set_quote_url(%{"quoteUrl" => quote_url} = object) when is_binary(quote_url) do + Map.put(object, "quoteUri", quote_url) + end + + def set_quote_url(obj), do: obj + + @doc """ Serialized Mastodon-compatible `replies` collection containing _self-replies_. Based on Mastodon's ActivityPub::NoteSerializer#replies. """ @@ -685,6 +717,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> prepare_attachments |> set_conversation |> set_reply_to_uri + |> set_quote_url |> set_replies |> strip_internal_fields |> strip_internal_tags diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 2866cf2ce..6c559d80c 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do alias Ecto.UUID alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -859,9 +860,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do [actor | reported_activities] = activity.data["object"] stripped_activities = - Enum.map(reported_activities, fn - act when is_map(act) -> act["id"] - act when is_binary(act) -> act + Enum.reduce(reported_activities, [], fn act, acc -> + case ObjectID.cast(act) do + {:ok, act} -> [act | acc] + _ -> acc + end end) new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) diff --git a/lib/pleroma/web/activity_pub/views/user_view.ex b/lib/pleroma/web/activity_pub/views/user_view.ex index f69fca075..24ee683ae 100644 --- a/lib/pleroma/web/activity_pub/views/user_view.ex +++ b/lib/pleroma/web/activity_pub/views/user_view.ex @@ -46,6 +46,7 @@ defmodule Pleroma.Web.ActivityPub.UserView do "following" => "#{user.ap_id}/following", "followers" => "#{user.ap_id}/followers", "inbox" => "#{user.ap_id}/inbox", + "outbox" => "#{user.ap_id}/outbox", "name" => "Pleroma", "summary" => "An internal service actor for this Pleroma instance. No user-serviceable parts inside.", diff --git a/lib/pleroma/web/api_spec.ex b/lib/pleroma/web/api_spec.ex index 2d56dc643..163226ce5 100644 --- a/lib/pleroma/web/api_spec.ex +++ b/lib/pleroma/web/api_spec.ex @@ -10,6 +10,14 @@ defmodule Pleroma.Web.ApiSpec do @behaviour OpenApi + defp streaming_paths do + %{ + "/api/v1/streaming" => %OpenApiSpex.PathItem{ + get: Pleroma.Web.ApiSpec.StreamingOperation.streaming_operation() + } + } + end + @impl OpenApi def spec(opts \\ []) do %OpenApi{ @@ -45,7 +53,7 @@ defmodule Pleroma.Web.ApiSpec do } }, # populate the paths from a phoenix router - paths: OpenApiSpex.Paths.from_router(Router), + paths: Map.merge(streaming_paths(), OpenApiSpex.Paths.from_router(Router)), components: %OpenApiSpex.Components{ parameters: %{ "accountIdOrNickname" => diff --git a/lib/pleroma/web/api_spec/operations/instance_operation.ex b/lib/pleroma/web/api_spec/operations/instance_operation.ex index a07be7e40..8e395bde8 100644 --- a/lib/pleroma/web/api_spec/operations/instance_operation.ex +++ b/lib/pleroma/web/api_spec/operations/instance_operation.ex @@ -23,6 +23,18 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + def show2_operation do + %Operation{ + tags: ["Instance misc"], + summary: "Retrieve instance information", + description: "Information about the server", + operationId: "InstanceController.show2", + responses: %{ + 200 => Operation.response("Instance", "application/json", instance2()) + } + } + end + def peers_operation do %Operation{ tags: ["Instance misc"], @@ -165,6 +177,166 @@ defmodule Pleroma.Web.ApiSpec.InstanceOperation do } end + defp instance2 do + %Schema{ + type: :object, + properties: %{ + domain: %Schema{type: :string, description: "The domain name of the instance"}, + title: %Schema{type: :string, description: "The title of the website"}, + version: %Schema{ + type: :string, + description: "The version of Pleroma installed on the instance" + }, + source_url: %Schema{ + type: :string, + description: "The version of Pleroma installed on the instance" + }, + description: %Schema{ + type: :string, + description: "Admin-defined description of the Pleroma site" + }, + usage: %Schema{ + type: :object, + description: "Instance usage statistics", + properties: %{ + users: %Schema{ + type: :object, + description: "User count statistics", + properties: %{ + active_month: %Schema{ + type: :integer, + description: "Monthly active users" + } + } + } + } + }, + email: %Schema{ + type: :string, + description: "An email that may be contacted for any inquiries", + format: :email + }, + urls: %Schema{ + type: :object, + description: "URLs of interest for clients apps", + properties: %{} + }, + stats: %Schema{ + type: :object, + description: "Statistics about how much information the instance contains", + properties: %{ + user_count: %Schema{ + type: :integer, + description: "Users registered on this instance" + }, + status_count: %Schema{ + type: :integer, + description: "Statuses authored by users on instance" + }, + domain_count: %Schema{ + type: :integer, + description: "Domains federated with this instance" + } + } + }, + thumbnail: %Schema{ + type: :object, + properties: %{ + url: %Schema{ + type: :string, + description: "Banner image for the website", + nullable: true + } + } + }, + languages: %Schema{ + type: :array, + items: %Schema{type: :string}, + description: "Primary langauges of the website and its staff" + }, + registrations: %Schema{ + type: :object, + description: "Registrations-related configuration", + properties: %{ + enabled: %Schema{ + type: :boolean, + description: "Whether registrations are enabled" + }, + approval_required: %Schema{ + type: :boolean, + description: "Whether users need to be manually approved by admin" + } + } + }, + configuration: %Schema{ + type: :object, + description: "Instance configuration", + properties: %{ + urls: %Schema{ + type: :object, + properties: %{ + streaming: %Schema{ + type: :string, + description: "Websockets address for push streaming" + } + } + }, + statuses: %Schema{ + type: :object, + description: "A map with poll limits for local statuses", + properties: %{ + max_characters: %Schema{ + type: :integer, + description: "Posts character limit (CW/Subject included in the counter)" + }, + max_media_attachments: %Schema{ + type: :integer, + description: "Media attachment limit" + } + } + }, + media_attachments: %Schema{ + type: :object, + description: "A map with poll limits for media attachments", + properties: %{ + image_size_limit: %Schema{ + type: :integer, + description: "File size limit of uploaded images" + }, + video_size_limit: %Schema{ + type: :integer, + description: "File size limit of uploaded videos" + } + } + }, + polls: %Schema{ + type: :object, + description: "A map with poll limits for local polls", + properties: %{ + max_options: %Schema{ + type: :integer, + description: "Maximum number of options." + }, + max_characters_per_option: %Schema{ + type: :integer, + description: "Maximum number of characters per option." + }, + min_expiration: %Schema{ + type: :integer, + description: "Minimum expiration time (in seconds)." + }, + max_expiration: %Schema{ + type: :integer, + description: "Maximum expiration time (in seconds)." + } + } + } + } + } + } + } + end + defp array_of_domains do %Schema{ type: :array, diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index ca40da930..141b60533 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -59,6 +59,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, length: %Schema{type: :integer, description: "The length of the media playing"}, + externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, visibility: %Schema{ allOf: [VisibilityScope], default: "public", @@ -69,7 +70,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "title" => "Some Title", "artist" => "Some Artist", "album" => "Some Album", - "length" => 180_000 + "length" => 180_000, + "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title" } } end @@ -83,6 +85,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do title: %Schema{type: :string, description: "The title of the media playing"}, album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, + externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, length: %Schema{ type: :integer, description: "The length of the media playing", @@ -97,6 +100,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, + "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at" => "2019-09-28T12:40:45.000Z" } } diff --git a/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex new file mode 100644 index 000000000..6e69c5269 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/pleroma_status_operation.ex @@ -0,0 +1,45 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.PleromaStatusOperation do + alias OpenApiSpex.Operation + alias Pleroma.Web.ApiSpec.Schemas.ApiError + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.StatusOperation + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def quotes_operation do + %Operation{ + tags: ["Retrieve status information"], + summary: "Quoted by", + description: "View quotes for a given status", + operationId: "PleromaAPI.StatusController.quotes", + parameters: [id_param() | pagination_params()], + security: [%{"oAuth" => ["read:statuses"]}], + responses: %{ + 200 => + Operation.response( + "Array of Status", + "application/json", + StatusOperation.array_of_statuses() + ), + 403 => Operation.response("Forbidden", "application/json", ApiError), + 404 => Operation.response("Not Found", "application/json", ApiError) + } + } + end + + def id_param do + Operation.parameter(:id, :path, FlakeID, "Status ID", + example: "9umDrYheeY451cQnEe", + required: true + ) + end +end diff --git a/lib/pleroma/web/api_spec/operations/status_operation.ex b/lib/pleroma/web/api_spec/operations/status_operation.ex index 5d6e82f3c..c133a3aac 100644 --- a/lib/pleroma/web/api_spec/operations/status_operation.ex +++ b/lib/pleroma/web/api_spec/operations/status_operation.ex @@ -581,6 +581,11 @@ defmodule Pleroma.Web.ApiSpec.StatusOperation do type: :string, description: "Will reply to a given conversation, addressing only the people who are part of the recipient set of that conversation. Sets the visibility to `direct`." + }, + quote_id: %Schema{ + nullable: true, + allOf: [FlakeID], + description: "ID of the status being quoted, if any" } }, example: %{ diff --git a/lib/pleroma/web/api_spec/operations/streaming_operation.ex b/lib/pleroma/web/api_spec/operations/streaming_operation.ex new file mode 100644 index 000000000..b580bc2f0 --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/streaming_operation.ex @@ -0,0 +1,464 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.StreamingOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Response + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.NotificationOperation + alias Pleroma.Web.ApiSpec.Schemas.Chat + alias Pleroma.Web.ApiSpec.Schemas.Conversation + alias Pleroma.Web.ApiSpec.Schemas.FlakeID + alias Pleroma.Web.ApiSpec.Schemas.Status + + require Pleroma.Constants + + @spec open_api_operation(atom) :: Operation.t() + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + @spec streaming_operation() :: Operation.t() + def streaming_operation do + %Operation{ + tags: ["Timelines"], + summary: "Establish streaming connection", + description: """ + Receive statuses in real-time via WebSocket. + + You can specify the access token on the query string or through the `sec-websocket-protocol` header. Using + the query string to authenticate is considered unsafe and should not be used unless you have to (e.g. to maintain + your client's compatibility with Mastodon). + + You may specify a stream on the query string. If you do so and you are connecting to a stream that requires logged-in users, + you must specify the access token at the time of the connection (i.e. via query string or header). + + Otherwise, you have the option to authenticate after you have established the connection through client-sent events. + + The "Request body" section below describes what events clients can send through WebSocket, and the "Responses" section + describes what events server will send through WebSocket. + """, + security: [%{"oAuth" => ["read:statuses", "read:notifications"]}], + operationId: "WebsocketHandler.streaming", + parameters: + [ + Operation.parameter(:connection, :header, %Schema{type: :string}, "connection header", + required: true + ), + Operation.parameter(:upgrade, :header, %Schema{type: :string}, "upgrade header", + required: true + ), + Operation.parameter( + :"sec-websocket-key", + :header, + %Schema{type: :string}, + "sec-websocket-key header", + required: true + ), + Operation.parameter( + :"sec-websocket-version", + :header, + %Schema{type: :string}, + "sec-websocket-version header", + required: true + ) + ] ++ stream_params() ++ access_token_params(), + requestBody: request_body("Client-sent events", client_sent_events()), + responses: %{ + 101 => switching_protocols_response(), + 200 => + Operation.response( + "Server-sent events", + "application/json", + server_sent_events() + ) + } + } + end + + defp stream_params do + stream_specifier() + |> Enum.map(fn {name, schema} -> + Operation.parameter(name, :query, schema, get_schema(schema).description) + end) + end + + defp access_token_params do + [ + Operation.parameter(:access_token, :query, token(), token().description), + Operation.parameter(:"sec-websocket-protocol", :header, token(), token().description) + ] + end + + defp switching_protocols_response do + %Response{ + description: "Switching protocols", + headers: %{ + "connection" => %OpenApiSpex.Header{required: true}, + "upgrade" => %OpenApiSpex.Header{required: true}, + "sec-websocket-accept" => %OpenApiSpex.Header{required: true} + } + } + end + + defp server_sent_events do + %Schema{ + oneOf: [ + update_event(), + status_update_event(), + notification_event(), + chat_update_event(), + follow_relationships_update_event(), + conversation_event(), + delete_event(), + pleroma_respond_event() + ] + } + end + + defp stream do + %Schema{ + type: :array, + title: "Stream", + description: """ + The stream identifier. + The first item is the name of the stream. If the stream needs a differentiator, the second item will be the corresponding identifier. + Currently, for the following stream types, there is a second element in the array: + + - `list`: The second element is the id of the list, as a string. + - `hashtag`: The second element is the name of the hashtag. + - `public:remote:media` and `public:remote`: The second element is the domain of the corresponding instance. + """, + maxItems: 2, + minItems: 1, + items: %Schema{type: :string}, + example: ["hashtag", "mew"] + } + end + + defp get_schema(%Schema{} = schema), do: schema + defp get_schema(schema), do: schema.schema + + defp server_sent_event_helper(name, description, type, payload, opts \\ []) do + payload_type = Keyword.get(opts, :payload_type, :json) + has_stream = Keyword.get(opts, :has_stream, true) + + stream_properties = + if has_stream do + %{stream: stream()} + else + %{} + end + + stream_example = if has_stream, do: %{"stream" => get_schema(stream()).example}, else: %{} + + stream_required = if has_stream, do: [:stream], else: [] + + payload_schema = + if payload_type == :json do + %Schema{ + title: "Event payload", + description: "JSON-encoded string of #{get_schema(payload).title}", + allOf: [payload] + } + else + payload + end + + payload_example = + if payload_type == :json do + get_schema(payload).example |> Jason.encode!() + else + get_schema(payload).example + end + + %Schema{ + type: :object, + title: name, + description: description, + required: [:event, :payload] ++ stream_required, + properties: + %{ + event: %Schema{ + title: "Event type", + description: "Type of the event.", + type: :string, + required: true, + enum: [type] + }, + payload: payload_schema + } + |> Map.merge(stream_properties), + example: + %{ + "event" => type, + "payload" => payload_example + } + |> Map.merge(stream_example) + } + end + + defp update_event do + server_sent_event_helper("New status", "A newly-posted status.", "update", Status) + end + + defp status_update_event do + server_sent_event_helper("Edit", "A status that was just edited", "status.update", Status) + end + + defp notification_event do + server_sent_event_helper( + "Notification", + "A new notification.", + "notification", + NotificationOperation.notification() + ) + end + + defp follow_relationships_update_event do + server_sent_event_helper( + "Follow relationships update", + "An update to follow relationships.", + "pleroma:follow_relationships_update", + %Schema{ + type: :object, + title: "Follow relationships update", + required: [:state, :follower, :following], + properties: %{ + state: %Schema{ + type: :string, + description: "Follow state of the relationship.", + enum: ["follow_pending", "follow_accept", "follow_reject", "unfollow"] + }, + follower: %Schema{ + type: :object, + description: "Information about the follower.", + required: [:id, :follower_count, :following_count], + properties: %{ + id: FlakeID, + follower_count: %Schema{type: :integer}, + following_count: %Schema{type: :integer} + } + }, + following: %Schema{ + type: :object, + description: "Information about the following person.", + required: [:id, :follower_count, :following_count], + properties: %{ + id: FlakeID, + follower_count: %Schema{type: :integer}, + following_count: %Schema{type: :integer} + } + } + }, + example: %{ + "state" => "follow_pending", + "follower" => %{ + "id" => "someUser1", + "follower_count" => 1, + "following_count" => 1 + }, + "following" => %{ + "id" => "someUser2", + "follower_count" => 1, + "following_count" => 1 + } + } + } + ) + end + + defp chat_update_event do + server_sent_event_helper( + "Chat update", + "A new chat message.", + "pleroma:chat_update", + Chat + ) + end + + defp conversation_event do + server_sent_event_helper( + "Conversation update", + "An update about a conversation", + "conversation", + Conversation + ) + end + + defp delete_event do + server_sent_event_helper( + "Delete", + "A status that was just deleted.", + "delete", + %Schema{ + type: :string, + title: "Status id", + description: "Id of the deleted status", + allOf: [FlakeID], + example: "some-opaque-id" + }, + payload_type: :string, + has_stream: false + ) + end + + defp pleroma_respond_event do + server_sent_event_helper( + "Server response", + "A response to a client-sent event.", + "pleroma:respond", + %Schema{ + type: :object, + title: "Results", + required: [:result, :type], + properties: %{ + result: %Schema{ + type: :string, + title: "Result of the request", + enum: ["success", "error", "ignored"] + }, + error: %Schema{ + type: :string, + title: "Error code", + description: "An error identifier. Only appears if `result` is `error`." + }, + type: %Schema{ + type: :string, + description: "Type of the request." + } + }, + example: %{"result" => "success", "type" => "pleroma:authenticate"} + }, + has_stream: false + ) + end + + defp client_sent_events do + %Schema{ + oneOf: [ + subscribe_event(), + unsubscribe_event(), + authenticate_event() + ] + } + end + + defp request_body(description, schema, opts \\ []) do + %OpenApiSpex.RequestBody{ + description: description, + content: %{ + "application/json" => %OpenApiSpex.MediaType{ + schema: schema, + example: opts[:example], + examples: opts[:examples] + } + } + } + end + + defp client_sent_event_helper(name, description, type, properties, opts) do + required = opts[:required] || [] + + %Schema{ + type: :object, + title: name, + required: [:type] ++ required, + description: description, + properties: + %{ + type: %Schema{type: :string, enum: [type], description: "Type of the event."} + } + |> Map.merge(properties), + example: opts[:example] + } + end + + defp subscribe_event do + client_sent_event_helper( + "Subscribe", + "Subscribe to a stream.", + "subscribe", + stream_specifier(), + required: [:stream], + example: %{"type" => "subscribe", "stream" => "list", "list" => "1"} + ) + end + + defp unsubscribe_event do + client_sent_event_helper( + "Unsubscribe", + "Unsubscribe from a stream.", + "unsubscribe", + stream_specifier(), + required: [:stream], + example: %{ + "type" => "unsubscribe", + "stream" => "public:remote:media", + "instance" => "example.org" + } + ) + end + + defp authenticate_event do + client_sent_event_helper( + "Authenticate", + "Authenticate via an access token.", + "pleroma:authenticate", + %{ + token: token() + }, + required: [:token] + ) + end + + defp token do + %Schema{ + type: :string, + description: "An OAuth access token with corresponding permissions.", + example: "some token" + } + end + + defp stream_specifier do + %{ + stream: %Schema{ + type: :string, + description: "The name of the stream.", + enum: + Pleroma.Constants.public_streams() ++ + [ + "public:remote", + "public:remote:media", + "user", + "user:pleroma_chat", + "user:notification", + "direct", + "list", + "hashtag" + ] + }, + list: %Schema{ + type: :string, + title: "List id", + description: "The id of the list. Required when `stream` is `list`.", + example: "some-id" + }, + tag: %Schema{ + type: :string, + title: "Hashtag name", + description: "The name of the hashtag. Required when `stream` is `hashtag`.", + example: "mew" + }, + instance: %Schema{ + type: :string, + title: "Domain name", + description: + "Domain name of the instance. Required when `stream` is `public:remote` or `public:remote:media`.", + example: "example.org" + } + } + end +end diff --git a/lib/pleroma/web/api_spec/schemas/status.ex b/lib/pleroma/web/api_spec/schemas/status.ex index bc29cf4a6..a4052803b 100644 --- a/lib/pleroma/web/api_spec/schemas/status.ex +++ b/lib/pleroma/web/api_spec/schemas/status.ex @@ -193,6 +193,30 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do nullable: true, description: "The `acct` property of User entity for replied user (if any)" }, + quote: %Schema{ + allOf: [%OpenApiSpex.Reference{"$ref": "#/components/schemas/Status"}], + nullable: true, + description: "Quoted status (if any)" + }, + quote_id: %Schema{ + nullable: true, + allOf: [FlakeID], + description: "ID of the status being quoted, if any" + }, + quote_url: %Schema{ + type: :string, + format: :uri, + nullable: true, + description: "URL of the quoted status" + }, + quote_visible: %Schema{ + type: :boolean, + description: "`true` if the quoted post is visible to the user" + }, + quotes_count: %Schema{ + type: :integer, + description: "How many statuses quoted this status" + }, local: %Schema{ type: :boolean, description: "`true` if the post was made on the local instance" @@ -347,7 +371,8 @@ defmodule Pleroma.Web.ApiSpec.Schemas.Status do "in_reply_to_account_acct" => nil, "local" => true, "spoiler_text" => %{"text/plain" => ""}, - "thread_muted" => false + "thread_muted" => false, + "quotes_count" => 0 }, "poll" => nil, "reblog" => nil, diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 77b3fa5d2..dfc0a625d 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -33,6 +33,7 @@ defmodule Pleroma.Web.CommonAPI do def post_chat_message(%User{} = user, %User{} = recipient, content, opts \\ []) do with maybe_attachment <- opts[:media_id] && Object.get_by_id(opts[:media_id]), + :ok <- validate_chat_attachment_attribution(maybe_attachment, user), :ok <- validate_chat_content_length(content, !!maybe_attachment), {_, {:ok, chat_message_data, _meta}} <- {:build_object, @@ -71,6 +72,17 @@ defmodule Pleroma.Web.CommonAPI do text end + defp validate_chat_attachment_attribution(nil, _), do: :ok + + defp validate_chat_attachment_attribution(attachment, user) do + with :ok <- Object.authorize_access(attachment, user) do + :ok + else + e -> + e + end + end + defp validate_chat_content_length(_, true), do: :ok defp validate_chat_content_length(nil, false), do: {:error, :no_content} @@ -538,7 +550,7 @@ defmodule Pleroma.Web.CommonAPI do remove_mute(user, activity) else {what, result} = error -> - Logger.warn( + Logger.warning( "CommonAPI.remove_mute/2 failed. #{what}: #{result}, user_id: #{user_id}, activity_id: #{activity_id}" ) diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 1b6118cf8..a625f1f84 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do alias Pleroma.Conversation.Participation alias Pleroma.Object alias Pleroma.Web.ActivityPub.Builder + alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI alias Pleroma.Web.CommonAPI.Utils @@ -14,6 +15,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do only: [is_good_locale_code?: 1] import Pleroma.Web.Gettext + import Pleroma.Web.Utils.Guards, only: [not_empty_string: 1] defstruct valid?: true, errors: [], @@ -25,6 +27,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do attachments: [], in_reply_to: nil, in_reply_to_conversation: nil, + quote_post: nil, visibility: nil, expires_at: nil, extra: nil, @@ -57,7 +60,9 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do |> poll() |> with_valid(&in_reply_to/1) |> with_valid(&in_reply_to_conversation/1) + |> with_valid("e_post/1) |> with_valid(&visibility/1) + |> with_valid("ing_visibility/1) |> content() |> with_valid(&to_and_cc/1) |> with_valid(&context/1) @@ -83,7 +88,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp listen_object(draft) do object = draft.params - |> Map.take([:album, :artist, :title, :length]) + |> Map.take([:album, :artist, :title, :length, :externalLink]) |> Map.new(fn {key, value} -> {to_string(key), value} end) |> Map.put("type", "Audio") |> Map.put("to", draft.to) @@ -116,7 +121,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end defp attachments(%{params: params} = draft) do - attachments = Utils.attachments_from_ids(params) + attachments = Utils.attachments_from_ids(params, draft.user) draft = %__MODULE__{draft | attachments: attachments} case Utils.validate_attachments_count(attachments) do @@ -137,6 +142,18 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp in_reply_to(draft), do: draft + defp quote_post(%{params: %{quote_id: id}} = draft) when not_empty_string(id) do + case Activity.get_by_id_with_object(id) do + %Activity{} = activity -> + %__MODULE__{draft | quote_post: activity} + + _ -> + draft + end + end + + defp quote_post(draft), do: draft + defp in_reply_to_conversation(draft) do in_reply_to_conversation = Participation.get(draft.params[:in_reply_to_conversation_id]) %__MODULE__{draft | in_reply_to_conversation: in_reply_to_conversation} @@ -152,6 +169,29 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end + defp can_quote?(_draft, _object, visibility) when visibility in ~w(public unlisted local) do + true + end + + defp can_quote?(draft, object, "private") do + draft.user.ap_id == object.data["actor"] + end + + defp can_quote?(_, _, _) do + false + end + + defp quoting_visibility(%{quote_post: %Activity{}} = draft) do + with %Object{} = object <- Object.normalize(draft.quote_post, fetch: false), + true <- can_quote?(draft, object, Visibility.get_visibility(object)) do + draft + else + _ -> add_error(draft, dgettext("errors", "Cannot quote private message")) + end + end + + defp quoting_visibility(draft), do: draft + defp expires_at(draft) do case CommonAPI.check_expiry_date(draft.params[:expires_in]) do {:ok, expires_at} -> %__MODULE__{draft | expires_at: expires_at} @@ -169,12 +209,15 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do end end - defp content(draft) do + defp content(%{mentions: mentions} = draft) do {content_html, mentioned_users, tags} = Utils.make_content_html(draft) + mentioned_ap_ids = + Enum.map(mentioned_users, fn {_, mentioned_user} -> mentioned_user.ap_id end) + mentions = - mentioned_users - |> Enum.map(fn {_, mentioned_user} -> mentioned_user.ap_id end) + mentions + |> Kernel.++(mentioned_ap_ids) |> Utils.get_addressed_users(draft.params[:to]) %__MODULE__{draft | content_html: content_html, mentions: mentions, tags: tags} diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b9fe0224c..dcda3e0e8 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -23,21 +23,21 @@ defmodule Pleroma.Web.CommonAPI.Utils do require Logger require Pleroma.Constants - def attachments_from_ids(%{media_ids: ids, descriptions: desc}) do - attachments_from_ids_descs(ids, desc) + def attachments_from_ids(%{media_ids: ids, descriptions: desc}, user) do + attachments_from_ids_descs(ids, desc, user) end - def attachments_from_ids(%{media_ids: ids}) do - attachments_from_ids_no_descs(ids) + def attachments_from_ids(%{media_ids: ids}, user) do + attachments_from_ids_no_descs(ids, user) end - def attachments_from_ids(_), do: [] + def attachments_from_ids(_, _), do: [] - def attachments_from_ids_no_descs([]), do: [] + def attachments_from_ids_no_descs([], _), do: [] - def attachments_from_ids_no_descs(ids) do + def attachments_from_ids_no_descs(ids, user) do Enum.map(ids, fn media_id -> - case get_attachment(media_id) do + case get_attachment(media_id, user) do %Object{data: data} -> data _ -> nil end @@ -45,22 +45,23 @@ defmodule Pleroma.Web.CommonAPI.Utils do |> Enum.reject(&is_nil/1) end - def attachments_from_ids_descs([], _), do: [] + def attachments_from_ids_descs([], _, _), do: [] - def attachments_from_ids_descs(ids, descs_str) do + def attachments_from_ids_descs(ids, descs_str, user) do {_, descs} = Jason.decode(descs_str) Enum.map(ids, fn media_id -> - with %Object{data: data} <- get_attachment(media_id) do + with %Object{data: data} <- get_attachment(media_id, user) do Map.put(data, "name", descs[media_id]) end end) |> Enum.reject(&is_nil/1) end - defp get_attachment(media_id) do + defp get_attachment(media_id, user) do with %Object{data: data} = object <- Repo.get(Object, media_id), - %{"type" => type} when type in Pleroma.Constants.upload_object_types() <- data do + %{"type" => type} when type in Pleroma.Constants.upload_object_types() <- data, + :ok <- Object.authorize_access(object, user) do object else _ -> nil @@ -320,13 +321,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do format_asctime(date) else _e -> - Logger.warn("Date #{date} in wrong format, must be ISO 8601") + Logger.warning("Date #{date} in wrong format, must be ISO 8601") "" end end def date_to_asctime(date) do - Logger.warn("Date #{date} in wrong format, must be ISO 8601") + Logger.warning("Date #{date} in wrong format, must be ISO 8601") "" end diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 574f3ab63..307fa069e 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,7 +9,20 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config - socket("/socket", Pleroma.Web.UserSocket) + socket("/socket", Pleroma.Web.UserSocket, + websocket: [ + path: "/websocket", + serializer: [ + {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"}, + {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"} + ], + timeout: 60_000, + transport_log: false, + compress: false + ], + longpoll: false + ) + socket("/live", Phoenix.LiveView.Socket) plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) @@ -138,47 +151,6 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Web.Plugs.RemoteIp) - defmodule Instrumenter do - use Prometheus.PhoenixInstrumenter - end - - defmodule PipelineInstrumenter do - use Prometheus.PlugPipelineInstrumenter - end - - defmodule MetricsExporter do - use Prometheus.PlugExporter - end - - defmodule MetricsExporterCaller do - @behaviour Plug - - def init(opts), do: opts - - def call(conn, opts) do - prometheus_config = Application.get_env(:prometheus, MetricsExporter, []) - ip_whitelist = List.wrap(prometheus_config[:ip_whitelist]) - - cond do - !prometheus_config[:enabled] -> - conn - - ip_whitelist != [] and - !Enum.find(ip_whitelist, fn ip -> - Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip} - end) -> - conn - - true -> - MetricsExporter.call(conn, opts) - end - end - end - - plug(PipelineInstrumenter) - - plug(MetricsExporterCaller) - plug(Pleroma.Web.Router) @doc """ diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 1a86f7a53..4a0885fab 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -17,10 +17,28 @@ defmodule Pleroma.Web.Fallback.RedirectController do |> json(%{error: "Not implemented"}) end + def add_generated_metadata(page_content, extra \\ "") do + title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>" + favicon = "<link rel='icon' href='#{Pleroma.Config.get([:instance, :favicon])}'>" + manifest = "<link rel='manifest' href='/manifest.json'>" + + page_content + |> String.replace( + "<!--server-generated-meta-->", + title <> favicon <> manifest <> extra + ) + end + def redirector(conn, _params, code \\ 200) do + {:ok, index_content} = File.read(index_file_path()) + + response = + index_content + |> add_generated_metadata() + conn |> put_resp_content_type("text/html") - |> send_file(code, index_file_path()) + |> send_resp(code, response) end def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} = params) do @@ -34,14 +52,12 @@ defmodule Pleroma.Web.Fallback.RedirectController do def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = build_tags(conn, params) preloads = preload_data(conn, params) - title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>" response = index_content - |> String.replace("<!--server-generated-meta-->", tags <> preloads <> title) + |> add_generated_metadata(tags <> preloads) conn |> put_resp_content_type("text/html") @@ -55,11 +71,10 @@ defmodule Pleroma.Web.Fallback.RedirectController do def redirector_with_preload(conn, params) do {:ok, index_content} = File.read(index_file_path()) preloads = preload_data(conn, params) - title = "<title>#{Pleroma.Config.get([:instance, :name])}</title>" response = index_content - |> String.replace("<!--server-generated-meta-->", preloads <> title) + |> add_generated_metadata(preloads) conn |> put_resp_content_type("text/html") diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 84b77cda1..8621d984c 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -35,6 +35,17 @@ defmodule Pleroma.Web.Federator do end # Client API + def incoming_ap_doc(%{params: params, req_headers: req_headers}) do + ReceiverWorker.enqueue( + "incoming_ap_doc", + %{"req_headers" => req_headers, "params" => params, "timeout" => :timer.seconds(20)}, + priority: 2 + ) + end + + def incoming_ap_doc(%{"type" => "Delete"} = params) do + ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}, priority: 3) + end def incoming_ap_doc(params) do ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params}) diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex index a45796e9d..8c6547208 100644 --- a/lib/pleroma/web/federator/publisher.ex +++ b/lib/pleroma/web/federator/publisher.ex @@ -29,11 +29,12 @@ defmodule Pleroma.Web.Federator.Publisher do @doc """ Enqueue publishing a single activity. """ - @spec enqueue_one(module(), Map.t()) :: :ok - def enqueue_one(module, %{} = params) do + @spec enqueue_one(module(), Map.t(), Keyword.t()) :: {:ok, %Oban.Job{}} + def enqueue_one(module, %{} = params, worker_args \\ []) do PublisherWorker.enqueue( "publish_one", - %{"module" => to_string(module), "params" => params} + %{"module" => to_string(module), "params" => params}, + worker_args ) end diff --git a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex index 6410e872c..3e664903a 100644 --- a/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/instance_controller.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do plug(Pleroma.Web.ApiSpec.CastAndValidate) - plug(:skip_auth when action in [:show, :peers]) + plug(:skip_auth when action in [:show, :show2, :peers]) defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.InstanceOperation @@ -16,6 +16,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceController do render(conn, "show.json") end + @doc "GET /api/v2/instance" + def show2(conn, _params) do + render(conn, "show2.json") + end + @doc "GET /api/v1/instance/peers" def peers(conn, _params) do json(conn, Pleroma.Stats.get_peers()) diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 5e6e04734..e4acba226 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do use Pleroma.Web, :controller - alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ControllerHelper @@ -100,7 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end) StatusView.render("index.json", activities: statuses, diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index cc3e3582f..237de3055 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.MastodonAPI.AccountView do @@ -249,6 +249,10 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do nil end + last_status_at = + user.last_status_at && + user.last_status_at |> NaiveDateTime.to_date() |> Date.to_iso8601() + %{ id: to_string(user.id), username: username_from_nickname(user.nickname), @@ -277,7 +281,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do actor_type: user.actor_type } }, - last_status_at: user.last_status_at, + last_status_at: last_status_at, # Pleroma extensions # Note: it's insecure to output :email but fully-qualified nickname may serve as safe stub diff --git a/lib/pleroma/web/mastodon_api/views/instance_view.ex b/lib/pleroma/web/mastodon_api/views/instance_view.ex index efd2a0af6..f95b5360a 100644 --- a/lib/pleroma/web/mastodon_api/views/instance_view.ex +++ b/lib/pleroma/web/mastodon_api/views/instance_view.ex @@ -13,12 +13,11 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do def render("show.json", _) do instance = Config.get(:instance) - %{ - uri: Pleroma.Web.Endpoint.url(), - title: Keyword.get(instance, :name), + common_information(instance) + |> Map.merge(%{ + uri: Pleroma.Web.WebFinger.host(), description: Keyword.get(instance, :description), short_description: Keyword.get(instance, :short_description), - version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", email: Keyword.get(instance, :email), urls: %{ streaming_api: Pleroma.Web.Endpoint.websocket_url() @@ -27,9 +26,9 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do thumbnail: URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) |> to_string, - languages: Keyword.get(instance, :languages, ["en"]), registrations: Keyword.get(instance, :registrations_open), approval_required: Keyword.get(instance, :account_approval_required), + configuration: configuration(), # Extra (not present in Mastodon): max_toot_chars: Keyword.get(instance, :limit), max_media_attachments: Keyword.get(instance, :max_media_attachments), @@ -41,19 +40,44 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do background_image: Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), shout_limit: Config.get([:shout, :limit]), description_limit: Keyword.get(instance, :description_limit), - pleroma: %{ - metadata: %{ - account_activation_required: Keyword.get(instance, :account_activation_required), - features: features(), - federation: federation(), - fields_limits: fields_limits(), - post_formats: Config.get([:instance, :allowed_post_formats]), - birthday_required: Config.get([:instance, :birthday_required]), - birthday_min_age: Config.get([:instance, :birthday_min_age]) - }, - stats: %{mau: Pleroma.User.active_user_count()}, - vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) - } + pleroma: pleroma_configuration(instance) + }) + end + + def render("show2.json", _) do + instance = Config.get(:instance) + + common_information(instance) + |> Map.merge(%{ + domain: Pleroma.Web.WebFinger.host(), + source_url: Pleroma.Application.repository(), + description: Keyword.get(instance, :short_description), + usage: %{users: %{active_month: Pleroma.User.active_user_count()}}, + thumbnail: %{ + url: + URI.merge(Pleroma.Web.Endpoint.url(), Keyword.get(instance, :instance_thumbnail)) + |> to_string + }, + configuration: configuration2(), + registrations: %{ + enabled: Keyword.get(instance, :registrations_open), + approval_required: Keyword.get(instance, :account_approval_required), + message: nil + }, + contact: %{ + email: Keyword.get(instance, :email), + account: nil + }, + # Extra (not present in Mastodon): + pleroma: pleroma_configuration2(instance) + }) + end + + defp common_information(instance) do + %{ + title: Keyword.get(instance, :name), + version: "#{@mastodon_api_level} (compatible; #{Pleroma.Application.named_version()})", + languages: Keyword.get(instance, :languages, ["en"]) } end @@ -69,6 +93,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do "multifetch", "pleroma:api/v1/notifications:include_types_filter", "editing", + "quote_posting", if Config.get([:activitypub, :blockers_visible]) do "blockers_visible" end, @@ -132,7 +157,7 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do |> Map.put(:enabled, Config.get([:instance, :federating])) end - def fields_limits do + defp fields_limits do %{ max_fields: Config.get([:instance, :max_account_fields]), max_remote_fields: Config.get([:instance, :max_remote_account_fields]), @@ -140,4 +165,65 @@ defmodule Pleroma.Web.MastodonAPI.InstanceView do value_length: Config.get([:instance, :account_field_value_length]) } end + + defp configuration do + %{ + statuses: %{ + max_characters: Config.get([:instance, :limit]), + max_media_attachments: Config.get([:instance, :max_media_attachments]) + }, + media_attachments: %{ + image_size_limit: Config.get([:instance, :upload_limit]), + video_size_limit: Config.get([:instance, :upload_limit]) + }, + polls: %{ + max_options: Config.get([:instance, :poll_limits, :max_options]), + max_characters_per_option: Config.get([:instance, :poll_limits, :max_option_chars]), + min_expiration: Config.get([:instance, :poll_limits, :min_expiration]), + max_expiration: Config.get([:instance, :poll_limits, :max_expiration]) + } + } + end + + defp configuration2 do + configuration() + |> Map.merge(%{ + urls: %{streaming: Pleroma.Web.Endpoint.websocket_url()} + }) + end + + defp pleroma_configuration(instance) do + %{ + metadata: %{ + account_activation_required: Keyword.get(instance, :account_activation_required), + features: features(), + federation: federation(), + fields_limits: fields_limits(), + post_formats: Config.get([:instance, :allowed_post_formats]), + birthday_required: Config.get([:instance, :birthday_required]), + birthday_min_age: Config.get([:instance, :birthday_min_age]) + }, + stats: %{mau: Pleroma.User.active_user_count()}, + vapid_public_key: Keyword.get(Pleroma.Web.Push.vapid_config(), :public_key) + } + end + + defp pleroma_configuration2(instance) do + configuration = pleroma_configuration(instance) + + configuration + |> Map.merge(%{ + metadata: + configuration.metadata + |> Map.merge(%{ + avatar_upload_limit: Keyword.get(instance, :avatar_upload_limit), + background_upload_limit: Keyword.get(instance, :background_upload_limit), + banner_upload_limit: Keyword.get(instance, :banner_upload_limit), + background_image: + Pleroma.Web.Endpoint.url() <> Keyword.get(instance, :background_image), + description_limit: Keyword.get(instance, :description_limit), + shout_limit: Config.get([:shout, :limit]) + }) + }) + end end diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 2ae5e14aa..4cdbfbdab 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -57,6 +57,27 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end) end + defp get_quoted_activities([]), do: %{} + + defp get_quoted_activities(activities) do + activities + |> Enum.map(fn + %{data: %{"type" => "Create"}} = activity -> + object = Object.normalize(activity, fetch: false) + object && object.data["quoteUrl"] != "" && object.data["quoteUrl"] + + _ -> + nil + end) + |> Enum.filter(& &1) + |> Activity.create_by_object_ap_id_with_object() + |> Repo.all() + |> Enum.reduce(%{}, fn activity, acc -> + object = Object.normalize(activity, fetch: false) + if object, do: Map.put(acc, object.data["id"], activity), else: acc + end) + end + # DEPRECATED This field seems to be a left-over from the StatusNet era. # If your application uses `pleroma.conversation_id`: this field is deprecated. # It is currently stubbed instead by doing a CRC32 of the context, and @@ -97,6 +118,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # length(activities_with_links) * timeout fetch_rich_media_for_activities(activities) replied_to_activities = get_replied_to_activities(activities) + quoted_activities = get_quoted_activities(activities) parent_activities = activities @@ -129,6 +151,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do opts = opts |> Map.put(:replied_to_activities, replied_to_activities) + |> Map.put(:quoted_activities, quoted_activities) |> Map.put(:parent_activities, parent_activities) |> Map.put(:relationships, relationships_opt) @@ -277,7 +300,6 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end reply_to = get_reply_to(activity, opts) - reply_to_user = reply_to && CommonAPI.get_user(reply_to.data["actor"]) history_len = @@ -290,6 +312,22 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do # Here the implicit index of the current content is 0 chrono_order = history_len - 1 + quote_activity = get_quote(activity, opts) + + quote_id = + case quote_activity do + %Activity{id: id} -> id + _ -> nil + end + + quote_post = + if visible_for_user?(quote_activity, opts[:for]) and opts[:show_quote] != false do + quote_rendering_opts = Map.merge(opts, %{activity: quote_activity, show_quote: false}) + render("show.json", quote_rendering_opts) + else + nil + end + content = object |> render_content() @@ -398,6 +436,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do conversation_id: get_context_id(activity), context: object.data["context"], in_reply_to_account_acct: reply_to_user && reply_to_user.nickname, + quote: quote_post, + quote_id: quote_id, + quote_url: object.data["quoteUrl"], + quote_visible: visible_for_user?(quote_activity, opts[:for]), content: %{"text/plain" => content_plaintext}, spoiler_text: %{"text/plain" => summary}, expires_at: expires_at, @@ -405,7 +447,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do thread_muted: thread_muted?, emoji_reactions: emoji_reactions, parent_visible: visible_for_user?(reply_to, opts[:for]), - pinned_at: pinned_at + pinned_at: pinned_at, + quotes_count: object.data["quotesCount"] || 0 } } end @@ -520,25 +563,24 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do page_url = page_url_data |> to_string - image_url_data = - if is_binary(rich_media["image"]) do - URI.parse(rich_media["image"]) - else - nil - end - - image_url = build_image_url(image_url_data, page_url_data) + image_url = proxied_url(rich_media["image"], page_url_data) + audio_url = proxied_url(rich_media["audio"], page_url_data) + video_url = proxied_url(rich_media["video"], page_url_data) %{ type: "link", provider_name: page_url_data.host, provider_url: page_url_data.scheme <> "://" <> page_url_data.host, url: page_url, - image: image_url |> MediaProxy.url(), + image: image_url, title: rich_media["title"] || "", description: rich_media["description"] || "", pleroma: %{ - opengraph: rich_media + opengraph: + rich_media + |> Maps.put_if_present("image", image_url) + |> Maps.put_if_present("audio", audio_url) + |> Maps.put_if_present("video", video_url) } } end @@ -633,6 +675,25 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end + def get_quote(activity, %{quoted_activities: quoted_activities}) do + object = Object.normalize(activity, fetch: false) + + with nil <- quoted_activities[object.data["quoteUrl"]] do + # For when a quote post is inside an Announce + Activity.get_create_by_object_ap_id_with_object(object.data["quoteUrl"]) + end + end + + def get_quote(%{data: %{"object" => _object}} = activity, _) do + object = Object.normalize(activity, fetch: false) + + if object.data["quoteUrl"] && object.data["quoteUrl"] != "" do + Activity.get_create_by_object_ap_id(object.data["quoteUrl"]) + else + nil + end + end + def render_content(%{data: %{"name" => name}} = object) when not is_nil(name) and name != "" do url = object.data["url"] || object.data["id"] @@ -760,4 +821,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do defp get_language(%{data: %{"language" => "und"}}), do: nil defp get_language(object), do: object.data["language"] + + defp proxied_url(url, page_url_data) do + if is_binary(url) do + build_image_url(URI.parse(url), page_url_data) |> MediaProxy.url() + else + nil + end + end end diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex index 88444106d..07c2b62e3 100644 --- a/lib/pleroma/web/mastodon_api/websocket_handler.ex +++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex @@ -9,6 +9,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do alias Pleroma.User alias Pleroma.Web.OAuth.Token alias Pleroma.Web.Streamer + alias Pleroma.Web.StreamerView @behaviour :cowboy_websocket @@ -32,8 +33,15 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do req end + topics = + if topic do + [topic] + else + [] + end + {:cowboy_websocket, req, - %{user: user, topic: topic, oauth_token: oauth_token, count: 0, timer: nil}, + %{user: user, topics: topics, oauth_token: oauth_token, count: 0, timer: nil}, %{idle_timeout: @timeout}} else {:error, :bad_topic} -> @@ -50,10 +58,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def websocket_init(state) do Logger.debug( - "#{__MODULE__} accepted websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topic #{state.topic}" + "#{__MODULE__} accepted websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics}" ) - Streamer.add_socket(state.topic, state.oauth_token) + Enum.each(state.topics, fn topic -> Streamer.add_socket(topic, state.oauth_token) end) {:ok, %{state | timer: timer()}} end @@ -66,16 +74,26 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do # We only receive pings for now def websocket_handle(:ping, state), do: {:ok, state} + def websocket_handle({:text, text}, state) do + with {:ok, %{} = event} <- Jason.decode(text) do + handle_client_event(event, state) + else + _ -> + Logger.error("#{__MODULE__} received non-JSON event: #{inspect(text)}") + {:ok, state} + end + end + def websocket_handle(frame, state) do Logger.error("#{__MODULE__} received frame: #{inspect(frame)}") {:ok, state} end - def websocket_info({:render_with_user, view, template, item}, state) do + def websocket_info({:render_with_user, view, template, item, topic}, state) do user = %User{} = User.get_cached_by_ap_id(state.user.ap_id) unless Streamer.filtered_by_user?(user, item) do - websocket_info({:text, view.render(template, item, user)}, %{state | user: user}) + websocket_info({:text, view.render(template, item, user, topic)}, %{state | user: user}) else {:ok, state} end @@ -109,10 +127,10 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do def terminate(reason, _req, state) do Logger.debug( - "#{__MODULE__} terminating websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topic #{state.topic || "?"}: #{inspect(reason)}" + "#{__MODULE__} terminating websocket connection for user #{(state.user || %{id: "anonymous"}).id}, topics #{state.topics || "?"}: #{inspect(reason)}" ) - Streamer.remove_socket(state.topic) + Enum.each(state.topics, fn topic -> Streamer.remove_socket(topic) end) :ok end @@ -137,4 +155,103 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do defp timer do Process.send_after(self(), :tick, @tick) end + + defp handle_client_event(%{"type" => "subscribe", "stream" => _topic} = params, state) do + with {_, {:ok, topic}} <- + {:topic, Streamer.get_topic(params["stream"], state.user, state.oauth_token, params)}, + {_, false} <- {:subscribed, topic in state.topics} do + Streamer.add_socket(topic, state.oauth_token) + + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "success"})} + ], %{state | topics: [topic | state.topics]}} + else + {:subscribed, true} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{type: "subscribe", result: "ignored"})} + ], state} + + {:topic, {:error, error}} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "subscribe", + result: "error", + error: error + })} + ], state} + end + end + + defp handle_client_event(%{"type" => "unsubscribe", "stream" => _topic} = params, state) do + with {_, {:ok, topic}} <- + {:topic, Streamer.get_topic(params["stream"], state.user, state.oauth_token, params)}, + {_, true} <- {:subscribed, topic in state.topics} do + Streamer.remove_socket(topic) + + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "success"})} + ], %{state | topics: List.delete(state.topics, topic)}} + else + {:subscribed, false} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{type: "unsubscribe", result: "ignored"})} + ], state} + + {:topic, {:error, error}} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "unsubscribe", + result: "error", + error: error + })} + ], state} + end + end + + defp handle_client_event( + %{"type" => "pleroma:authenticate", "token" => access_token} = _params, + state + ) do + with {:auth, nil, nil} <- {:auth, state.user, state.oauth_token}, + {:ok, user, oauth_token} <- authenticate_request(access_token, nil) do + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma:authenticate", + result: "success" + })} + ], %{state | user: user, oauth_token: oauth_token}} + else + {:auth, _, _} -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma:authenticate", + result: "error", + error: :already_authenticated + })} + ], state} + + _ -> + {[ + {:text, + StreamerView.render("pleroma_respond.json", %{ + type: "pleroma:authenticate", + result: "error", + error: :unauthorized + })} + ], state} + end + end + + defp handle_client_event(params, state) do + Logger.error("#{__MODULE__} received unknown event: #{inspect(params)}") + {[], state} + end end diff --git a/lib/pleroma/web/pleroma_api/controllers/status_controller.ex b/lib/pleroma/web/pleroma_api/controllers/status_controller.ex new file mode 100644 index 000000000..482662fdd --- /dev/null +++ b/lib/pleroma/web/pleroma_api/controllers/status_controller.ex @@ -0,0 +1,66 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.PleromaAPI.StatusController do + use Pleroma.Web, :controller + + import Pleroma.Web.ControllerHelper, only: [add_link_headers: 2] + + require Ecto.Query + require Pleroma.Constants + + alias Pleroma.Activity + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Visibility + alias Pleroma.Web.MastodonAPI.StatusView + alias Pleroma.Web.Plugs.OAuthScopesPlug + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + action_fallback(Pleroma.Web.MastodonAPI.FallbackController) + + plug( + OAuthScopesPlug, + %{scopes: ["read:statuses"], fallback: :proceed_unauthenticated} when action == :quotes + ) + + defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaStatusOperation + + @doc "GET /api/v1/pleroma/statuses/:id/quotes" + def quotes(%{assigns: %{user: user}} = conn, %{id: id} = params) do + with %Activity{object: object} = activity <- Activity.get_by_id_with_object(id), + true <- Visibility.visible_for_user?(activity, user) do + params = + params + |> Map.put(:type, "Create") + |> Map.put(:blocking_user, user) + |> Map.put(:quote_url, object.data["id"]) + + recipients = + if user do + [Pleroma.Constants.as_public()] ++ [user.ap_id | User.following(user)] + else + [Pleroma.Constants.as_public()] + end + + activities = + recipients + |> ActivityPub.fetch_activities(params) + |> Enum.reverse() + + conn + |> add_link_headers(activities) + |> put_view(StatusView) + |> render("index.json", + activities: activities, + for: user, + as: :activity + ) + else + nil -> {:error, :not_found} + false -> {:error, :not_found} + end + end +end diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index a5985fb2a..edf0a2390 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -27,6 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do title: object.data["title"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(), album: object.data["album"] |> HTML.strip_tags(), + externalLink: object.data["externalLink"], length: object.data["length"] } end diff --git a/lib/pleroma/web/plugs/http_security_plug.ex b/lib/pleroma/web/plugs/http_security_plug.ex index 34895c8d5..a27dcd0ab 100644 --- a/lib/pleroma/web/plugs/http_security_plug.ex +++ b/lib/pleroma/web/plugs/http_security_plug.ex @@ -93,19 +93,27 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do img_src = "img-src 'self' data: blob:" media_src = "media-src 'self'" + connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] # Strict multimedia CSP enforcement only when MediaProxy is enabled - {img_src, media_src} = + {img_src, media_src, connect_src} = if Config.get([:media_proxy, :enabled]) && !Config.get([:media_proxy, :proxy_opts, :redirect_on_failure]) do sources = build_csp_multimedia_source_list() - {[img_src, sources], [media_src, sources]} + + { + [img_src, sources], + [media_src, sources], + [connect_src, sources] + } else - {[img_src, " https:"], [media_src, " https:"]} + { + [img_src, " https:"], + [media_src, " https:"], + [connect_src, " https:"] + } end - connect_src = ["connect-src 'self' blob: ", static_url, ?\s, websocket_url] - connect_src = if Config.get(:env) == :dev do [connect_src, " http://localhost:3035/"] @@ -193,7 +201,7 @@ defmodule Pleroma.Web.Plugs.HTTPSecurityPlug do def warn_if_disabled do unless Config.get([:http_security, :enabled]) do - Logger.warn(" + Logger.warning(" .i;;;;i. iYcviii;vXY: .YXi .i1c. diff --git a/lib/pleroma/web/plugs/rate_limiter.ex b/lib/pleroma/web/plugs/rate_limiter.ex index 2080b06bd..aa79dbf6b 100644 --- a/lib/pleroma/web/plugs/rate_limiter.ex +++ b/lib/pleroma/web/plugs/rate_limiter.ex @@ -89,7 +89,7 @@ defmodule Pleroma.Web.Plugs.RateLimiter do end defp handle_disabled(conn) do - Logger.warn( + Logger.warning( "Rate limiter disabled due to forwarded IP not being found. Please ensure your reverse proxy is providing the X-Forwarded-For header or disable the RemoteIP plug/rate limiter." ) diff --git a/lib/pleroma/web/push.ex b/lib/pleroma/web/push.ex index 9665b0b4a..0d43f402e 100644 --- a/lib/pleroma/web/push.ex +++ b/lib/pleroma/web/push.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.Push do def init do unless enabled() do - Logger.warn(""" + Logger.warning(""" VAPID key pair is not found. If you wish to enabled web push, please run mix web_push.gen.keypair diff --git a/lib/pleroma/web/push/impl.ex b/lib/pleroma/web/push/impl.ex index 3c5f00764..36f44d8e8 100644 --- a/lib/pleroma/web/push/impl.ex +++ b/lib/pleroma/web/push/impl.ex @@ -57,7 +57,7 @@ defmodule Pleroma.Web.Push.Impl do end def perform(_) do - Logger.warn("Unknown notification type") + Logger.warning("Unknown notification type") {:error, :unknown_type} end diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 0488df30e..61000bb9b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -4,11 +4,12 @@ defmodule Pleroma.Web.RichMedia.Helpers do alias Pleroma.Activity - alias Pleroma.Config alias Pleroma.HTML alias Pleroma.Object alias Pleroma.Web.RichMedia.Parser + @config_impl Application.compile_env(:pleroma, [__MODULE__, :config_impl], Pleroma.Config) + @options [ pool: :media, max_body: 2_000_000, @@ -17,7 +18,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do @spec validate_page_url(URI.t() | binary()) :: :ok | :error defp validate_page_url(page_url) when is_binary(page_url) do - validate_tld = Config.get([Pleroma.Formatter, :validate_tld]) + validate_tld = @config_impl.get([Pleroma.Formatter, :validate_tld]) page_url |> Linkify.Parser.url?(validate_tld: validate_tld) @@ -27,10 +28,10 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp validate_page_url(%URI{host: host, scheme: "https", authority: authority}) when is_binary(authority) do cond do - host in Config.get([:rich_media, :ignore_hosts], []) -> + host in @config_impl.get([:rich_media, :ignore_hosts], []) -> :error - get_tld(host) in Config.get([:rich_media, :ignore_tld], []) -> + get_tld(host) in @config_impl.get([:rich_media, :ignore_tld], []) -> :error true -> @@ -56,7 +57,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do end def fetch_data_for_object(object) do - with true <- Config.get([:rich_media, :enabled]), + with true <- @config_impl.get([:rich_media, :enabled]), {:ok, page_url} <- HTML.extract_first_external_url_from_object(object), :ok <- validate_page_url(page_url), @@ -68,7 +69,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do end def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do - with true <- Config.get([:rich_media, :enabled]), + with true <- @config_impl.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity, fetch: false) do fetch_data_for_object(object) else diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index dbe81eabb..c37c45963 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -75,7 +75,7 @@ defmodule Pleroma.Web.RichMedia.Parser do end defp log_error(url, reason) do - Logger.warn(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) + Logger.warning(fn -> "Rich media error for #{url}: #{inspect(reason)}" end) end end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 6b9e158a3..22c85e34b 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -224,6 +224,12 @@ defmodule Pleroma.Web.Router do post("/remote_interaction", UtilController, :remote_interaction) end + scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do + pipe_through(:pleroma_api) + + get("/federation_status", InstancesController, :show) + end + scope "/api/v1/pleroma", Pleroma.Web do pipe_through(:pleroma_api) post("/uploader_callback/:upload_path", UploaderController, :callback) @@ -465,6 +471,8 @@ defmodule Pleroma.Web.Router do get("/main/ostatus", UtilController, :show_subscribe_form) get("/ostatus_subscribe", RemoteFollowController, :follow) post("/ostatus_subscribe", RemoteFollowController, :do_follow) + + get("/authorize_interaction", RemoteFollowController, :authorize_interaction) end scope "/api/pleroma", Pleroma.Web.TwitterAPI do @@ -578,6 +586,8 @@ defmodule Pleroma.Web.Router do pipe_through(:api) get("/accounts/:id/favourites", AccountController, :favourites) get("/accounts/:id/endorsements", AccountController, :endorsements) + + get("/statuses/:id/quotes", StatusController, :quotes) end scope [] do @@ -602,7 +612,6 @@ defmodule Pleroma.Web.Router do scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do pipe_through(:api) get("/accounts/:id/scrobbles", ScrobbleController, :index) - get("/federation_status", InstancesController, :show) end scope "/api/v2/pleroma", Pleroma.Web.PleromaAPI do @@ -774,11 +783,14 @@ defmodule Pleroma.Web.Router do scope "/api/v2", Pleroma.Web.MastodonAPI do pipe_through(:api) + get("/search", SearchController, :search2) post("/media", MediaController, :create2) get("/suggestions", SuggestionController, :index2) + + get("/instance", InstanceController, :show2) end scope "/api", Pleroma.Web do @@ -1003,9 +1015,8 @@ defmodule Pleroma.Web.Router do options("/*path", RedirectController, :empty) end - # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+ def get_api_routes do - __MODULE__.__routes__() + Phoenix.Router.routes(__MODULE__) |> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end) |> Enum.map(fn r -> r.path diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index b9a04cc76..48ca82421 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -4,6 +4,7 @@ defmodule Pleroma.Web.Streamer do require Logger + require Pleroma.Constants alias Pleroma.Activity alias Pleroma.Chat.MessageReference @@ -24,7 +25,7 @@ defmodule Pleroma.Web.Streamer do def registry, do: @registry - @public_streams ["public", "public:local", "public:media", "public:local:media"] + @public_streams Pleroma.Constants.public_streams() @local_streams ["public:local", "public:local:media"] @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"] @@ -59,10 +60,14 @@ defmodule Pleroma.Web.Streamer do end @doc "Expand and authorizes a stream" - @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) :: - {:ok, topic :: String.t()} | {:error, :bad_topic} + @spec get_topic(stream :: String.t() | nil, User.t() | nil, Token.t() | nil, Map.t()) :: + {:ok, topic :: String.t() | nil} | {:error, :bad_topic} def get_topic(stream, user, oauth_token, params \\ %{}) + def get_topic(nil = _stream, _user, _oauth_token, _params) do + {:ok, nil} + end + # Allow all public steams if the instance allows unauthenticated access. # Otherwise, only allow users with valid oauth tokens. def get_topic(stream, user, oauth_token, _params) when stream in @public_streams do @@ -219,8 +224,8 @@ defmodule Pleroma.Web.Streamer do end defp do_stream("follow_relationship", item) do - text = StreamerView.render("follow_relationships_update.json", item) user_topic = "user:#{item.follower.id}" + text = StreamerView.render("follow_relationships_update.json", item, user_topic) Logger.debug("Trying to push follow relationship update to #{user_topic}\n\n") @@ -266,9 +271,11 @@ defmodule Pleroma.Web.Streamer do defp do_stream(topic, %Notification{} = item) when topic in ["user", "user:notification"] do - Registry.dispatch(@registry, "#{topic}:#{item.user_id}", fn list -> + user_topic = "#{topic}:#{item.user_id}" + + Registry.dispatch(@registry, user_topic, fn list -> Enum.each(list, fn {pid, _auth} -> - send(pid, {:render_with_user, StreamerView, "notification.json", item}) + send(pid, {:render_with_user, StreamerView, "notification.json", item, user_topic}) end) end) end @@ -277,7 +284,7 @@ defmodule Pleroma.Web.Streamer do when topic in ["user", "user:pleroma_chat"] do topic = "#{topic}:#{user.id}" - text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}) + text = StreamerView.render("chat_update.json", %{chat_message_reference: cm_ref}, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, _auth} -> @@ -305,7 +312,7 @@ defmodule Pleroma.Web.Streamer do end defp push_to_socket(topic, %Participation{} = participation) do - rendered = StreamerView.render("conversation.json", participation) + rendered = StreamerView.render("conversation.json", participation, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, _} -> @@ -333,12 +340,15 @@ defmodule Pleroma.Web.Streamer do Pleroma.Activity.get_create_by_object_ap_id(item.object.data["id"]) |> Map.put(:object, item.object) - anon_render = StreamerView.render("status_update.json", create_activity) + anon_render = StreamerView.render("status_update.json", create_activity, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, auth?} -> if auth? do - send(pid, {:render_with_user, StreamerView, "status_update.json", create_activity}) + send( + pid, + {:render_with_user, StreamerView, "status_update.json", create_activity, topic} + ) else send(pid, {:text, anon_render}) end @@ -347,12 +357,12 @@ defmodule Pleroma.Web.Streamer do end defp push_to_socket(topic, item) do - anon_render = StreamerView.render("update.json", item) + anon_render = StreamerView.render("update.json", item, topic) Registry.dispatch(@registry, topic, fn list -> Enum.each(list, fn {pid, auth?} -> if auth? do - send(pid, {:render_with_user, StreamerView, "update.json", item}) + send(pid, {:render_with_user, StreamerView, "update.json", item, topic}) else send(pid, {:text, anon_render}) end diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index e45d13bdf..e3639aae7 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> -<p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p> +<%= if Phoenix.Flash.get(@flash, :info) do %> +<p class="alert alert-info" role="alert"><%= Phoenix.Flash.get(@flash, :info) %></p> <% end %> -<%= if get_flash(@conn, :error) do %> -<p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p> +<%= if Phoenix.Flash.get(@flash, :error) do %> +<p class="alert alert-danger" role="alert"><%= Phoenix.Flash.get(@flash, :error) %></p> <% end %> <h2><%= Gettext.dpgettext("static_pages", "mfa recover page title", "Two-factor recovery") %></h2> diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index 50e6c04b6..f995b8805 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> -<p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p> +<%= if Phoenix.Flash.get(@flash, :info) do %> +<p class="alert alert-info" role="alert"><%= Phoenix.Flash.get(@flash, :info) %></p> <% end %> -<%= if get_flash(@conn, :error) do %> -<p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p> +<%= if Phoenix.Flash.get(@flash, :error) do %> +<p class="alert alert-danger" role="alert"><%= Phoenix.Flash.get(@flash, :error) %></p> <% end %> <h2><%= Gettext.dpgettext("static_pages", "mfa auth page title", "Two-factor authentication") %></h2> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index 1f661efb2..e7f65266f 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - <p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p> +<%= if Phoenix.Flash.get(@flash, :info) do %> + <p class="alert alert-info" role="alert"><%= Phoenix.Flash.get(@flash, :info) %></p> <% end %> -<%= if get_flash(@conn, :error) do %> - <p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p> +<%= if Phoenix.Flash.get(@flash, :error) do %> + <p class="alert alert-danger" role="alert"><%= Phoenix.Flash.get(@flash, :error) %></p> <% end %> <h2><%= Gettext.dpgettext("static_pages", "oauth register page title", "Registration Details") %></h2> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index b3654f3eb..5b38f7142 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> -<p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p> +<%= if Phoenix.Flash.get(@flash, :info) do %> +<p class="alert alert-info" role="alert"><%= Phoenix.Flash.get(@flash, :info) %></p> <% end %> -<%= if get_flash(@conn, :error) do %> -<p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p> +<%= if Phoenix.Flash.get(@flash, :error) do %> +<p class="alert alert-danger" role="alert"><%= Phoenix.Flash.get(@flash, :error) %></p> <% end %> <%= form_for @conn, Routes.o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> diff --git a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex index 6229d5d05..178ad2b43 100644 --- a/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/remote_follow_controller.ex @@ -121,6 +121,13 @@ defmodule Pleroma.Web.TwitterAPI.RemoteFollowController do render(conn, "followed.html", %{error: "Insufficient permissions: follow | write:follows."}) end + # GET /authorize_interaction + # + def authorize_interaction(conn, %{"uri" => uri}) do + conn + |> redirect(to: Routes.remote_follow_path(conn, :follow, %{acct: uri})) + end + defp handle_follow_error(conn, {:mfa_token, followee, _} = _) do render(conn, "follow_login.html", %{error: "Wrong username or password", followee: followee}) end diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index d5a24ae6c..ca8a98960 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -345,13 +345,16 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do end def healthcheck(conn, _params) do - with true <- Config.get([:instance, :healthcheck]), + with {:cfg, true} <- {:cfg, Config.get([:instance, :healthcheck])}, %{healthy: true} = info <- Healthcheck.system_info() do json(conn, info) else %{healthy: false} = info -> service_unavailable(conn, info) + {:cfg, false} -> + service_unavailable(conn, %{"error" => "Healthcheck disabled"}) + _ -> service_unavailable(conn, %{}) end diff --git a/lib/pleroma/web/views/streamer_view.ex b/lib/pleroma/web/views/streamer_view.ex index 6a55242b0..f97570b0a 100644 --- a/lib/pleroma/web/views/streamer_view.ex +++ b/lib/pleroma/web/views/streamer_view.ex @@ -11,8 +11,11 @@ defmodule Pleroma.Web.StreamerView do alias Pleroma.User alias Pleroma.Web.MastodonAPI.NotificationView - def render("update.json", %Activity{} = activity, %User{} = user) do + require Pleroma.Constants + + def render("update.json", %Activity{} = activity, %User{} = user, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -25,8 +28,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("status_update.json", %Activity{} = activity, %User{} = user) do + def render("status_update.json", %Activity{} = activity, %User{} = user, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "status.update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -39,8 +43,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("notification.json", %Notification{} = notify, %User{} = user) do + def render("notification.json", %Notification{} = notify, %User{} = user, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "notification", payload: NotificationView.render( @@ -52,8 +57,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("update.json", %Activity{} = activity) do + def render("update.json", %Activity{} = activity, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -65,8 +71,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("status_update.json", %Activity{} = activity) do + def render("status_update.json", %Activity{} = activity, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "status.update", payload: Pleroma.Web.MastodonAPI.StatusView.render( @@ -78,7 +85,7 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("chat_update.json", %{chat_message_reference: cm_ref}) do + def render("chat_update.json", %{chat_message_reference: cm_ref}, topic) do # Explicitly giving the cmr for the object here, so we don't accidentally # send a later 'last_message' that was inserted between inserting this and # streaming it out @@ -93,6 +100,7 @@ defmodule Pleroma.Web.StreamerView do ) %{ + stream: render("stream.json", %{topic: topic}), event: "pleroma:chat_update", payload: representation @@ -101,8 +109,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("follow_relationships_update.json", item) do + def render("follow_relationships_update.json", item, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "pleroma:follow_relationships_update", payload: %{ @@ -123,8 +132,9 @@ defmodule Pleroma.Web.StreamerView do |> Jason.encode!() end - def render("conversation.json", %Participation{} = participation) do + def render("conversation.json", %Participation{} = participation, topic) do %{ + stream: render("stream.json", %{topic: topic}), event: "conversation", payload: Pleroma.Web.MastodonAPI.ConversationView.render("participation.json", %{ @@ -135,4 +145,39 @@ defmodule Pleroma.Web.StreamerView do } |> Jason.encode!() end + + def render("pleroma_respond.json", %{type: type, result: result} = params) do + %{ + event: "pleroma:respond", + payload: + %{ + result: result, + type: type + } + |> Map.merge(maybe_error(params)) + |> Jason.encode!() + } + |> Jason.encode!() + end + + def render("stream.json", %{topic: "user:pleroma_chat:" <> _}), do: ["user:pleroma_chat"] + def render("stream.json", %{topic: "user:notification:" <> _}), do: ["user:notification"] + def render("stream.json", %{topic: "user:" <> _}), do: ["user"] + def render("stream.json", %{topic: "direct:" <> _}), do: ["direct"] + def render("stream.json", %{topic: "list:" <> id}), do: ["list", id] + def render("stream.json", %{topic: "hashtag:" <> tag}), do: ["hashtag", tag] + + def render("stream.json", %{topic: "public:remote:media:" <> instance}), + do: ["public:remote:media", instance] + + def render("stream.json", %{topic: "public:remote:" <> instance}), + do: ["public:remote", instance] + + def render("stream.json", %{topic: stream}) when stream in Pleroma.Constants.public_streams(), + do: [stream] + + defp maybe_error(%{error: :bad_topic}), do: %{error: "bad_topic"} + defp maybe_error(%{error: :unauthorized}), do: %{error: "unauthorized"} + defp maybe_error(%{error: :already_authenticated}), do: %{error: "already_authenticated"} + defp maybe_error(_), do: %{} end diff --git a/lib/pleroma/web/web_finger.ex b/lib/pleroma/web/web_finger.ex index f95dc2458..0684a770c 100644 --- a/lib/pleroma/web/web_finger.ex +++ b/lib/pleroma/web/web_finger.ex @@ -70,7 +70,7 @@ defmodule Pleroma.Web.WebFinger do def represent_user(user, "JSON") do %{ - "subject" => "acct:#{user.nickname}@#{domain()}", + "subject" => "acct:#{user.nickname}@#{host()}", "aliases" => gather_aliases(user), "links" => gather_links(user) } @@ -90,13 +90,13 @@ defmodule Pleroma.Web.WebFinger do :XRD, %{xmlns: "http://docs.oasis-open.org/ns/xri/xrd-1.0"}, [ - {:Subject, "acct:#{user.nickname}@#{domain()}"} + {:Subject, "acct:#{user.nickname}@#{host()}"} ] ++ aliases ++ links } |> XmlBuilder.to_doc() end - defp domain do + def host do Pleroma.Config.get([__MODULE__, :domain]) || Pleroma.Web.Endpoint.host() end @@ -163,7 +163,7 @@ defmodule Pleroma.Web.WebFinger do get_template_from_xml(body) else error -> - Logger.warn("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}") + Logger.warning("Can't find LRDD template in #{inspect(meta_url)}: #{inspect(error)}") {:error, :lrdd_not_found} end end diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 1540c1605..0292bbb3b 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do The worker to send digest emails. """ - use Oban.Worker, queue: "digest_emails" + use Oban.Worker, queue: "mailer" alias Pleroma.Config alias Pleroma.Emails diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 267fe2837..1c3e445aa 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "new_users_digest" + use Pleroma.Workers.WorkerHelper, queue: "mailer" @impl Oban.Worker def perform(_job) do diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex index cf1bb62b4..1dddd8d2e 100644 --- a/lib/pleroma/workers/receiver_worker.ex +++ b/lib/pleroma/workers/receiver_worker.ex @@ -3,24 +3,56 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.ReceiverWorker do + alias Pleroma.Signature + alias Pleroma.User alias Pleroma.Web.Federator use Pleroma.Workers.WorkerHelper, queue: "federator_incoming" @impl Oban.Worker + + def perform(%Job{ + args: %{"op" => "incoming_ap_doc", "req_headers" => req_headers, "params" => params} + }) do + # Oban's serialization converts our tuple headers to lists. + # Revert it for the signature validation. + req_headers = Enum.into(req_headers, [], &List.to_tuple(&1)) + + conn_data = %{params: params, req_headers: req_headers} + + with {:ok, %User{} = _actor} <- User.get_or_fetch_by_ap_id(conn_data.params["actor"]), + {:ok, _public_key} <- Signature.refetch_public_key(conn_data), + {:signature, true} <- {:signature, HTTPSignatures.validate_conn(conn_data)}, + {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do + {:ok, res} + else + e -> process_errors(e) + end + end + def perform(%Job{args: %{"op" => "incoming_ap_doc", "params" => params}}) do with {:ok, res} <- Federator.perform(:incoming_ap_doc, params) do {:ok, res} else + e -> process_errors(e) + end + end + + @impl Oban.Worker + def timeout(%_{args: %{"timeout" => timeout}}), do: timeout + + def timeout(_job), do: :timer.seconds(5) + + defp process_errors(errors) do + case errors do {:error, :origin_containment_failed} -> {:cancel, :origin_containment_failed} {:error, :already_present} -> {:cancel, :already_present} {:error, {:validate_object, reason}} -> {:cancel, reason} {:error, {:error, {:validate, reason}}} -> {:cancel, reason} {:error, {:reject, reason}} -> {:cancel, reason} + {:signature, false} -> {:cancel, :invalid_signature} + {:error, {:error, reason = "Object has been deleted"}} -> {:cancel, reason} e -> e end end - - @impl Oban.Worker - def timeout(_job), do: :timer.seconds(5) end diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex new file mode 100644 index 000000000..8476a2be5 --- /dev/null +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -0,0 +1,23 @@ +defmodule Pleroma.Workers.SearchIndexingWorker do + use Pleroma.Workers.WorkerHelper, queue: "search_indexing" + + @impl Oban.Worker + + alias Pleroma.Config.Getting, as: Config + + def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do + activity = Pleroma.Activity.get_by_id_with_object(activity_id) + + search_module = Config.get([Pleroma.Search, :module]) + + search_module.add_to_index(activity) + end + + def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do + object = Pleroma.Object.get_by_id(object_id) + + search_module = Config.get([Pleroma.Search, :module]) + + search_module.remove_from_index(object) + end +end |