From 66cb3294ed942d461cabc32881e2a10ebfd182af Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 2 Nov 2022 22:49:55 -0400 Subject: Switch to PromEx for prometheus metrics Recommending use of the separate HTTP server for exposing the metrics and securing it externally on your firewall or reverse proxy. It will listen on port 4021 by default. --- lib/pleroma/application.ex | 25 +------------------ lib/pleroma/config/transfer_task.ex | 3 +-- lib/pleroma/prom_ex.ex | 49 +++++++++++++++++++++++++++++++++++++ lib/pleroma/repo.ex | 2 -- lib/pleroma/web/endpoint.ex | 41 ------------------------------- 5 files changed, 51 insertions(+), 69 deletions(-) create mode 100644 lib/pleroma/prom_ex.ex (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 7bbc132f1..52cd6e9a9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -54,7 +54,6 @@ defmodule Pleroma.Application do Config.DeprecationWarnings.warn() Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() Pleroma.ApplicationRequirements.verify!() - setup_instrumenters() load_custom_modules() Pleroma.Docs.JSON.compile() limiters_setup() @@ -91,6 +90,7 @@ defmodule Pleroma.Application do # Define workers and child supervisors to be supervised children = [ + Pleroma.PromEx, Pleroma.Repo, Config.TransferTask, Pleroma.Emoji, @@ -170,29 +170,6 @@ defmodule Pleroma.Application do end end - defp setup_instrumenters do - require Prometheus.Registry - - if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do - :ok = - :telemetry.attach( - "prometheus-ecto", - [:pleroma, :repo, :query], - &Pleroma.Repo.Instrumenter.handle_event/4, - %{} - ) - - Pleroma.Repo.Instrumenter.setup() - end - - Pleroma.Web.Endpoint.MetricsExporter.setup() - Pleroma.Web.Endpoint.PipelineInstrumenter.setup() - - # Note: disabled until prometheus-phx is integrated into prometheus-phoenix: - # Pleroma.Web.Endpoint.Instrumenter.setup() - PrometheusPhx.setup() - end - defp cachex_children do [ build_cachex("used_captcha", ttl_interval: seconds_valid_interval()), diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 44a984019..6fd05b0e0 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -55,8 +55,7 @@ defmodule Pleroma.Config.TransferTask do started_applications = Application.started_applications() - # TODO: some problem with prometheus after restart! - reject = [nil, :prometheus, :postgrex] + reject = [nil, :postgrex] reject = if restart_pleroma? do diff --git a/lib/pleroma/prom_ex.ex b/lib/pleroma/prom_ex.ex new file mode 100644 index 000000000..6608708b7 --- /dev/null +++ b/lib/pleroma/prom_ex.ex @@ -0,0 +1,49 @@ +defmodule Pleroma.PromEx do + use PromEx, otp_app: :pleroma + + alias PromEx.Plugins + + @impl true + def plugins do + [ + # PromEx built in plugins + Plugins.Application, + Plugins.Beam, + {Plugins.Phoenix, router: Pleroma.Web.Router, endpoint: Pleroma.Web.Endpoint}, + Plugins.Ecto, + Plugins.Oban + # Plugins.PhoenixLiveView, + # Plugins.Absinthe, + # Plugins.Broadway, + + # Add your own PromEx metrics plugins + # Pleroma.Users.PromExPlugin + ] + end + + @impl true + def dashboard_assigns do + [ + datasource_id: Pleroma.Config.get([Pleroma.PromEx, :datasource]), + default_selected_interval: "30s" + ] + end + + @impl true + def dashboards do + [ + # PromEx built in Grafana dashboards + {:prom_ex, "application.json"}, + {:prom_ex, "beam.json"}, + {:prom_ex, "phoenix.json"}, + {:prom_ex, "ecto.json"}, + {:prom_ex, "oban.json"} + # {:prom_ex, "phoenix_live_view.json"}, + # {:prom_ex, "absinthe.json"}, + # {:prom_ex, "broadway.json"}, + + # Add your dashboard definitions here with the format: {:otp_app, "path_in_priv"} + # {:pleroma, "/grafana_dashboards/user_metrics.json"} + ] + end +end diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 515b0c1ff..a50a59b3b 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -11,8 +11,6 @@ defmodule Pleroma.Repo do import Ecto.Query require Logger - defmodule Instrumenter, do: use(Prometheus.EctoInstrumenter) - @doc """ Dynamically loads the repository url from the DATABASE_URL environment variable. diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 65dd72c49..307fa069e 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -151,47 +151,6 @@ defmodule Pleroma.Web.Endpoint do plug(Pleroma.Web.Plugs.RemoteIp) - defmodule Instrumenter do - use Prometheus.PhoenixInstrumenter - end - - defmodule PipelineInstrumenter do - use Prometheus.PlugPipelineInstrumenter - end - - defmodule MetricsExporter do - use Prometheus.PlugExporter - end - - defmodule MetricsExporterCaller do - @behaviour Plug - - def init(opts), do: opts - - def call(conn, opts) do - prometheus_config = Application.get_env(:prometheus, MetricsExporter, []) - ip_whitelist = List.wrap(prometheus_config[:ip_whitelist]) - - cond do - !prometheus_config[:enabled] -> - conn - - ip_whitelist != [] and - !Enum.find(ip_whitelist, fn ip -> - Pleroma.Helpers.InetHelper.parse_address(ip) == {:ok, conn.remote_ip} - end) -> - conn - - true -> - MetricsExporter.call(conn, opts) - end - end - end - - plug(PipelineInstrumenter) - - plug(MetricsExporterCaller) - plug(Pleroma.Web.Router) @doc """ -- cgit v1.2.3 From 13baba90f6deb27b0d6301a705db753cc32bb141 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 9 Nov 2022 13:45:57 -0500 Subject: Replace ImageMagick with Vips for Media Preview Proxy --- lib/pleroma/helpers/media_helper.ex | 57 +++++++++---------------------------- 1 file changed, 13 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/helpers/media_helper.ex b/lib/pleroma/helpers/media_helper.ex index 24c845fcd..07dfea55b 100644 --- a/lib/pleroma/helpers/media_helper.ex +++ b/lib/pleroma/helpers/media_helper.ex @@ -8,11 +8,12 @@ defmodule Pleroma.Helpers.MediaHelper do """ alias Pleroma.HTTP + alias Vix.Vips.Operation require Logger def missing_dependencies do - Enum.reduce([imagemagick: "convert", ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> + Enum.reduce([ffmpeg: "ffmpeg"], [], fn {sym, executable}, acc -> if Pleroma.Utils.command_available?(executable) do acc else @@ -22,54 +23,22 @@ defmodule Pleroma.Helpers.MediaHelper do end def image_resize(url, options) do - with executable when is_binary(executable) <- System.find_executable("convert"), - {:ok, args} <- prepare_image_resize_args(options), - {:ok, env} <- HTTP.get(url, [], pool: :media), - {:ok, fifo_path} <- mkfifo() do - args = List.flatten([fifo_path, args]) - run_fifo(fifo_path, env, executable, args) + with {:ok, env} <- HTTP.get(url, [], pool: :media), + {:ok, resized} <- + Operation.thumbnail_buffer(env.body, options.max_width, + height: options.max_height, + size: :VIPS_SIZE_DOWN + ) do + if options[:format] == "png" do + Operation.pngsave_buffer(resized, Q: options[:quality]) + else + Operation.jpegsave_buffer(resized, Q: options[:quality], interlace: true) + end else - nil -> {:error, {:convert, :command_not_found}} {:error, _} = error -> error end end - defp prepare_image_resize_args( - %{max_width: max_width, max_height: max_height, format: "png"} = options - ) do - quality = options[:quality] || 85 - resize = Enum.join([max_width, "x", max_height, ">"]) - - args = [ - "-resize", - resize, - "-quality", - to_string(quality), - "png:-" - ] - - {:ok, args} - end - - defp prepare_image_resize_args(%{max_width: max_width, max_height: max_height} = options) do - quality = options[:quality] || 85 - resize = Enum.join([max_width, "x", max_height, ">"]) - - args = [ - "-interlace", - "Plane", - "-resize", - resize, - "-quality", - to_string(quality), - "jpg:-" - ] - - {:ok, args} - end - - defp prepare_image_resize_args(_), do: {:error, :missing_options} - # Note: video thumbnail is intentionally not resized (always has original dimensions) def video_framegrab(url) do with executable when is_binary(executable) <- System.find_executable("ffmpeg"), -- cgit v1.2.3 From 481b6ac0d5c610e840fc4b88a357efa28f510ab8 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 10 Nov 2022 11:07:49 -0500 Subject: Add Pleroma.Upload.Filter.HeifToJpeg based on vips --- lib/pleroma/upload/filter/heif_to_jpeg.ex | 36 +++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 lib/pleroma/upload/filter/heif_to_jpeg.ex (limited to 'lib') diff --git a/lib/pleroma/upload/filter/heif_to_jpeg.ex b/lib/pleroma/upload/filter/heif_to_jpeg.ex new file mode 100644 index 000000000..a2095ba01 --- /dev/null +++ b/lib/pleroma/upload/filter/heif_to_jpeg.ex @@ -0,0 +1,36 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.HeifToJpeg do + @behaviour Pleroma.Upload.Filter + alias Pleroma.Upload + alias Vix.Vips.Operation + + @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} + @type conversions :: conversion() | [conversion()] + + @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} + def filter(%Pleroma.Upload{content_type: "image/avif"} = upload), do: apply_filter(upload) + def filter(%Pleroma.Upload{content_type: "image/heic"} = upload), do: apply_filter(upload) + def filter(%Pleroma.Upload{content_type: "image/heif"} = upload), do: apply_filter(upload) + + def filter(_), do: {:ok, :noop} + + defp apply_filter(%Pleroma.Upload{name: name, path: path, tempfile: tempfile} = upload) do + ext = String.split(path, ".") |> List.last() + + try do + name = name |> String.replace_suffix(ext, "jpg") + path = path |> String.replace_suffix(ext, "jpg") + {:ok, {vixdata, _vixflags}} = Operation.heifload(tempfile) + {:ok, jpegdata} = Operation.jpegsave_buffer(vixdata) + :ok = File.write(tempfile, jpegdata) + + {:ok, :filtered, %Upload{upload | name: name, path: path, content_type: "image/jpeg"}} + rescue + e in ErlangError -> + {:error, "#{__MODULE__}: #{inspect(e)}"} + end + end +end -- cgit v1.2.3 From a4b6e5613fe02fac1a0ec80b8fd6940d2b22f85d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 7 Nov 2023 22:03:20 +0000 Subject: Revert "Add Pleroma.Upload.Filter.HeifToJpeg based on vips" This reverts commit 31d4448ee61b4afac6aa23f8c0287d13aed411a1. This functionality is not reliably working with vips/vix due to codec patent junk --- lib/pleroma/upload/filter/heif_to_jpeg.ex | 36 ------------------------------- 1 file changed, 36 deletions(-) delete mode 100644 lib/pleroma/upload/filter/heif_to_jpeg.ex (limited to 'lib') diff --git a/lib/pleroma/upload/filter/heif_to_jpeg.ex b/lib/pleroma/upload/filter/heif_to_jpeg.ex deleted file mode 100644 index a2095ba01..000000000 --- a/lib/pleroma/upload/filter/heif_to_jpeg.ex +++ /dev/null @@ -1,36 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Upload.Filter.HeifToJpeg do - @behaviour Pleroma.Upload.Filter - alias Pleroma.Upload - alias Vix.Vips.Operation - - @type conversion :: action :: String.t() | {action :: String.t(), opts :: String.t()} - @type conversions :: conversion() | [conversion()] - - @spec filter(Pleroma.Upload.t()) :: {:ok, :atom} | {:error, String.t()} - def filter(%Pleroma.Upload{content_type: "image/avif"} = upload), do: apply_filter(upload) - def filter(%Pleroma.Upload{content_type: "image/heic"} = upload), do: apply_filter(upload) - def filter(%Pleroma.Upload{content_type: "image/heif"} = upload), do: apply_filter(upload) - - def filter(_), do: {:ok, :noop} - - defp apply_filter(%Pleroma.Upload{name: name, path: path, tempfile: tempfile} = upload) do - ext = String.split(path, ".") |> List.last() - - try do - name = name |> String.replace_suffix(ext, "jpg") - path = path |> String.replace_suffix(ext, "jpg") - {:ok, {vixdata, _vixflags}} = Operation.heifload(tempfile) - {:ok, jpegdata} = Operation.jpegsave_buffer(vixdata) - :ok = File.write(tempfile, jpegdata) - - {:ok, :filtered, %Upload{upload | name: name, path: path, content_type: "image/jpeg"}} - rescue - e in ErlangError -> - {:error, "#{__MODULE__}: #{inspect(e)}"} - end - end -end -- cgit v1.2.3 From 66f5ae0c5a0ab57dc6bf3f52bbf976128259800f Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 8 Aug 2023 19:08:59 +0200 Subject: router: Make /federation_status publicly available --- lib/pleroma/web/router.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 9abad65b0..eb8576b02 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -224,6 +224,12 @@ defmodule Pleroma.Web.Router do post("/remote_interaction", UtilController, :remote_interaction) end + scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do + pipe_through(:pleroma_api) + + get("/federation_status", InstancesController, :show) + end + scope "/api/v1/pleroma", Pleroma.Web do pipe_through(:pleroma_api) post("/uploader_callback/:upload_path", UploaderController, :callback) @@ -604,7 +610,6 @@ defmodule Pleroma.Web.Router do scope "/api/v1/pleroma", Pleroma.Web.PleromaAPI do pipe_through(:api) get("/accounts/:id/scrobbles", ScrobbleController, :index) - get("/federation_status", InstancesController, :show) end scope "/api/v2/pleroma", Pleroma.Web.PleromaAPI do -- cgit v1.2.3 From a5aa8ea79603e22541de04c26293dc87bd2f2ed8 Mon Sep 17 00:00:00 2001 From: Henry Jameson Date: Sun, 29 Oct 2023 18:58:57 +0200 Subject: Add support for configuring a favicon and embed PWA manifest in server-generated-meta --- lib/pleroma/web/fallback/redirect_controller.ex | 26 ++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 1a86f7a53..005a5da8b 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -18,9 +18,22 @@ defmodule Pleroma.Web.Fallback.RedirectController do end def redirector(conn, _params, code \\ 200) do + {:ok, index_content} = File.read(index_file_path()) + + title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" + + response = + index_content + |> String.replace( + "", + title <> favicon <> manifest + ) + conn |> put_resp_content_type("text/html") - |> send_file(code, index_file_path()) + |> send_resp(code, response) end def redirector_with_meta(conn, %{"maybe_nickname_or_id" => maybe_nickname_or_id} = params) do @@ -38,10 +51,15 @@ defmodule Pleroma.Web.Fallback.RedirectController do tags = build_tags(conn, params) preloads = preload_data(conn, params) title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" response = index_content - |> String.replace("", tags <> preloads <> title) + |> String.replace( + "", + tags <> preloads <> title <> favicon <> manifest + ) conn |> put_resp_content_type("text/html") @@ -56,10 +74,12 @@ defmodule Pleroma.Web.Fallback.RedirectController do {:ok, index_content} = File.read(index_file_path()) preloads = preload_data(conn, params) title = "#{Pleroma.Config.get([:instance, :name])}" + favicon = "" + manifest = "" response = index_content - |> String.replace("", preloads <> title) + |> String.replace("", preloads <> title <> favicon <> manifest) conn |> put_resp_content_type("text/html") -- cgit v1.2.3 From 5d3e145dc4090dc5709a649ee5d646b637da252e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Tue, 14 Nov 2023 11:01:02 +0100 Subject: RedirectController: Unify server-generated-meta insertion code --- lib/pleroma/web/fallback/redirect_controller.ex | 33 +++++++++++-------------- 1 file changed, 14 insertions(+), 19 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/fallback/redirect_controller.ex b/lib/pleroma/web/fallback/redirect_controller.ex index 005a5da8b..4a0885fab 100644 --- a/lib/pleroma/web/fallback/redirect_controller.ex +++ b/lib/pleroma/web/fallback/redirect_controller.ex @@ -17,19 +17,24 @@ defmodule Pleroma.Web.Fallback.RedirectController do |> json(%{error: "Not implemented"}) end - def redirector(conn, _params, code \\ 200) do - {:ok, index_content} = File.read(index_file_path()) - + def add_generated_metadata(page_content, extra \\ "") do title = "#{Pleroma.Config.get([:instance, :name])}" favicon = "" manifest = "" + page_content + |> String.replace( + "", + title <> favicon <> manifest <> extra + ) + end + + def redirector(conn, _params, code \\ 200) do + {:ok, index_content} = File.read(index_file_path()) + response = index_content - |> String.replace( - "", - title <> favicon <> manifest - ) + |> add_generated_metadata() conn |> put_resp_content_type("text/html") @@ -47,19 +52,12 @@ defmodule Pleroma.Web.Fallback.RedirectController do def redirector_with_meta(conn, params) do {:ok, index_content} = File.read(index_file_path()) - tags = build_tags(conn, params) preloads = preload_data(conn, params) - title = "#{Pleroma.Config.get([:instance, :name])}" - favicon = "" - manifest = "" response = index_content - |> String.replace( - "", - tags <> preloads <> title <> favicon <> manifest - ) + |> add_generated_metadata(tags <> preloads) conn |> put_resp_content_type("text/html") @@ -73,13 +71,10 @@ defmodule Pleroma.Web.Fallback.RedirectController do def redirector_with_preload(conn, params) do {:ok, index_content} = File.read(index_file_path()) preloads = preload_data(conn, params) - title = "#{Pleroma.Config.get([:instance, :name])}" - favicon = "" - manifest = "" response = index_content - |> String.replace("", preloads <> title <> favicon <> manifest) + |> add_generated_metadata(preloads) conn |> put_resp_content_type("text/html") -- cgit v1.2.3 From 0c6a54b37cb208e577fc9e40782bb8b820730428 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 Nov 2023 17:04:47 -0500 Subject: Upload.Filter.AnalyzeMetadata: Blurhash with a Rust NIF, and use Vix to retrieve image metadata --- lib/pleroma/upload/filter/analyze_metadata.ex | 36 +++++++++++++++++++++------ 1 file changed, 28 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 9a76a998b..15dec5564 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -8,22 +8,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do """ require Logger + alias Vix.Vips.Image + alias Vix.Vips.Operation + @behaviour Pleroma.Upload.Filter @spec filter(Pleroma.Upload.t()) :: {:ok, :filtered, Pleroma.Upload.t()} | {:ok, :noop} | {:error, String.t()} def filter(%Pleroma.Upload{tempfile: file, content_type: "image" <> _} = upload) do try do - image = - file - |> Mogrify.open() - |> Mogrify.verbose() + {:ok, image} = Image.new_from_file(file) + {width, height} = {Image.width(image), Image.height(image)} upload = upload - |> Map.put(:width, image.width) - |> Map.put(:height, image.height) - |> Map.put(:blurhash, get_blurhash(file)) + |> Map.put(:width, width) + |> Map.put(:height, height) + |> Map.put(:blurhash, get_blurhash(image)) {:ok, :filtered, upload} rescue @@ -53,7 +54,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do def filter(_), do: {:ok, :noop} defp get_blurhash(file) do - with {:ok, blurhash} <- :eblurhash.magick(file) do + with {:ok, blurhash} <- vips_blurhash(file) do blurhash else _ -> nil @@ -80,4 +81,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {:error, _} = error -> error end end + + defp vips_blurhash(image = %Vix.Vips.Image{}) do + {:ok, resized_image} = Operation.thumbnail_image(image, 20) + {height, width} = {Image.height(resized_image), Image.width(resized_image)} + max = max(height, width) + {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} + + {:ok, rgba} = + if Image.has_alpha?(resized_image) do + Image.to_list(resized_image) + else + Operation.bandjoin_const!(resized_image, [255]) + |> Image.to_list() + end + + rgba = List.flatten(rgba) + + Blurhash.encode(x, y, width, height, rgba) + end end -- cgit v1.2.3 From 88cc7e6a0431385b5fce292179c7eb05e64e3a24 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Fri, 17 Nov 2023 11:06:31 -0500 Subject: Resize images to 100 pixels before hashing --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 15dec5564..99d1ec5bc 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end defp vips_blurhash(image = %Vix.Vips.Image{}) do - {:ok, resized_image} = Operation.thumbnail_image(image, 20) + {:ok, resized_image} = Operation.thumbnail_image(image, 100) {height, width} = {Image.height(resized_image), Image.width(resized_image)} max = max(height, width) {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} -- cgit v1.2.3 From 510a7b64f1354d4313ab565d557f422b7e059432 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Wed, 15 Nov 2023 00:43:58 -0700 Subject: Add optional URL value for scrobbles --- lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex | 6 +++++- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/pleroma_api/views/scrobble_view.ex | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index ca40da930..68c586b73 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -59,6 +59,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, length: %Schema{type: :integer, description: "The length of the media playing"}, + url: %Schema{type: :string, description: "A URL referencing the media playing"}, visibility: %Schema{ allOf: [VisibilityScope], default: "public", @@ -69,7 +70,8 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "title" => "Some Title", "artist" => "Some Artist", "album" => "Some Album", - "length" => 180_000 + "length" => 180_000, + "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title" } } end @@ -83,6 +85,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do title: %Schema{type: :string, description: "The title of the media playing"}, album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, + url: %Schema{type: :string, description: "A URL referencing the media playing"}, length: %Schema{ type: :integer, description: "The length of the media playing", @@ -97,6 +100,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, + "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at" => "2019-09-28T12:40:45.000Z" } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index ca1329284..00cbacbb4 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp listen_object(draft) do object = draft.params - |> Map.take([:album, :artist, :title, :length]) + |> Map.take([:album, :artist, :title, :length, :url]) |> Map.new(fn {key, value} -> {to_string(key), value} end) |> Map.put("type", "Audio") |> Map.put("to", draft.to) diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index a5985fb2a..7a983f8b5 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -27,6 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do title: object.data["title"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(), album: object.data["album"] |> HTML.strip_tags(), + url: object.data["url"], length: object.data["length"] } end -- cgit v1.2.3 From 299c548b124377e51f6c089bc0df31b2989be3ef Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 23 Nov 2023 16:15:53 -0500 Subject: Prevent a blurhash failure from breaking all metadata collection --- lib/pleroma/upload/filter/analyze_metadata.ex | 34 ++++++++++++++------------- 1 file changed, 18 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 99d1ec5bc..92b80b1b1 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -83,21 +83,23 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end defp vips_blurhash(image = %Vix.Vips.Image{}) do - {:ok, resized_image} = Operation.thumbnail_image(image, 100) - {height, width} = {Image.height(resized_image), Image.width(resized_image)} - max = max(height, width) - {x, y} = {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} - - {:ok, rgba} = - if Image.has_alpha?(resized_image) do - Image.to_list(resized_image) - else - Operation.bandjoin_const!(resized_image, [255]) - |> Image.to_list() - end - - rgba = List.flatten(rgba) - - Blurhash.encode(x, y, width, height, rgba) + with {:ok, resized_image} <- Operation.thumbnail_image(image, 100), + {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, + max <- max(height, width), + {x, y} <- {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} do + {:ok, rgba} = + if Image.has_alpha?(resized_image) do + Image.to_list(resized_image) + else + Operation.bandjoin_const!(resized_image, [255]) + |> Image.to_list() + end + + rgba = List.flatten(rgba) + + Blurhash.encode(x, y, width, height, rgba) + else + _ -> nil + end end end -- cgit v1.2.3 From 27df2c0ce6c214f36db742af702fc239f80764a7 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Mon, 27 Nov 2023 03:34:31 -0700 Subject: Fix #strip_report_status_data --- lib/pleroma/web/activity_pub/utils.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 437220077..073ccd615 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -855,6 +855,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do Enum.map(reported_activities, fn act when is_map(act) -> act["id"] act when is_binary(act) -> act + _other -> nil end) new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) -- cgit v1.2.3 From 4ef56c5b65a4d1e7e90a87f1a1a507df523a4b4b Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Mon, 27 Nov 2023 18:44:11 +0400 Subject: ActivityPub.Utils: Only treat object ids as valid while stripping --- lib/pleroma/web/activity_pub/utils.ex | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index 073ccd615..b32f19740 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.Utils do alias Ecto.UUID alias Pleroma.Activity alias Pleroma.Config + alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID alias Pleroma.Maps alias Pleroma.Notification alias Pleroma.Object @@ -852,10 +853,11 @@ defmodule Pleroma.Web.ActivityPub.Utils do [actor | reported_activities] = activity.data["object"] stripped_activities = - Enum.map(reported_activities, fn - act when is_map(act) -> act["id"] - act when is_binary(act) -> act - _other -> nil + Enum.reduce(reported_activities, [], fn act, acc -> + case ObjectID.cast(act) do + {:ok, act} -> [act | acc] + _ -> acc + end end) new_data = put_in(activity.data, ["object"], [actor | stripped_activities]) -- cgit v1.2.3 From 03db495e1d88f34bef8d556b0f88806c3260d403 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 12:23:41 +0400 Subject: AnalyzeMetadata: Switch to rinpatch_blurhash --- lib/pleroma/upload/filter/analyze_metadata.ex | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 92b80b1b1..710fd02d2 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -87,17 +87,17 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, max <- max(height, width), {x, y} <- {max(round(width * 5 / max), 1), max(round(height * 5 / max), 1)} do - {:ok, rgba} = + {:ok, rgb} = if Image.has_alpha?(resized_image) do - Image.to_list(resized_image) + # remove alpha channel + resized_image + |> Operation.extract_band!(0, n: 3) + |> Image.write_to_binary() else - Operation.bandjoin_const!(resized_image, [255]) - |> Image.to_list() + Image.write_to_binary(resized_image) end - rgba = List.flatten(rgba) - - Blurhash.encode(x, y, width, height, rgba) + Blurhash.encode(rgb, width, height, x, y) else _ -> nil end -- cgit v1.2.3 From b3214be32f83541626b198b86ee065b92862f4cf Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 12:33:54 +0400 Subject: AnayzeMetadata: Fix error case that would never match --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index 710fd02d2..ef75d73d3 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -78,7 +78,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do %{width: width, height: height} else nil -> {:error, {:ffprobe, :command_not_found}} - {:error, _} = error -> error + error -> {:error, error} end end -- cgit v1.2.3 From ccc2adee4111367d67646f6f2828e03b861dd393 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 28 Nov 2023 13:13:43 +0400 Subject: Linting --- lib/pleroma/upload/filter/analyze_metadata.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/upload/filter/analyze_metadata.ex b/lib/pleroma/upload/filter/analyze_metadata.ex index ef75d73d3..e510ae3e6 100644 --- a/lib/pleroma/upload/filter/analyze_metadata.ex +++ b/lib/pleroma/upload/filter/analyze_metadata.ex @@ -82,7 +82,7 @@ defmodule Pleroma.Upload.Filter.AnalyzeMetadata do end end - defp vips_blurhash(image = %Vix.Vips.Image{}) do + defp vips_blurhash(%Vix.Vips.Image{} = image) do with {:ok, resized_image} <- Operation.thumbnail_image(image, 100), {height, width} <- {Image.height(resized_image), Image.width(resized_image)}, max <- max(height, width), -- cgit v1.2.3 From e216603477e2c393a586f7eb0bc8183e73bf4cd7 Mon Sep 17 00:00:00 2001 From: NEETzsche Date: Wed, 29 Nov 2023 07:55:44 -0700 Subject: Change url to externalLink as requested by hj here: https://shigusegubu.club/notice/AcIjZjackKAt6e522a --- lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex | 8 ++++---- lib/pleroma/web/common_api/activity_draft.ex | 2 +- lib/pleroma/web/pleroma_api/views/scrobble_view.ex | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex index 68c586b73..141b60533 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_scrobble_operation.ex @@ -59,7 +59,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, length: %Schema{type: :integer, description: "The length of the media playing"}, - url: %Schema{type: :string, description: "A URL referencing the media playing"}, + externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, visibility: %Schema{ allOf: [VisibilityScope], default: "public", @@ -71,7 +71,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, - "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title" + "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title" } } end @@ -85,7 +85,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do title: %Schema{type: :string, description: "The title of the media playing"}, album: %Schema{type: :string, description: "The album of the media playing"}, artist: %Schema{type: :string, description: "The artist of the media playing"}, - url: %Schema{type: :string, description: "A URL referencing the media playing"}, + externalLink: %Schema{type: :string, description: "A URL referencing the media playing"}, length: %Schema{ type: :integer, description: "The length of the media playing", @@ -100,7 +100,7 @@ defmodule Pleroma.Web.ApiSpec.PleromaScrobbleOperation do "artist" => "Some Artist", "album" => "Some Album", "length" => 180_000, - "url" => "https://www.last.fm/music/Some+Artist/_/Some+Title", + "externalLink" => "https://www.last.fm/music/Some+Artist/_/Some+Title", "created_at" => "2019-09-28T12:40:45.000Z" } } diff --git a/lib/pleroma/web/common_api/activity_draft.ex b/lib/pleroma/web/common_api/activity_draft.ex index 00cbacbb4..8910ad5b8 100644 --- a/lib/pleroma/web/common_api/activity_draft.ex +++ b/lib/pleroma/web/common_api/activity_draft.ex @@ -83,7 +83,7 @@ defmodule Pleroma.Web.CommonAPI.ActivityDraft do defp listen_object(draft) do object = draft.params - |> Map.take([:album, :artist, :title, :length, :url]) + |> Map.take([:album, :artist, :title, :length, :externalLink]) |> Map.new(fn {key, value} -> {to_string(key), value} end) |> Map.put("type", "Audio") |> Map.put("to", draft.to) diff --git a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex index 7a983f8b5..edf0a2390 100644 --- a/lib/pleroma/web/pleroma_api/views/scrobble_view.ex +++ b/lib/pleroma/web/pleroma_api/views/scrobble_view.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.PleromaAPI.ScrobbleView do title: object.data["title"] |> HTML.strip_tags(), artist: object.data["artist"] |> HTML.strip_tags(), album: object.data["album"] |> HTML.strip_tags(), - url: object.data["url"], + externalLink: object.data["externalLink"], length: object.data["length"] } end -- cgit v1.2.3 From 15a8acbd6cc6ecc89b49956acfeba49d02270c1d Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 30 Nov 2023 09:40:17 +0400 Subject: MRF, Docs.Generator: Ensure code is loaded before checking it --- lib/pleroma/docs/generator.ex | 2 ++ lib/pleroma/web/activity_pub/mrf.ex | 3 +++ 2 files changed, 5 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/docs/generator.ex b/lib/pleroma/docs/generator.ex index 6508f1947..456a8fd54 100644 --- a/lib/pleroma/docs/generator.ex +++ b/lib/pleroma/docs/generator.ex @@ -17,6 +17,8 @@ defmodule Pleroma.Docs.Generator do # This shouldn't be needed as all modules are expected to have module_info/1, # but in test enviroments some transient modules `:elixir_compiler_XX` # are loaded for some reason (where XX is a random integer). + Code.ensure_loaded(module) + if function_exported?(module, :module_info, 1) do module.module_info(:attributes) |> Keyword.get_values(:behaviour) diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index ff9f84497..a0deac300 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -54,6 +54,8 @@ defmodule Pleroma.Web.ActivityPub.MRF do @required_description_keys [:key, :related_policy] def filter_one(policy, message) do + Code.ensure_loaded(policy) + should_plug_history? = if function_exported?(policy, :history_awareness, 0) do policy.history_awareness() @@ -188,6 +190,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do def config_descriptions(policies) do Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc -> + Code.ensure_loaded(policy) if function_exported?(policy, :config_description, 0) do description = @default_description -- cgit v1.2.3 From eb6be30602f95cf86aef465229dacbdcb92c3c58 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 30 Nov 2023 10:05:00 +0400 Subject: Linting --- lib/pleroma/web/activity_pub/mrf.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf.ex b/lib/pleroma/web/activity_pub/mrf.ex index a0deac300..69054e25b 100644 --- a/lib/pleroma/web/activity_pub/mrf.ex +++ b/lib/pleroma/web/activity_pub/mrf.ex @@ -191,6 +191,7 @@ defmodule Pleroma.Web.ActivityPub.MRF do def config_descriptions(policies) do Enum.reduce(policies, @mrf_config_descriptions, fn policy, acc -> Code.ensure_loaded(policy) + if function_exported?(policy, :config_description, 0) do description = @default_description -- cgit v1.2.3 From 1ad0d94d6f1f11c848f277877cc14b3a47a7ae95 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 5 Dec 2023 16:35:41 -0500 Subject: Change set_reachable/1 to an upsert --- lib/pleroma/instances/instance.ex | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 9756c66dc..101e5dc88 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -97,13 +97,9 @@ defmodule Pleroma.Instances.Instance do def reachable?(url_or_host) when is_binary(url_or_host), do: true def set_reachable(url_or_host) when is_binary(url_or_host) do - with host <- host(url_or_host), - %Instance{} = existing_record <- Repo.get_by(Instance, %{host: host}) do - {:ok, _instance} = - existing_record - |> changeset(%{unreachable_since: nil}) - |> Repo.update() - end + %Instance{host: host(url_or_host)} + |> changeset(%{unreachable_since: nil}) + |> Repo.insert(on_conflict: {:replace, [:unreachable_since]}, conflict_target: :host) end def set_reachable(_), do: {:error, nil} -- cgit v1.2.3