From cb40602a167f4637dc6df6633ec2dfe33f774177 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 15 May 2020 21:34:46 +0300 Subject: added media proxy invalidation --- lib/pleroma/object.ex | 34 ++++++++++----- lib/pleroma/web/media_proxy/invalidation.ex | 19 +++++++++ lib/pleroma/web/media_proxy/invalidations/nginx.ex | 12 ++++++ .../web/media_proxy/invalidations/script.ex | 10 +++++ lib/pleroma/workers/attachments_cleanup_worker.ex | 49 +++++++++++++--------- 5 files changed, 94 insertions(+), 30 deletions(-) create mode 100644 lib/pleroma/web/media_proxy/invalidation.ex create mode 100644 lib/pleroma/web/media_proxy/invalidations/nginx.ex create mode 100644 lib/pleroma/web/media_proxy/invalidations/script.ex (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index e678fd415..66b233498 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -9,11 +9,13 @@ defmodule Pleroma.Object do import Ecto.Changeset alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object alias Pleroma.Object.Fetcher alias Pleroma.ObjectTombstone alias Pleroma.Repo alias Pleroma.User + alias Pleroma.Workers.AttachmentsCleanupWorker require Logger @@ -183,27 +185,37 @@ defmodule Pleroma.Object do def delete(%Object{data: %{"id" => id}} = object) do with {:ok, _obj} = swap_object_with_tombstone(object), deleted_activity = Activity.delete_all_by_object_ap_id(id), - {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), - {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do - with true <- Pleroma.Config.get([:instance, :cleanup_attachments]) do - {:ok, _} = - Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{ - "object" => object - }) - end + {:ok, _} <- invalid_object_cache(object) do + cleanup_attachments( + Config.get([:instance, :cleanup_attachments]), + %{"object" => object} + ) {:ok, object, deleted_activity} end end - def prune(%Object{data: %{"id" => id}} = object) do + @spec cleanup_attachments(boolean(), %{required(:object) => map()}) :: + {:ok, Oban.Job.t() | nil} + def cleanup_attachments(true, %{"object" => _} = params) do + AttachmentsCleanupWorker.enqueue("cleanup_attachments", params) + end + + def cleanup_attachments(_, _), do: {:ok, nil} + + def prune(%Object{data: %{"id" => _id}} = object) do with {:ok, object} <- Repo.delete(object), - {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), - {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do + {:ok, _} <- invalid_object_cache(object) do {:ok, object} end end + def invalid_object_cache(%Object{data: %{"id" => id}}) do + with {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do + Cachex.del(:web_resp_cache, URI.parse(id).path) + end + end + def set_cache(%Object{data: %{"id" => ap_id}} = object) do Cachex.put(:object_cache, "object:#{ap_id}", object) {:ok, object} diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex new file mode 100644 index 000000000..dd9a53a27 --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -0,0 +1,19 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation do + @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} + + alias Pleroma.Config + + def purge(urls) do + [:media_proxy, :invalidation, :enabled] + |> Config.get() + |> do_purge(urls) + end + + defp do_purge(true, urls) do + config = Config.get([:media_proxy, :invalidation]) + config[:provider].purge(urls, config[:options]) + :ok + end + + defp do_purge(_, _), do: :ok +end diff --git a/lib/pleroma/web/media_proxy/invalidations/nginx.ex b/lib/pleroma/web/media_proxy/invalidations/nginx.ex new file mode 100644 index 000000000..5bfdd505c --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidations/nginx.ex @@ -0,0 +1,12 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Nginx do + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, _opts) do + Enum.each(urls, fn url -> + Pleroma.HTTP.request(:purge, url, "", [], []) + end) + + {:ok, "success"} + end +end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex new file mode 100644 index 000000000..f458845a0 --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -0,0 +1,10 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Script do + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, %{script_path: script_path} = options) do + script_args = List.wrap(Map.get(options, :script_args, [])) + System.cmd(Path.expand(script_path), [urls] ++ script_args) + {:ok, "success"} + end +end diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 3c5820a86..49352db2a 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -27,8 +27,20 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + prefix = + case Pleroma.Config.get([Pleroma.Upload, :base_url]) do + nil -> "media" + _ -> "" + end + + base_url = + String.trim_trailing( + Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()), + "/" + ) + # find all objects for copies of the attachments, name and actor doesn't matter here - delete_ids = + object_ids_and_hrefs = from(o in Object, where: fragment( @@ -67,29 +79,28 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do |> Enum.map(fn {href, %{id: id, count: count}} -> # only delete files that have single instance with 1 <- count do - prefix = - case Pleroma.Config.get([Pleroma.Upload, :base_url]) do - nil -> "media" - _ -> "" - end - - base_url = - String.trim_trailing( - Pleroma.Config.get([Pleroma.Upload, :base_url], Pleroma.Web.base_url()), - "/" - ) - - file_path = String.trim_leading(href, "#{base_url}/#{prefix}") + href + |> String.trim_leading("#{base_url}/#{prefix}") + |> uploader.delete_file() - uploader.delete_file(file_path) + {id, href} + else + _ -> {id, nil} end - - id end) - from(o in Object, where: o.id in ^delete_ids) + object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end) + + from(o in Object, where: o.id in ^object_ids) |> Repo.delete_all() + + object_ids_and_hrefs + |> Enum.filter(fn {_, href} -> not is_nil(href) end) + |> Enum.map(&elem(&1, 1)) + |> Pleroma.Web.MediaProxy.Invalidation.purge() + + {:ok, :success} end - def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: :ok + def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} end -- cgit v1.2.3 From 3f8d68bdf3224cd6023b3d7f8e64221222872820 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sat, 16 May 2020 15:16:33 +0300 Subject: added example cache purge script --- lib/pleroma/web/media_proxy/invalidations/http.ex | 16 ++++++++++++++++ lib/pleroma/web/media_proxy/invalidations/nginx.ex | 12 ------------ lib/pleroma/web/media_proxy/invalidations/script.ex | 11 ++++++++--- 3 files changed, 24 insertions(+), 15 deletions(-) create mode 100644 lib/pleroma/web/media_proxy/invalidations/http.ex delete mode 100644 lib/pleroma/web/media_proxy/invalidations/nginx.ex (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex new file mode 100644 index 000000000..40c624efc --- /dev/null +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -0,0 +1,16 @@ +defmodule Pleroma.Web.MediaProxy.Invalidation.Http do + @behaviour Pleroma.Web.MediaProxy.Invalidation + + @impl Pleroma.Web.MediaProxy.Invalidation + def purge(urls, opts) do + method = Map.get(opts, :http_method, :purge) + headers = Map.get(opts, :http_headers, []) + options = Map.get(opts, :http_options, []) + + Enum.each(urls, fn url -> + Pleroma.HTTP.request(method, url, "", headers, options) + end) + + {:ok, "success"} + end +end diff --git a/lib/pleroma/web/media_proxy/invalidations/nginx.ex b/lib/pleroma/web/media_proxy/invalidations/nginx.ex deleted file mode 100644 index 5bfdd505c..000000000 --- a/lib/pleroma/web/media_proxy/invalidations/nginx.ex +++ /dev/null @@ -1,12 +0,0 @@ -defmodule Pleroma.Web.MediaProxy.Invalidation.Nginx do - @behaviour Pleroma.Web.MediaProxy.Invalidation - - @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, _opts) do - Enum.each(urls, fn url -> - Pleroma.HTTP.request(:purge, url, "", [], []) - end) - - {:ok, "success"} - end -end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index f458845a0..94c79511a 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -2,9 +2,14 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do @behaviour Pleroma.Web.MediaProxy.Invalidation @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, %{script_path: script_path} = options) do - script_args = List.wrap(Map.get(options, :script_args, [])) - System.cmd(Path.expand(script_path), [urls] ++ script_args) + def purge(urls, %{script_path: script_path} = _options) do + args = + urls + |> List.wrap() + |> Enum.uniq() + |> Enum.join(" ") + + System.cmd(Path.expand(script_path), [args]) {:ok, "success"} end end -- cgit v1.2.3 From c33a4315fb09e67d0ed5f644877054a3fb7b1fe1 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 18 May 2020 06:48:19 +0300 Subject: updated docs --- lib/pleroma/web/media_proxy/invalidation.ex | 5 +++-- lib/pleroma/web/media_proxy/invalidations/http.ex | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index dd9a53a27..371aa8ae0 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -10,8 +10,9 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do end defp do_purge(true, urls) do - config = Config.get([:media_proxy, :invalidation]) - config[:provider].purge(urls, config[:options]) + provider = Config.get([:media_proxy, :invalidation, :provider]) + options = Config.get(provider) + provider.purge(urls, options) :ok end diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 40c624efc..66fafa7ba 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -3,9 +3,9 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, opts) do - method = Map.get(opts, :http_method, :purge) - headers = Map.get(opts, :http_headers, []) - options = Map.get(opts, :http_options, []) + method = Map.get(opts, :method, :purge) + headers = Map.get(opts, :headers, []) + options = Map.get(opts, :options, []) Enum.each(urls, fn url -> Pleroma.HTTP.request(method, url, "", headers, options) -- cgit v1.2.3 From 5f0a3ac74d51333a778e6be26876fe26b0ff625b Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 18 May 2020 09:22:26 +0300 Subject: added tests --- lib/pleroma/web/media_proxy/invalidation.ex | 8 +++++- lib/pleroma/web/media_proxy/invalidations/http.ex | 26 ++++++++++++++++++- .../web/media_proxy/invalidations/script.ex | 30 ++++++++++++++++++++-- 3 files changed, 60 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 371aa8ae0..c037ff13e 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -1,8 +1,15 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.MediaProxy.Invalidation do + @moduledoc false + @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} alias Pleroma.Config + @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()} def purge(urls) do [:media_proxy, :invalidation, :enabled] |> Config.get() @@ -13,7 +20,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do provider = Config.get([:media_proxy, :invalidation, :provider]) options = Config.get(provider) provider.purge(urls, options) - :ok end defp do_purge(_, _), do: :ok diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 66fafa7ba..07248df6e 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -1,16 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.MediaProxy.Invalidation.Http do + @moduledoc false @behaviour Pleroma.Web.MediaProxy.Invalidation + require Logger + @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, opts) do method = Map.get(opts, :method, :purge) headers = Map.get(opts, :headers, []) options = Map.get(opts, :options, []) + Logger.debug("Running cache purge: #{inspect(urls)}") + Enum.each(urls, fn url -> - Pleroma.HTTP.request(method, url, "", headers, options) + with {:error, error} <- do_purge(method, url, headers, options) do + Logger.error("Error while cache purge: url - #{url}, error: #{inspect(error)}") + end end) {:ok, "success"} end + + defp do_purge(method, url, headers, options) do + case Pleroma.HTTP.request(method, url, "", headers, options) do + {:ok, %{status: status} = env} when 400 <= status and status < 500 -> + {:error, env} + + {:error, error} = error -> + error + + _ -> + {:ok, "success"} + end + end end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 94c79511a..6be782132 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -1,6 +1,14 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.MediaProxy.Invalidation.Script do + @moduledoc false + @behaviour Pleroma.Web.MediaProxy.Invalidation + require Logger + @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, %{script_path: script_path} = _options) do args = @@ -9,7 +17,25 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do |> Enum.uniq() |> Enum.join(" ") - System.cmd(Path.expand(script_path), [args]) - {:ok, "success"} + path = Path.expand(script_path) + + Logger.debug("Running cache purge: #{inspect(urls)}, #{path}") + + case do_purge(path, [args]) do + {result, exit_status} when exit_status > 0 -> + Logger.error("Error while cache purge: #{inspect(result)}") + {:error, inspect(result)} + + _ -> + {:ok, "success"} + end + end + + def purge(_, _), do: {:error, "not found script path"} + + defp do_purge(path, args) do + System.cmd(path, args) + rescue + error -> {inspect(error), 1} end end -- cgit v1.2.3