From 77e39e6aae0852df13b9576a081f30da13d7ec5b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 23 Dec 2020 15:06:20 -0600 Subject: Create dir for EmojiStealPolicy automatically --- lib/pleroma/application.ex | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index bd568d858..a1655721a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -114,6 +114,9 @@ defmodule Pleroma.Application do set_postgres_server_version() + # Requires Config.TransferTask so ConfigDB values are loaded + steal_emoji_policy_setup() + result end @@ -300,4 +303,20 @@ defmodule Pleroma.Application do [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy] |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0)) end + + @spec steal_emoji_policy_setup() :: :ok + def steal_emoji_policy_setup() do + with true <- + Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy in Config.get!([:mrf, :policies]) do + path = + [:instance, :static_dir] + |> Config.get!() + |> Path.join("emoji/steal") + + if !File.exists?(path), do: File.mkdir_p!(path) + else + _ -> + :ok + end + end end -- cgit v1.2.3 From 72aeb2e73b77611504e1cd524a9cb47faef6816f Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 23 Dec 2020 15:16:58 -0600 Subject: Mark private --- lib/pleroma/application.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index a1655721a..e3edd05ca 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -305,7 +305,7 @@ defmodule Pleroma.Application do end @spec steal_emoji_policy_setup() :: :ok - def steal_emoji_policy_setup() do + defp steal_emoji_policy_setup() do with true <- Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy in Config.get!([:mrf, :policies]) do path = -- cgit v1.2.3 From e02889edb29ca5631c346b59c299907e9f7b0161 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 23 Dec 2020 15:19:08 -0600 Subject: Add MRFs to the list of things that may need a soft reboot --- lib/pleroma/config/transfer_task.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index a0d7b7d71..8a6d66891 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -28,7 +28,8 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Pleroma.Captcha, [:seconds_valid]}, {:pleroma, Pleroma.Upload, [:proxy_remote]}, {:pleroma, :instance, [:upload_limit]}, - {:pleroma, :gopher, [:enabled]} + {:pleroma, :gopher, [:enabled]}, + {:pleroma, :mrf, [:policies]} ] def start_link(restart_pleroma? \\ true) do -- cgit v1.2.3 From aafd7b44ceaf50788f12061ee88071a288155b95 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 24 Dec 2020 20:27:28 +0300 Subject: check dir existence in policy --- lib/pleroma/application.ex | 19 ------------ .../web/activity_pub/mrf/steal_emoji_policy.ex | 34 ++++++++++++++-------- 2 files changed, 22 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e3edd05ca..bd568d858 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -114,9 +114,6 @@ defmodule Pleroma.Application do set_postgres_server_version() - # Requires Config.TransferTask so ConfigDB values are loaded - steal_emoji_policy_setup() - result end @@ -303,20 +300,4 @@ defmodule Pleroma.Application do [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.MediaProxy] |> Enum.each(&ConcurrentLimiter.new(&1, 1, 0)) end - - @spec steal_emoji_policy_setup() :: :ok - defp steal_emoji_policy_setup() do - with true <- - Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy in Config.get!([:mrf, :policies]) do - path = - [:instance, :static_dir] - |> Config.get!() - |> Path.join("emoji/steal") - - if !File.exists?(path), do: File.mkdir_p!(path) - else - _ -> - :ok - end - end end diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 2858af9eb..eabee6542 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -14,18 +14,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) - defp steal_emoji({shortcode, url}) do + defp steal_emoji({shortcode, url}, emoji_dir_path) do url = Pleroma.Web.MediaProxy.url(url) {:ok, response} = Pleroma.HTTP.get(url) size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) if byte_size(response.body) <= size_limit do - emoji_dir_path = - Config.get( - [:mrf_steal_emoji, :path], - Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") - ) - extension = url |> URI.parse() @@ -35,11 +29,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")]) - try do - :ok = File.write(file_path, response.body) - + with :ok <- File.write(file_path, response.body) do shortcode - rescue + else e -> Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") nil @@ -66,6 +58,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do if remote_host?(host) and accept_host?(host) do installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) + emoji_dir_path = + Config.get( + [:mrf_steal_emoji, :path], + Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") + ) + + if not Config.get([:mrf_steal_emoji, :dir_exists?], false) do + create_dir(emoji_dir_path) + end + new_emojis = foreign_emojis |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end) @@ -76,7 +78,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do !reject_emoji? end) - |> Enum.map(&steal_emoji(&1)) + |> Enum.map(&steal_emoji(&1, emoji_dir_path)) |> Enum.filter(& &1) if !Enum.empty?(new_emojis) do @@ -94,4 +96,12 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do def describe do {:ok, %{}} end + + defp create_dir(path) do + if not File.exists?(path) do + File.mkdir_p!(path) + end + + Config.put([:mrf_steal_emoji, :dir_exists?], true) + end end -- cgit v1.2.3 From 7bfb041658708f002e5da8f7b911b6011169dfef Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 25 Dec 2020 11:30:36 +0300 Subject: insreasing test coverage for StealEmojiPolicy --- .../web/activity_pub/mrf/steal_emoji_policy.ex | 64 +++++++++++----------- 1 file changed, 33 insertions(+), 31 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index eabee6542..0311ca433 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -10,52 +10,53 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do @moduledoc "Detect new emojis by their shortcode and steals them" @behaviour Pleroma.Web.ActivityPub.MRF - defp remote_host?(host), do: host != Config.get([Pleroma.Web.Endpoint, :url, :host]) - defp accept_host?(host), do: host in Config.get([:mrf_steal_emoji, :hosts], []) defp steal_emoji({shortcode, url}, emoji_dir_path) do url = Pleroma.Web.MediaProxy.url(url) - {:ok, response} = Pleroma.HTTP.get(url) - size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) - if byte_size(response.body) <= size_limit do - extension = - url - |> URI.parse() - |> Map.get(:path) - |> Path.basename() - |> Path.extname() + with {:ok, %{status: status} = response} when status in 200..299 <- Pleroma.HTTP.get(url) do + size_limit = Config.get([:mrf_steal_emoji, :size_limit], 50_000) + + if byte_size(response.body) <= size_limit do + extension = + url + |> URI.parse() + |> Map.get(:path) + |> Path.basename() + |> Path.extname() + + file_path = Path.join(emoji_dir_path, shortcode <> (extension || ".png")) - file_path = Path.join([emoji_dir_path, shortcode <> (extension || ".png")]) + case File.write(file_path, response.body) do + :ok -> + shortcode - with :ok <- File.write(file_path, response.body) do - shortcode + e -> + Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") + nil + end else - e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to write to #{file_path}: #{inspect(e)}") - nil + Logger.debug( + "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{ + size_limit + } B)" + ) + + nil end else - Logger.debug( - "MRF.StealEmojiPolicy: :#{shortcode}: at #{url} (#{byte_size(response.body)} B) over size limit (#{ - size_limit - } B)" - ) - - nil + e -> + Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") + nil end - rescue - e -> - Logger.warn("MRF.StealEmojiPolicy: Failed to fetch #{url}: #{inspect(e)}") - nil end @impl true def filter(%{"object" => %{"emoji" => foreign_emojis, "actor" => actor}} = message) do host = URI.parse(actor).host - if remote_host?(host) and accept_host?(host) do + if host != Pleroma.Web.Endpoint.host() and accept_host?(host) do installed_emoji = Pleroma.Emoji.get_all() |> Enum.map(fn {k, _} -> k end) emoji_dir_path = @@ -70,10 +71,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do new_emojis = foreign_emojis - |> Enum.filter(fn {shortcode, _url} -> shortcode not in installed_emoji end) + |> Enum.reject(fn {shortcode, _url} -> shortcode in installed_emoji end) |> Enum.filter(fn {shortcode, _url} -> reject_emoji? = - Config.get([:mrf_steal_emoji, :rejected_shortcodes], []) + [:mrf_steal_emoji, :rejected_shortcodes] + |> Config.get([]) |> Enum.find(false, fn regex -> String.match?(shortcode, regex) end) !reject_emoji? -- cgit v1.2.3 From dad76703aaf750e3811b0c963a92e39aa06b9c76 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 25 Dec 2020 11:34:09 +0300 Subject: not needed --- lib/pleroma/config/transfer_task.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index 8a6d66891..a0d7b7d71 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -28,8 +28,7 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Pleroma.Captcha, [:seconds_valid]}, {:pleroma, Pleroma.Upload, [:proxy_remote]}, {:pleroma, :instance, [:upload_limit]}, - {:pleroma, :gopher, [:enabled]}, - {:pleroma, :mrf, [:policies]} + {:pleroma, :gopher, [:enabled]} ] def start_link(restart_pleroma? \\ true) do -- cgit v1.2.3 From 2e859794ee25bdf22216587f8c8260e47b2a038f Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sun, 27 Dec 2020 21:58:15 +0300 Subject: non condition dir creation --- lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex index 0311ca433..788f21261 100644 --- a/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/steal_emoji_policy.ex @@ -65,9 +65,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do Path.join(Config.get([:instance, :static_dir]), "emoji/stolen") ) - if not Config.get([:mrf_steal_emoji, :dir_exists?], false) do - create_dir(emoji_dir_path) - end + File.mkdir_p(emoji_dir_path) new_emojis = foreign_emojis @@ -98,12 +96,4 @@ defmodule Pleroma.Web.ActivityPub.MRF.StealEmojiPolicy do def describe do {:ok, %{}} end - - defp create_dir(path) do - if not File.exists?(path) do - File.mkdir_p!(path) - end - - Config.put([:mrf_steal_emoji, :dir_exists?], true) - end end -- cgit v1.2.3