From 2ee8f4f0629b950349409d69395015369c45b633 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Tue, 23 Jul 2024 09:43:41 -0400 Subject: Fix dialyzer error lib/pleroma/application.ex:1:pattern_match The pattern can never match the type true. --- lib/pleroma/application.ex | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index d98293e8e..cb15dc1e9 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -14,7 +14,6 @@ defmodule Pleroma.Application do @name Mix.Project.config()[:name] @version Mix.Project.config()[:version] @repository Mix.Project.config()[:source_url] - @compile_env Mix.env() def name, do: @name def version, do: @version @@ -53,7 +52,7 @@ defmodule Pleroma.Application do Pleroma.Config.Oban.warn() Config.DeprecationWarnings.warn() - if @compile_env != :test do + if Config.get([Pleroma.Web.Plugs.HTTPSecurityPlug, :enable], true) do Pleroma.Web.Plugs.HTTPSecurityPlug.warn_if_disabled() end -- cgit v1.2.3 From 858fd01c012a48928b55999e8209371a5049c3e6 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 24 Jul 2024 15:40:15 -0400 Subject: Pleroma.HTTP: permit passing through custom Tesla Middlware for requests --- lib/pleroma/http.ex | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/http.ex b/lib/pleroma/http.ex index ec837e509..c11317850 100644 --- a/lib/pleroma/http.ex +++ b/lib/pleroma/http.ex @@ -68,7 +68,9 @@ defmodule Pleroma.HTTP do adapter = Application.get_env(:tesla, :adapter) - client = Tesla.client(adapter_middlewares(adapter), adapter) + extra_middleware = options[:tesla_middleware] || [] + + client = Tesla.client(adapter_middlewares(adapter, extra_middleware), adapter) maybe_limit( fn -> @@ -102,20 +104,21 @@ defmodule Pleroma.HTTP do fun.() end - defp adapter_middlewares(Tesla.Adapter.Gun) do - [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] + defp adapter_middlewares(Tesla.Adapter.Gun, extra_middleware) do + [Tesla.Middleware.FollowRedirects, Pleroma.Tesla.Middleware.ConnectionPool] ++ + extra_middleware end - defp adapter_middlewares({Tesla.Adapter.Finch, _}) do - [Tesla.Middleware.FollowRedirects] + defp adapter_middlewares({Tesla.Adapter.Finch, _}, extra_middleware) do + [Tesla.Middleware.FollowRedirects] ++ extra_middleware end - defp adapter_middlewares(_) do + defp adapter_middlewares(_, extra_middleware) do if Pleroma.Config.get(:env) == :test do # Emulate redirects in test env, which are handled by adapters in other environments [Tesla.Middleware.FollowRedirects] else - [] + extra_middleware end end end -- cgit v1.2.3 From 731f7b87d24bd58d5349c7d1f564cd73dbf59aa9 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 24 Jul 2024 15:42:50 -0400 Subject: Pad RichMediaWorker timeout to be 2s longer than the Rich Media HTTP timeout --- lib/pleroma/web/rich_media/helpers.ex | 5 ++++- lib/pleroma/workers/rich_media_worker.ex | 10 +++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index fba23c657..e2889b351 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -69,9 +69,12 @@ defmodule Pleroma.Web.RichMedia.Helpers do end defp http_options do + timeout = Config.get!([:rich_media, :timeout]) + [ pool: :rich_media, - max_body: Config.get([:rich_media, :max_body], 5_000_000) + max_body: Config.get([:rich_media, :max_body], 5_000_000), + tesla_middleware: [{Tesla.Middleware.Timeout, timeout: timeout}] ] end end diff --git a/lib/pleroma/workers/rich_media_worker.ex b/lib/pleroma/workers/rich_media_worker.ex index 30f9d9e9e..0a1c6d58c 100644 --- a/lib/pleroma/workers/rich_media_worker.ex +++ b/lib/pleroma/workers/rich_media_worker.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Workers.RichMediaWorker do + alias Pleroma.Config alias Pleroma.Web.RichMedia.Backfill alias Pleroma.Web.RichMedia.Card @@ -31,6 +32,13 @@ defmodule Pleroma.Workers.RichMediaWorker do end end + # There is timeout value enforced by Tesla.Middleware.Timeout + # which can be found in the RichMedia.Helpers module to allow us to detect + # a slow/infinite data stream and insert a negative cache entry for the URL + # We pad it by 2 seconds to be certain a slow connection is detected and we + # can inject a negative cache entry for the URL @impl Oban.Worker - def timeout(_job), do: :timer.seconds(5) + def timeout(_job) do + Config.get!([:rich_media, :timeout]) + :timer.seconds(2) + end end -- cgit v1.2.3 From 5a62868106465bd30be11922b4ff3b11b3c174aa Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 24 Jul 2024 15:43:45 -0400 Subject: Consider errors during HTTP GET and HEAD to be unrecoverable and insert a negative cache entry This is for a normal HTTP error response or timeout while receiving the data. A hard error from a process crash, DNS lookup failure, etc should produce a different response than {:ok, %Tesla.Env{}} and the request/job will be retryable. --- lib/pleroma/web/rich_media/backfill.ex | 6 +----- lib/pleroma/workers/rich_media_worker.ex | 6 +----- 2 files changed, 2 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index a66422e71..cf638fa29 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -36,13 +36,9 @@ defmodule Pleroma.Web.RichMedia.Backfill do :ok {:error, type} = error - when type in [:invalid_metadata, :body_too_large, :content_type, :validate] -> + when type in [:invalid_metadata, :body_too_large, :content_type, :validate, :get, :head] -> negative_cache(url_hash) error - - {:error, type} = error - when type in [:get, :head] -> - error end end diff --git a/lib/pleroma/workers/rich_media_worker.ex b/lib/pleroma/workers/rich_media_worker.ex index 0a1c6d58c..2ebf42d4f 100644 --- a/lib/pleroma/workers/rich_media_worker.ex +++ b/lib/pleroma/workers/rich_media_worker.ex @@ -20,13 +20,9 @@ defmodule Pleroma.Workers.RichMediaWorker do :ok {:error, type} - when type in [:invalid_metadata, :body_too_large, :content_type, :validate] -> + when type in [:invalid_metadata, :body_too_large, :content_type, :validate, :get, :head] -> {:cancel, type} - {:error, type} - when type in [:get, :head] -> - {:error, type} - error -> {:error, error} end -- cgit v1.2.3 From 97d488aea3ce75e52d4e6ba9a3b5e4447b535879 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 24 Jul 2024 15:45:35 -0400 Subject: Fix RichMedia negative cache entries The negative cache entry was a nil value, but that is an expected response when the cache is missing an entry so it didn't work as intended. --- lib/pleroma/web/rich_media/backfill.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/backfill.ex b/lib/pleroma/web/rich_media/backfill.ex index cf638fa29..1cd90629f 100644 --- a/lib/pleroma/web/rich_media/backfill.ex +++ b/lib/pleroma/web/rich_media/backfill.ex @@ -64,5 +64,5 @@ defmodule Pleroma.Web.RichMedia.Backfill do defp warm_cache(key, val), do: @cachex.put(:rich_media_cache, key, val) defp negative_cache(key, ttl \\ :timer.minutes(15)), - do: @cachex.put(:rich_media_cache, key, nil, ttl: ttl) + do: @cachex.put(:rich_media_cache, key, :error, ttl: ttl) end -- cgit v1.2.3