summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/mix/tasks/pleroma/database.ex4
-rw-r--r--lib/mix/tasks/pleroma/user.ex4
-rw-r--r--lib/pleroma/gun/conn.ex12
-rw-r--r--lib/pleroma/gun/connection_pool/worker.ex22
-rw-r--r--lib/pleroma/instances/instance.ex12
-rw-r--r--lib/pleroma/migration_helper/notification_backfill.ex15
-rw-r--r--lib/pleroma/repo.ex23
-rw-r--r--lib/pleroma/repo_streamer.ex34
-rw-r--r--lib/pleroma/telemetry/logger.ex18
-rw-r--r--lib/pleroma/user.ex3
-rw-r--r--lib/pleroma/web/activity_pub/activity_pub.ex4
-rw-r--r--lib/pleroma/web/activity_pub/mrf/simple_policy.ex3
-rw-r--r--lib/pleroma/web/activity_pub/transmogrifier.ex2
-rw-r--r--lib/pleroma/web/rich_media/helpers.ex46
-rw-r--r--lib/pleroma/web/rich_media/parser.ex8
15 files changed, 134 insertions, 76 deletions
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 7f1108dcf..a01c36ece 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -99,7 +99,7 @@ defmodule Mix.Tasks.Pleroma.Database do
where: fragment("(?)->>'likes' is not null", object.data),
select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)}
)
- |> Pleroma.RepoStreamer.chunk_stream(100)
+ |> Pleroma.Repo.chunk_stream(100, :batches)
|> Stream.each(fn objects ->
ids =
objects
@@ -145,7 +145,7 @@ defmodule Mix.Tasks.Pleroma.Database do
|> where(local: true)
|> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data))
|> where([_a, o], fragment("?->>'type' = 'Note'", o.data))
- |> Pleroma.RepoStreamer.chunk_stream(100)
+ |> Pleroma.Repo.chunk_stream(100, :batches)
|> Stream.each(fn activities ->
Enum.each(activities, fn activity ->
expires_at =
diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex
index 01824aa18..b20c49d89 100644
--- a/lib/mix/tasks/pleroma/user.ex
+++ b/lib/mix/tasks/pleroma/user.ex
@@ -179,7 +179,7 @@ defmodule Mix.Tasks.Pleroma.User do
start_pleroma()
Pleroma.User.Query.build(%{nickname: "@#{instance}"})
- |> Pleroma.RepoStreamer.chunk_stream(500)
+ |> Pleroma.Repo.chunk_stream(500, :batches)
|> Stream.each(fn users ->
users
|> Enum.each(fn user ->
@@ -370,7 +370,7 @@ defmodule Mix.Tasks.Pleroma.User do
start_pleroma()
Pleroma.User.Query.build(%{local: true})
- |> Pleroma.RepoStreamer.chunk_stream(500)
+ |> Pleroma.Repo.chunk_stream(500, :batches)
|> Stream.each(fn users ->
users
|> Enum.each(fn user ->
diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex
index 75b1ffc0a..477e19c6e 100644
--- a/lib/pleroma/gun/conn.ex
+++ b/lib/pleroma/gun/conn.ex
@@ -50,10 +50,10 @@ defmodule Pleroma.Gun.Conn do
with open_opts <- Map.delete(opts, :tls_opts),
{:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts),
- {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]),
+ {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]),
stream <- Gun.connect(conn, connect_opts),
{:response, :fin, 200, _} <- Gun.await(conn, stream) do
- {:ok, conn}
+ {:ok, conn, protocol}
else
error ->
Logger.warn(
@@ -88,8 +88,8 @@ defmodule Pleroma.Gun.Conn do
|> Map.put(:socks_opts, socks_opts)
with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts),
- {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
- {:ok, conn}
+ {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
+ {:ok, conn, protocol}
else
error ->
Logger.warn(
@@ -106,8 +106,8 @@ defmodule Pleroma.Gun.Conn do
host = Pleroma.HTTP.AdapterHelper.parse_host(host)
with {:ok, conn} <- Gun.open(host, port, opts),
- {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do
- {:ok, conn}
+ {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do
+ {:ok, conn, protocol}
else
error ->
Logger.warn(
diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex
index c36332817..49d41e4c7 100644
--- a/lib/pleroma/gun/connection_pool/worker.ex
+++ b/lib/pleroma/gun/connection_pool/worker.ex
@@ -15,7 +15,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
@impl true
def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do
- with {:ok, conn_pid} <- Gun.Conn.open(uri, opts),
+ with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts),
Process.link(conn_pid) do
time = :erlang.monotonic_time(:millisecond)
@@ -27,8 +27,12 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
send(client_pid, {:conn_pid, conn_pid})
{:noreply,
- %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}},
- :hibernate}
+ %{
+ key: key,
+ timer: nil,
+ client_monitors: %{client_pid => Process.monitor(client_pid)},
+ protocol: protocol
+ }, :hibernate}
else
err ->
{:stop, {:shutdown, err}, nil}
@@ -53,14 +57,20 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
end
@impl true
- def handle_call(:add_client, {client_pid, _}, %{key: key} = state) do
+ def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do
time = :erlang.monotonic_time(:millisecond)
- {{conn_pid, _, _, _}, _} =
+ {{conn_pid, used_by, _, _}, _} =
Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} ->
{conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time}
end)
+ :telemetry.execute(
+ [:pleroma, :connection_pool, :client, :add],
+ %{client_pid: client_pid, clients: used_by},
+ %{key: state.key, protocol: protocol}
+ )
+
state =
if state.timer != nil do
Process.cancel_timer(state[:timer])
@@ -131,7 +141,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do
@impl true
def handle_info({:DOWN, _ref, :process, pid, reason}, state) do
:telemetry.execute(
- [:pleroma, :connection_pool, :client_death],
+ [:pleroma, :connection_pool, :client, :dead],
%{client_pid: pid, reason: reason},
%{key: state.key}
)
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index 8bf53c090..6948651c7 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -159,13 +159,11 @@ defmodule Pleroma.Instances.Instance do
Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}],
adapter: [pool: :media]
),
- favicon_rel <-
- html
- |> Floki.parse_document!()
- |> Floki.attribute("link[rel=icon]", "href")
- |> List.first(),
- favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(),
- true <- is_binary(favicon) do
+ {_, [favicon_rel | _]} when is_binary(favicon_rel) <-
+ {:parse,
+ html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")},
+ {_, favicon} when is_binary(favicon) <-
+ {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do
favicon
else
_ -> nil
diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex
index d260e62ca..24f4733fe 100644
--- a/lib/pleroma/migration_helper/notification_backfill.ex
+++ b/lib/pleroma/migration_helper/notification_backfill.ex
@@ -19,13 +19,13 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
query
|> Repo.chunk_stream(100)
|> Enum.each(fn notification ->
- type =
- notification.activity
- |> type_from_activity()
+ if notification.activity do
+ type = type_from_activity(notification.activity)
- notification
- |> Ecto.Changeset.change(%{type: type})
- |> Repo.update()
+ notification
+ |> Ecto.Changeset.change(%{type: type})
+ |> Repo.update()
+ end
end)
end
@@ -72,8 +72,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do
"pleroma:emoji_reaction"
"Create" ->
- activity
- |> type_from_activity_object()
+ type_from_activity_object(activity)
t ->
raise "No notification type for activity type #{t}"
diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex
index f317e4d58..4524bd5e2 100644
--- a/lib/pleroma/repo.ex
+++ b/lib/pleroma/repo.ex
@@ -49,7 +49,21 @@ defmodule Pleroma.Repo do
end
end
- def chunk_stream(query, chunk_size) do
+ @doc """
+ Returns a lazy enumerable that emits all entries from the data store matching the given query.
+
+ `returns_as` use to group records. use the `batches` option to fetch records in bulk.
+
+ ## Examples
+
+ # fetch records one-by-one
+ iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500)
+
+ # fetch records in bulk
+ iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches)
+ """
+ @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t()
+ def chunk_stream(query, chunk_size, returns_as \\ :one) do
# We don't actually need start and end funcitons of resource streaming,
# but it seems to be the only way to not fetch records one-by-one and
# have individual records be the elements of the stream, instead of
@@ -69,7 +83,12 @@ defmodule Pleroma.Repo do
records ->
last_id = List.last(records).id
- {records, last_id}
+
+ if returns_as == :one do
+ {records, last_id}
+ else
+ {[records], last_id}
+ end
end
end,
fn _ -> :ok end
diff --git a/lib/pleroma/repo_streamer.ex b/lib/pleroma/repo_streamer.ex
deleted file mode 100644
index cb4d7bb7a..000000000
--- a/lib/pleroma/repo_streamer.ex
+++ /dev/null
@@ -1,34 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.RepoStreamer do
- alias Pleroma.Repo
- import Ecto.Query
-
- def chunk_stream(query, chunk_size) do
- Stream.unfold(0, fn
- :halt ->
- {[], :halt}
-
- last_id ->
- query
- |> order_by(asc: :id)
- |> where([r], r.id > ^last_id)
- |> limit(^chunk_size)
- |> Repo.all()
- |> case do
- [] ->
- {[], :halt}
-
- records ->
- last_id = List.last(records).id
- {records, last_id}
- end
- end)
- |> Stream.take_while(fn
- [] -> false
- _ -> true
- end)
- end
-end
diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex
index 4cacae02f..197b1d091 100644
--- a/lib/pleroma/telemetry/logger.ex
+++ b/lib/pleroma/telemetry/logger.ex
@@ -7,7 +7,8 @@ defmodule Pleroma.Telemetry.Logger do
[:pleroma, :connection_pool, :reclaim, :start],
[:pleroma, :connection_pool, :reclaim, :stop],
[:pleroma, :connection_pool, :provision_failure],
- [:pleroma, :connection_pool, :client_death]
+ [:pleroma, :connection_pool, :client, :dead],
+ [:pleroma, :connection_pool, :client, :add]
]
def attach do
:telemetry.attach_many("pleroma-logger", @events, &handle_event/4, [])
@@ -62,7 +63,7 @@ defmodule Pleroma.Telemetry.Logger do
end
def handle_event(
- [:pleroma, :connection_pool, :client_death],
+ [:pleroma, :connection_pool, :client, :dead],
%{client_pid: client_pid, reason: reason},
%{key: key},
_
@@ -73,4 +74,17 @@ defmodule Pleroma.Telemetry.Logger do
}"
end)
end
+
+ def handle_event(
+ [:pleroma, :connection_pool, :client, :add],
+ %{clients: [_, _ | _] = clients},
+ %{key: key, protocol: :http},
+ _
+ ) do
+ Logger.info(fn ->
+ "Pool worker for #{key}: #{length(clients)} clients are using an HTTP1 connection at the same time, head-of-line blocking might occur."
+ end)
+ end
+
+ def handle_event([:pleroma, :connection_pool, :client, :add], _, _, _), do: :ok
end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index e73d19964..57497eb83 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -25,7 +25,6 @@ defmodule Pleroma.User do
alias Pleroma.Object
alias Pleroma.Registration
alias Pleroma.Repo
- alias Pleroma.RepoStreamer
alias Pleroma.User
alias Pleroma.UserRelationship
alias Pleroma.Web
@@ -1775,7 +1774,7 @@ defmodule Pleroma.User do
def delete_user_activities(%User{ap_id: ap_id} = user) do
ap_id
|> Activity.Queries.by_actor()
- |> RepoStreamer.chunk_stream(50)
+ |> Repo.chunk_stream(50, :batches)
|> Stream.each(fn activities ->
Enum.each(activities, fn activity -> delete_activity(activity, user) end)
end)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 3ab045737..f5cbb3e01 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -767,7 +767,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
defp restrict_replies(query, %{
- reply_filtering_user: user,
+ reply_filtering_user: %User{} = user,
reply_visibility: "self"
}) do
from(
@@ -783,7 +783,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
defp restrict_replies(query, %{
- reply_filtering_user: user,
+ reply_filtering_user: %User{} = user,
reply_visibility: "following"
}) do
from(
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index bb193475a..161177727 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -66,7 +66,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
"type" => "Create",
"object" => child_object
} = object
- ) do
+ )
+ when is_map(child_object) do
media_nsfw =
Config.get([:mrf_simple, :media_nsfw])
|> MRF.subdomains_regex()
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 80f529704..fcca014f0 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -290,7 +290,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do
emoji =
tags
- |> Enum.filter(fn data -> data["type"] == "Emoji" and data["icon"] end)
+ |> Enum.filter(fn data -> is_map(data) and data["type"] == "Emoji" and data["icon"] end)
|> Enum.reduce(%{}, fn data, mapping ->
name = String.trim(data["name"], ":")
diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex
index bd7f03cbe..d7a19df4a 100644
--- a/lib/pleroma/web/rich_media/helpers.ex
+++ b/lib/pleroma/web/rich_media/helpers.ex
@@ -87,6 +87,50 @@ defmodule Pleroma.Web.RichMedia.Helpers do
def rich_media_get(url) do
headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}]
- Pleroma.HTTP.get(url, headers, @options)
+ head_check =
+ case Pleroma.HTTP.head(url, headers, @options) do
+ # If the HEAD request didn't reach the server for whatever reason,
+ # we assume the GET that comes right after won't either
+ {:error, _} = e ->
+ e
+
+ {:ok, %Tesla.Env{status: 200, headers: headers}} ->
+ with :ok <- check_content_type(headers),
+ :ok <- check_content_length(headers),
+ do: :ok
+
+ _ ->
+ :ok
+ end
+
+ with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options)
+ end
+
+ defp check_content_type(headers) do
+ case List.keyfind(headers, "content-type", 0) do
+ {_, content_type} ->
+ case Plug.Conn.Utils.media_type(content_type) do
+ {:ok, "text", "html", _} -> :ok
+ _ -> {:error, {:content_type, content_type}}
+ end
+
+ _ ->
+ :ok
+ end
+ end
+
+ @max_body @options[:max_body]
+ defp check_content_length(headers) do
+ case List.keyfind(headers, "content-length", 0) do
+ {_, maybe_content_length} ->
+ case Integer.parse(maybe_content_length) do
+ {content_length, ""} when content_length <= @max_body -> :ok
+ {_, ""} -> {:error, :body_too_large}
+ _ -> :ok
+ end
+
+ _ ->
+ :ok
+ end
end
end
diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex
index 5727fda18..33f6f1fa1 100644
--- a/lib/pleroma/web/rich_media/parser.ex
+++ b/lib/pleroma/web/rich_media/parser.ex
@@ -36,6 +36,14 @@ defmodule Pleroma.Web.RichMedia.Parser do
{:ok, _data} = res ->
res
+ {:error, :body_too_large} = e ->
+ e
+
+ {:error, {:content_type, _}} = e ->
+ e
+
+ # The TTL is not set for the errors above, since they are unlikely to change
+ # with time
{:error, _} = e ->
ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000)
Cachex.expire(:rich_media_cache, url, ttl)
generated by cgit v1.2.3 (git 2.25.1) at 2025-05-20 01:38:35 +0000