diff options
Diffstat (limited to 'lib')
25 files changed, 352 insertions, 242 deletions
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 7f1108dcf..a01c36ece 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -99,7 +99,7 @@ defmodule Mix.Tasks.Pleroma.Database do where: fragment("(?)->>'likes' is not null", object.data), select: %{id: object.id, likes: fragment("(?)->>'likes'", object.data)} ) - |> Pleroma.RepoStreamer.chunk_stream(100) + |> Pleroma.Repo.chunk_stream(100, :batches) |> Stream.each(fn objects -> ids = objects @@ -145,7 +145,7 @@ defmodule Mix.Tasks.Pleroma.Database do |> where(local: true) |> where([a], fragment("(? ->> 'type'::text) = 'Create'", a.data)) |> where([_a, o], fragment("?->>'type' = 'Note'", o.data)) - |> Pleroma.RepoStreamer.chunk_stream(100) + |> Pleroma.Repo.chunk_stream(100, :batches) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> expires_at = diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 01824aa18..b20c49d89 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -179,7 +179,7 @@ defmodule Mix.Tasks.Pleroma.User do start_pleroma() Pleroma.User.Query.build(%{nickname: "@#{instance}"}) - |> Pleroma.RepoStreamer.chunk_stream(500) + |> Pleroma.Repo.chunk_stream(500, :batches) |> Stream.each(fn users -> users |> Enum.each(fn user -> @@ -370,7 +370,7 @@ defmodule Mix.Tasks.Pleroma.User do start_pleroma() Pleroma.User.Query.build(%{local: true}) - |> Pleroma.RepoStreamer.chunk_stream(500) + |> Pleroma.Repo.chunk_stream(500, :batches) |> Stream.each(fn users -> users |> Enum.each(fn user -> diff --git a/lib/pleroma/gun/conn.ex b/lib/pleroma/gun/conn.ex index 75b1ffc0a..477e19c6e 100644 --- a/lib/pleroma/gun/conn.ex +++ b/lib/pleroma/gun/conn.ex @@ -50,10 +50,10 @@ defmodule Pleroma.Gun.Conn do with open_opts <- Map.delete(opts, :tls_opts), {:ok, conn} <- Gun.open(proxy_host, proxy_port, open_opts), - {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]), + {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]), stream <- Gun.connect(conn, connect_opts), {:response, :fin, 200, _} <- Gun.await(conn, stream) do - {:ok, conn} + {:ok, conn, protocol} else error -> Logger.warn( @@ -88,8 +88,8 @@ defmodule Pleroma.Gun.Conn do |> Map.put(:socks_opts, socks_opts) with {:ok, conn} <- Gun.open(proxy_host, proxy_port, opts), - {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do - {:ok, conn} + {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol} else error -> Logger.warn( @@ -106,8 +106,8 @@ defmodule Pleroma.Gun.Conn do host = Pleroma.HTTP.AdapterHelper.parse_host(host) with {:ok, conn} <- Gun.open(host, port, opts), - {:ok, _} <- Gun.await_up(conn, opts[:connect_timeout]) do - {:ok, conn} + {:ok, protocol} <- Gun.await_up(conn, opts[:connect_timeout]) do + {:ok, conn, protocol} else error -> Logger.warn( diff --git a/lib/pleroma/gun/connection_pool/worker.ex b/lib/pleroma/gun/connection_pool/worker.ex index c36332817..49d41e4c7 100644 --- a/lib/pleroma/gun/connection_pool/worker.ex +++ b/lib/pleroma/gun/connection_pool/worker.ex @@ -15,7 +15,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do @impl true def handle_continue({:connect, [key, uri, opts, client_pid]}, _) do - with {:ok, conn_pid} <- Gun.Conn.open(uri, opts), + with {:ok, conn_pid, protocol} <- Gun.Conn.open(uri, opts), Process.link(conn_pid) do time = :erlang.monotonic_time(:millisecond) @@ -27,8 +27,12 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do send(client_pid, {:conn_pid, conn_pid}) {:noreply, - %{key: key, timer: nil, client_monitors: %{client_pid => Process.monitor(client_pid)}}, - :hibernate} + %{ + key: key, + timer: nil, + client_monitors: %{client_pid => Process.monitor(client_pid)}, + protocol: protocol + }, :hibernate} else err -> {:stop, {:shutdown, err}, nil} @@ -53,14 +57,20 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do end @impl true - def handle_call(:add_client, {client_pid, _}, %{key: key} = state) do + def handle_call(:add_client, {client_pid, _}, %{key: key, protocol: protocol} = state) do time = :erlang.monotonic_time(:millisecond) - {{conn_pid, _, _, _}, _} = + {{conn_pid, used_by, _, _}, _} = Registry.update_value(@registry, key, fn {conn_pid, used_by, crf, last_reference} -> {conn_pid, [client_pid | used_by], crf(time - last_reference, crf), time} end) + :telemetry.execute( + [:pleroma, :connection_pool, :client, :add], + %{client_pid: client_pid, clients: used_by}, + %{key: state.key, protocol: protocol} + ) + state = if state.timer != nil do Process.cancel_timer(state[:timer]) @@ -131,7 +141,7 @@ defmodule Pleroma.Gun.ConnectionPool.Worker do @impl true def handle_info({:DOWN, _ref, :process, pid, reason}, state) do :telemetry.execute( - [:pleroma, :connection_pool, :client_death], + [:pleroma, :connection_pool, :client, :dead], %{client_pid: pid, reason: reason}, %{key: state.key} ) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index 8bf53c090..6948651c7 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -159,13 +159,11 @@ defmodule Pleroma.Instances.Instance do Pleroma.HTTP.get(to_string(instance_uri), [{"accept", "text/html"}], adapter: [pool: :media] ), - favicon_rel <- - html - |> Floki.parse_document!() - |> Floki.attribute("link[rel=icon]", "href") - |> List.first(), - favicon <- URI.merge(instance_uri, favicon_rel) |> to_string(), - true <- is_binary(favicon) do + {_, [favicon_rel | _]} when is_binary(favicon_rel) <- + {:parse, + html |> Floki.parse_document!() |> Floki.attribute("link[rel=icon]", "href")}, + {_, favicon} when is_binary(favicon) <- + {:merge, URI.merge(instance_uri, favicon_rel) |> to_string()} do favicon else _ -> nil diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex index d260e62ca..24f4733fe 100644 --- a/lib/pleroma/migration_helper/notification_backfill.ex +++ b/lib/pleroma/migration_helper/notification_backfill.ex @@ -19,13 +19,13 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do query |> Repo.chunk_stream(100) |> Enum.each(fn notification -> - type = - notification.activity - |> type_from_activity() + if notification.activity do + type = type_from_activity(notification.activity) - notification - |> Ecto.Changeset.change(%{type: type}) - |> Repo.update() + notification + |> Ecto.Changeset.change(%{type: type}) + |> Repo.update() + end end) end @@ -72,8 +72,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do "pleroma:emoji_reaction" "Create" -> - activity - |> type_from_activity_object() + type_from_activity_object(activity) t -> raise "No notification type for activity type #{t}" diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index f317e4d58..4524bd5e2 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -49,7 +49,21 @@ defmodule Pleroma.Repo do end end - def chunk_stream(query, chunk_size) do + @doc """ + Returns a lazy enumerable that emits all entries from the data store matching the given query. + + `returns_as` use to group records. use the `batches` option to fetch records in bulk. + + ## Examples + + # fetch records one-by-one + iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500) + + # fetch records in bulk + iex> Pleroma.Repo.chunk_stream(Pleroma.Activity.Queries.by_actor(ap_id), 500, :batches) + """ + @spec chunk_stream(Ecto.Query.t(), integer(), atom()) :: Enumerable.t() + def chunk_stream(query, chunk_size, returns_as \\ :one) do # We don't actually need start and end funcitons of resource streaming, # but it seems to be the only way to not fetch records one-by-one and # have individual records be the elements of the stream, instead of @@ -69,7 +83,12 @@ defmodule Pleroma.Repo do records -> last_id = List.last(records).id - {records, last_id} + + if returns_as == :one do + {records, last_id} + else + {[records], last_id} + end end end, fn _ -> :ok end diff --git a/lib/pleroma/repo_streamer.ex b/lib/pleroma/repo_streamer.ex deleted file mode 100644 index cb4d7bb7a..000000000 --- a/lib/pleroma/repo_streamer.ex +++ /dev/null @@ -1,34 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.RepoStreamer do - alias Pleroma.Repo - import Ecto.Query - - def chunk_stream(query, chunk_size) do - Stream.unfold(0, fn - :halt -> - {[], :halt} - - last_id -> - query - |> order_by(asc: :id) - |> where([r], r.id > ^last_id) - |> limit(^chunk_size) - |> Repo.all() - |> case do - [] -> - {[], :halt} - - records -> - last_id = List.last(records).id - {records, last_id} - end - end) - |> Stream.take_while(fn - [] -> false - _ -> true - end) - end -end diff --git a/lib/pleroma/telemetry/logger.ex b/lib/pleroma/telemetry/logger.ex index 4cacae02f..197b1d091 100644 --- a/lib/pleroma/telemetry/logger.ex +++ b/lib/pleroma/telemetry/logger.ex @@ -7,7 +7,8 @@ defmodule Pleroma.Telemetry.Logger do [:pleroma, :connection_pool, :reclaim, :start], [:pleroma, :connection_pool, :reclaim, :stop], [:pleroma, :connection_pool, :provision_failure], - [:pleroma, :connection_pool, :client_death] + [:pleroma, :connection_pool, :client, :dead], + [:pleroma, :connection_pool, :client, :add] ] def attach do :telemetry.attach_many("pleroma-logger", @events, &handle_event/4, []) @@ -62,7 +63,7 @@ defmodule Pleroma.Telemetry.Logger do end def handle_event( - [:pleroma, :connection_pool, :client_death], + [:pleroma, :connection_pool, :client, :dead], %{client_pid: client_pid, reason: reason}, %{key: key}, _ @@ -73,4 +74,17 @@ defmodule Pleroma.Telemetry.Logger do }" end) end + + def handle_event( + [:pleroma, :connection_pool, :client, :add], + %{clients: [_, _ | _] = clients}, + %{key: key, protocol: :http}, + _ + ) do + Logger.info(fn -> + "Pool worker for #{key}: #{length(clients)} clients are using an HTTP1 connection at the same time, head-of-line blocking might occur." + end) + end + + def handle_event([:pleroma, :connection_pool, :client, :add], _, _, _), do: :ok end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e73d19964..57497eb83 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -25,7 +25,6 @@ defmodule Pleroma.User do alias Pleroma.Object alias Pleroma.Registration alias Pleroma.Repo - alias Pleroma.RepoStreamer alias Pleroma.User alias Pleroma.UserRelationship alias Pleroma.Web @@ -1775,7 +1774,7 @@ defmodule Pleroma.User do def delete_user_activities(%User{ap_id: ap_id} = user) do ap_id |> Activity.Queries.by_actor() - |> RepoStreamer.chunk_stream(50) + |> Repo.chunk_stream(50, :batches) |> Stream.each(fn activities -> Enum.each(activities, fn activity -> delete_activity(activity, user) end) end) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index b2205bff7..92fc1e422 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -84,7 +84,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp increase_replies_count_if_reply(_create_data), do: :noop - @object_types ~w[ChatMessage Question Answer Audio Event] + @object_types ~w[ChatMessage Question Answer Audio Video Event Article] @spec persist(map(), keyword()) :: {:ok, Activity.t() | Object.t()} def persist(%{"type" => type} = object, meta) when type in @object_types do with {:ok, object} <- Object.create(object) do @@ -767,7 +767,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_replies(query, %{ - reply_filtering_user: user, + reply_filtering_user: %User{} = user, reply_visibility: "self" }) do from( @@ -783,7 +783,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end defp restrict_replies(query, %{ - reply_filtering_user: user, + reply_filtering_user: %User{} = user, reply_visibility: "following" }) do from( diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index bb193475a..161177727 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -66,7 +66,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do "type" => "Create", "object" => child_object } = object - ) do + ) + when is_map(child_object) do media_nsfw = Config.get([:mrf_simple, :media_nsfw]) |> MRF.subdomains_regex() diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index b77c06395..bd0a2a8dc 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -12,11 +12,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Activity alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object + alias Pleroma.Object.Containment alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AcceptRejectValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.AnswerValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator alias Pleroma.Web.ActivityPub.ObjectValidators.BlockValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator alias Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator @@ -149,10 +151,20 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do end end - def validate(%{"type" => "Audio"} = object, meta) do + def validate(%{"type" => type} = object, meta) when type in ~w[Audio Video] do with {:ok, object} <- object - |> AudioValidator.cast_and_validate() + |> AudioVideoValidator.cast_and_validate() + |> Ecto.Changeset.apply_action(:insert) do + object = stringify_keys(object) + {:ok, object, meta} + end + end + + def validate(%{"type" => "Article"} = object, meta) do + with {:ok, object} <- + object + |> ArticleNoteValidator.cast_and_validate() |> Ecto.Changeset.apply_action(:insert) do object = stringify_keys(object) {:ok, object, meta} @@ -198,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do %{"type" => "Create", "object" => %{"type" => objtype} = object} = create_activity, meta ) - when objtype in ~w[Question Answer Audio Event] do + when objtype in ~w[Question Answer Audio Video Event Article] do with {:ok, object_data} <- cast_and_apply(object), meta = Keyword.put(meta, :object_data, object_data |> stringify_keys), {:ok, create_activity} <- @@ -232,14 +244,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do AnswerValidator.cast_and_apply(object) end - def cast_and_apply(%{"type" => "Audio"} = object) do - AudioValidator.cast_and_apply(object) + def cast_and_apply(%{"type" => type} = object) when type in ~w[Audio Video] do + AudioVideoValidator.cast_and_apply(object) end def cast_and_apply(%{"type" => "Event"} = object) do EventValidator.cast_and_apply(object) end + def cast_and_apply(%{"type" => "Article"} = object) do + ArticleNoteValidator.cast_and_apply(object) + end + def cast_and_apply(o), do: {:error, {:validator_not_set, o}} # is_struct/1 isn't present in Elixir 1.8.x @@ -262,7 +278,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def stringify_keys(object), do: object def fetch_actor(object) do - with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do + with actor <- Containment.get_actor(object), + {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do User.get_or_fetch_by_ap_id(actor) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex index 1a97c504a..5b7dad517 100644 --- a/lib/pleroma/web/activity_pub/object_validators/audio_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/article_note_validator.ex @@ -2,7 +2,7 @@ # Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do +defmodule Pleroma.Web.ActivityPub.ObjectValidators.ArticleNoteValidator do use Ecto.Schema alias Pleroma.EctoType.ActivityPub.ObjectValidators @@ -25,14 +25,19 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do # TODO: Write type field(:tag, {:array, :map}, default: []) field(:type, :string) + + field(:name, :string) + field(:summary, :string) field(:content, :string) + field(:context, :string) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) # TODO: Remove actor on objects field(:actor, ObjectValidators.ObjectID) field(:attributedTo, ObjectValidators.ObjectID) - field(:summary, :string) field(:published, ObjectValidators.DateTime) field(:emoji, ObjectValidators.Emoji, default: %{}) field(:sensitive, :boolean, default: false) @@ -40,13 +45,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do field(:replies_count, :integer, default: 0) field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) - field(:inReplyTo, :string) + field(:inReplyTo, ObjectValidators.ObjectID) field(:url, ObjectValidators.Uri) - # short identifier for PleromaFE to group statuses by context - field(:context_id, :integer) - field(:likes, {:array, :string}, default: []) - field(:announcements, {:array, :string}, default: []) + field(:likes, {:array, ObjectValidators.ObjectID}, default: []) + field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) end def cast_and_apply(data) do @@ -62,19 +65,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do end def cast_data(data) do + data = fix(data) + %__MODULE__{} |> changeset(data) end - defp fix_url(%{"url" => url} = data) when is_list(url) do - attachment = - Enum.find(url, fn x -> is_map(x) and String.starts_with?(x["mimeType"], "audio/") end) - - link_element = Enum.find(url, fn x -> is_map(x) and x["mimeType"] == "text/html" end) - - data - |> Map.put("attachment", [attachment]) - |> Map.put("url", link_element["href"]) + defp fix_url(%{"url" => url} = data) when is_map(url) do + Map.put(data, "url", url["href"]) end defp fix_url(data), do: data @@ -83,8 +81,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do data |> CommonFixes.fix_defaults() |> CommonFixes.fix_attribution() - |> Transmogrifier.fix_emoji() + |> CommonFixes.fix_actor() |> fix_url() + |> Transmogrifier.fix_emoji() end def changeset(struct, data) do @@ -97,8 +96,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioValidator do def validate_data(data_cng) do data_cng - |> validate_inclusion(:type, ["Audio"]) - |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment]) + |> validate_inclusion(:type, ["Article", "Note"]) + |> validate_required([:id, :actor, :attributedTo, :type, :context, :context_id]) |> CommonValidations.validate_any_presence([:cc, :to]) |> CommonValidations.validate_fields_match([:actor, :attributedTo]) |> CommonValidations.validate_actor_presence() diff --git a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex index c8b148280..df102a134 100644 --- a/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/attachment_validator.ex @@ -5,6 +5,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator import Ecto.Changeset @@ -15,7 +16,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do field(:mediaType, :string, default: "application/octet-stream") field(:name, :string) - embeds_many(:url, UrlObjectValidator) + embeds_many :url, UrlObjectValidator, primary_key: false do + field(:type, :string) + field(:href, ObjectValidators.Uri) + field(:mediaType, :string, default: "application/octet-stream") + end end def cast_and_validate(data) do @@ -37,7 +42,18 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do struct |> cast(data, [:type, :mediaType, :name]) - |> cast_embed(:url, required: true) + |> cast_embed(:url, with: &url_changeset/2) + |> validate_inclusion(:type, ~w[Link Document Audio Image Video]) + |> validate_required([:type, :mediaType, :url]) + end + + def url_changeset(struct, data) do + data = fix_media_type(data) + + struct + |> cast(data, [:type, :href, :mediaType]) + |> validate_inclusion(:type, ["Link"]) + |> validate_required([:type, :href, :mediaType]) end def fix_media_type(data) do @@ -75,6 +91,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator do def validate_data(cng) do cng + |> validate_inclusion(:type, ~w[Document Audio Image Video]) |> validate_required([:mediaType, :url, :type]) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex new file mode 100644 index 000000000..16973e5db --- /dev/null +++ b/lib/pleroma/web/activity_pub/object_validators/audio_video_validator.ex @@ -0,0 +1,134 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ActivityPub.ObjectValidators.AudioVideoValidator do + use Ecto.Schema + + alias Pleroma.EarmarkRenderer + alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations + alias Pleroma.Web.ActivityPub.Transmogrifier + + import Ecto.Changeset + + @primary_key false + @derive Jason.Encoder + + embedded_schema do + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:bto, ObjectValidators.Recipients, default: []) + field(:bcc, ObjectValidators.Recipients, default: []) + # TODO: Write type + field(:tag, {:array, :map}, default: []) + field(:type, :string) + + field(:name, :string) + field(:summary, :string) + field(:content, :string) + + field(:context, :string) + # short identifier for PleromaFE to group statuses by context + field(:context_id, :integer) + + # TODO: Remove actor on objects + field(:actor, ObjectValidators.ObjectID) + + field(:attributedTo, ObjectValidators.ObjectID) + field(:published, ObjectValidators.DateTime) + field(:emoji, ObjectValidators.Emoji, default: %{}) + field(:sensitive, :boolean, default: false) + embeds_many(:attachment, AttachmentValidator) + field(:replies_count, :integer, default: 0) + field(:like_count, :integer, default: 0) + field(:announcement_count, :integer, default: 0) + field(:inReplyTo, ObjectValidators.ObjectID) + field(:url, ObjectValidators.Uri) + + field(:likes, {:array, ObjectValidators.ObjectID}, default: []) + field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) + end + + def cast_and_apply(data) do + data + |> cast_data + |> apply_action(:insert) + end + + def cast_and_validate(data) do + data + |> cast_data() + |> validate_data() + end + + def cast_data(data) do + %__MODULE__{} + |> changeset(data) + end + + defp fix_url(%{"url" => url} = data) when is_list(url) do + attachment = + Enum.find(url, fn x -> + mime_type = x["mimeType"] || x["mediaType"] || "" + + is_map(x) and String.starts_with?(mime_type, ["video/", "audio/"]) + end) + + link_element = + Enum.find(url, fn x -> + mime_type = x["mimeType"] || x["mediaType"] || "" + + is_map(x) and mime_type == "text/html" + end) + + data + |> Map.put("attachment", [attachment]) + |> Map.put("url", link_element["href"]) + end + + defp fix_url(data), do: data + + defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = data) + when is_binary(content) do + content = + content + |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer}) + |> Pleroma.HTML.filter_tags() + + Map.put(data, "content", content) + end + + defp fix_content(data), do: data + + defp fix(data) do + data + |> CommonFixes.fix_defaults() + |> CommonFixes.fix_attribution() + |> CommonFixes.fix_actor() + |> Transmogrifier.fix_emoji() + |> fix_url() + |> fix_content() + end + + def changeset(struct, data) do + data = fix(data) + + struct + |> cast(data, __schema__(:fields) -- [:attachment]) + |> cast_embed(:attachment) + end + + def validate_data(data_cng) do + data_cng + |> validate_inclusion(:type, ["Audio", "Video"]) + |> validate_required([:id, :actor, :attributedTo, :type, :context, :attachment]) + |> CommonValidations.validate_any_presence([:cc, :to]) + |> CommonValidations.validate_fields_match([:actor, :attributedTo]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_host_match() + end +end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex index 720213d73..b3638cfc7 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fixes.ex @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do + alias Pleroma.Object.Containment alias Pleroma.Web.ActivityPub.Utils # based on Pleroma.Web.ActivityPub.Utils.lazy_put_objects_defaults @@ -19,4 +20,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes do data |> Map.put_new("actor", data["attributedTo"]) end + + def fix_actor(data) do + actor = Containment.get_actor(data) + + data + |> Map.put("actor", actor) + |> Map.put("attributedTo", actor) + end end diff --git a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex index b3dbeea57..422ee07be 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_generic_validator.ex @@ -10,9 +10,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonFixes + alias Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations import Ecto.Changeset - import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @primary_key false @@ -75,14 +76,15 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateGenericValidator do data |> fix_context(meta) |> fix_addressing(meta) + |> CommonFixes.fix_actor() end def validate_data(cng, meta \\ []) do cng |> validate_required([:actor, :type, :object]) |> validate_inclusion(:type, ["Create"]) - |> validate_actor_presence() - |> validate_any_presence([:to, :cc]) + |> CommonValidations.validate_actor_presence() + |> CommonValidations.validate_any_presence([:to, :cc]) |> validate_actors_match(meta) |> validate_context_match(meta) |> validate_object_nonexistence() diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex deleted file mode 100644 index ab4469a59..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ /dev/null @@ -1,73 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do - use Ecto.Schema - - alias Pleroma.EctoType.ActivityPub.ObjectValidators - alias Pleroma.Web.ActivityPub.Transmogrifier - - import Ecto.Changeset - - @primary_key false - - embedded_schema do - field(:id, ObjectValidators.ObjectID, primary_key: true) - field(:to, ObjectValidators.Recipients, default: []) - field(:cc, ObjectValidators.Recipients, default: []) - field(:bto, ObjectValidators.Recipients, default: []) - field(:bcc, ObjectValidators.Recipients, default: []) - # TODO: Write type - field(:tag, {:array, :map}, default: []) - field(:type, :string) - - field(:name, :string) - field(:summary, :string) - field(:content, :string) - - field(:context, :string) - # short identifier for PleromaFE to group statuses by context - field(:context_id, :integer) - - field(:actor, ObjectValidators.ObjectID) - field(:attributedTo, ObjectValidators.ObjectID) - field(:published, ObjectValidators.DateTime) - field(:emoji, ObjectValidators.Emoji, default: %{}) - field(:sensitive, :boolean, default: false) - # TODO: Write type - field(:attachment, {:array, :map}, default: []) - field(:replies_count, :integer, default: 0) - field(:like_count, :integer, default: 0) - field(:announcement_count, :integer, default: 0) - field(:inReplyTo, ObjectValidators.ObjectID) - field(:url, ObjectValidators.Uri) - - field(:likes, {:array, :string}, default: []) - field(:announcements, {:array, :string}, default: []) - end - - def cast_and_validate(data) do - data - |> cast_data() - |> validate_data() - end - - defp fix(data) do - data - |> Transmogrifier.fix_emoji() - end - - def cast_data(data) do - data = fix(data) - - %__MODULE__{} - |> cast(data, __schema__(:fields)) - end - - def validate_data(data_cng) do - data_cng - |> validate_inclusion(:type, ["Note"]) - |> validate_required([:id, :actor, :to, :cc, :type, :content, :context]) - end -end diff --git a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex index 934d3c1ea..9310485dc 100644 --- a/lib/pleroma/web/activity_pub/object_validators/question_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/question_validator.ex @@ -47,8 +47,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.QuestionValidator do # short identifier for PleromaFE to group statuses by context field(:context_id, :integer) - field(:likes, {:array, :string}, default: []) - field(:announcements, {:array, :string}, default: []) + field(:likes, {:array, ObjectValidators.ObjectID}, default: []) + field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) field(:closed, ObjectValidators.DateTime) field(:voters, {:array, ObjectValidators.ObjectID}, default: []) diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex deleted file mode 100644 index 881030f38..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex +++ /dev/null @@ -1,24 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do - use Ecto.Schema - - alias Pleroma.EctoType.ActivityPub.ObjectValidators - - import Ecto.Changeset - @primary_key false - - embedded_schema do - field(:type, :string) - field(:href, ObjectValidators.Uri) - field(:mediaType, :string, default: "application/octet-stream") - end - - def changeset(struct, data) do - struct - |> cast(data, __schema__(:fields)) - |> validate_required([:type, :href, :mediaType]) - end -end diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 46a8be767..b9a83a544 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -336,7 +336,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end def handle_object_creation(%{"type" => objtype} = object, meta) - when objtype in ~w[Audio Question Event] do + when objtype in ~w[Audio Video Question Event Article] do with {:ok, object, meta} <- Pipeline.common_pipeline(object, meta) do {:ok, object, meta} end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index af4384213..fcca014f0 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -7,7 +7,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do A module to handle coding from internal to wire ActivityPub and back. """ alias Pleroma.Activity - alias Pleroma.EarmarkRenderer alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Maps alias Pleroma.Object @@ -45,7 +44,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_addressing |> fix_summary |> fix_type(options) - |> fix_content end def fix_summary(%{"summary" => nil} = object) do @@ -274,24 +272,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do Map.put(object, "url", url["href"]) end - def fix_url(%{"type" => "Video", "url" => url} = object) when is_list(url) do - attachment = - Enum.find(url, fn x -> - media_type = x["mediaType"] || x["mimeType"] || "" - - is_map(x) and String.starts_with?(media_type, "video/") - end) - - link_element = - Enum.find(url, fn x -> is_map(x) and (x["mediaType"] || x["mimeType"]) == "text/html" end) - - object - |> Map.put("attachment", [attachment]) - |> Map.put("url", link_element["href"]) - end - - def fix_url(%{"type" => object_type, "url" => url} = object) - when object_type != "Video" and is_list(url) do + def fix_url(%{"url" => url} = object) when is_list(url) do first_element = Enum.at(url, 0) url_string = @@ -309,7 +290,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_emoji(%{"tag" => tags} = object) when is_list(tags) do emoji = tags - |> Enum.filter(fn data -> data["type"] == "Emoji" and data["icon"] end) + |> Enum.filter(fn data -> is_map(data) and data["type"] == "Emoji" and data["icon"] end) |> Enum.reduce(%{}, fn data, mapping -> name = String.trim(data["name"], ":") @@ -371,18 +352,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_type(object, _), do: object - defp fix_content(%{"mediaType" => "text/markdown", "content" => content} = object) - when is_binary(content) do - html_content = - content - |> Earmark.as_html!(%Earmark.Options{renderer: EarmarkRenderer}) - |> Pleroma.HTML.filter_tags() - - Map.merge(object, %{"content" => html_content, "mediaType" => "text/html"}) - end - - defp fix_content(object), do: object - # Reduce the object list to find the reported user. defp get_reported(objects) do Enum.reduce_while(objects, nil, fn ap_id, _ -> @@ -455,7 +424,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => "Create", "object" => %{"type" => objtype} = object} = data, options ) - when objtype in ~w{Article Note Video Page} do + when objtype in ~w{Note Page} do actor = Containment.get_actor(data) with nil <- Activity.get_create_by_object_ap_id(object["id"]), @@ -549,7 +518,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do %{"type" => "Create", "object" => %{"type" => objtype}} = data, _options ) - when objtype in ~w{Question Answer ChatMessage Audio Event} do + when objtype in ~w{Question Answer ChatMessage Audio Video Event Article} do + data = Map.put(data, "object", strip_internal_fields(data["object"])) + with {:ok, %User{}} <- ObjectValidator.fetch_actor(data), {:ok, activity, _} <- Pipeline.common_pipeline(data, local: false) do {:ok, activity} diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index bd7f03cbe..d7a19df4a 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -87,6 +87,50 @@ defmodule Pleroma.Web.RichMedia.Helpers do def rich_media_get(url) do headers = [{"user-agent", Pleroma.Application.user_agent() <> "; Bot"}] - Pleroma.HTTP.get(url, headers, @options) + head_check = + case Pleroma.HTTP.head(url, headers, @options) do + # If the HEAD request didn't reach the server for whatever reason, + # we assume the GET that comes right after won't either + {:error, _} = e -> + e + + {:ok, %Tesla.Env{status: 200, headers: headers}} -> + with :ok <- check_content_type(headers), + :ok <- check_content_length(headers), + do: :ok + + _ -> + :ok + end + + with :ok <- head_check, do: Pleroma.HTTP.get(url, headers, @options) + end + + defp check_content_type(headers) do + case List.keyfind(headers, "content-type", 0) do + {_, content_type} -> + case Plug.Conn.Utils.media_type(content_type) do + {:ok, "text", "html", _} -> :ok + _ -> {:error, {:content_type, content_type}} + end + + _ -> + :ok + end + end + + @max_body @options[:max_body] + defp check_content_length(headers) do + case List.keyfind(headers, "content-length", 0) do + {_, maybe_content_length} -> + case Integer.parse(maybe_content_length) do + {content_length, ""} when content_length <= @max_body -> :ok + {_, ""} -> {:error, :body_too_large} + _ -> :ok + end + + _ -> + :ok + end end end diff --git a/lib/pleroma/web/rich_media/parser.ex b/lib/pleroma/web/rich_media/parser.ex index 5727fda18..33f6f1fa1 100644 --- a/lib/pleroma/web/rich_media/parser.ex +++ b/lib/pleroma/web/rich_media/parser.ex @@ -36,6 +36,14 @@ defmodule Pleroma.Web.RichMedia.Parser do {:ok, _data} = res -> res + {:error, :body_too_large} = e -> + e + + {:error, {:content_type, _}} = e -> + e + + # The TTL is not set for the errors above, since they are unlikely to change + # with time {:error, _} = e -> ttl = Pleroma.Config.get([:rich_media, :failure_backoff], 60_000) Cachex.expire(:rich_media_cache, url, ttl) |