diff options
Diffstat (limited to 'lib')
25 files changed, 429 insertions, 157 deletions
diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex index a4890b489..b346b39d6 100644 --- a/lib/pleroma/ecto_enums.ex +++ b/lib/pleroma/ecto_enums.ex @@ -27,3 +27,11 @@ defenum(Pleroma.DataMigration.State, failed: 4, manual: 5 ) + +defenum(Pleroma.User.Backup.State, + pending: 1, + running: 2, + complete: 3, + failed: 4, + invalid: 5 +) diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex new file mode 100644 index 000000000..1038296e7 --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/bare_uri.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.BareUri do + use Ecto.Type + + def type, do: :string + + def cast(uri) when is_binary(uri) do + case URI.parse(uri) do + %URI{scheme: nil} -> :error + %URI{} -> {:ok, uri} + _ -> :error + end + end + + def cast(_), do: :error + + def dump(data), do: {:ok, data} + + def load(data), do: {:ok, data} +end diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index a5529ad44..9756c66dc 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Instances.Instance do alias Pleroma.Instances alias Pleroma.Instances.Instance + alias Pleroma.Maps alias Pleroma.Repo alias Pleroma.User alias Pleroma.Workers.BackgroundWorker @@ -24,6 +25,14 @@ defmodule Pleroma.Instances.Instance do field(:favicon, :string) field(:favicon_updated_at, :naive_datetime) + embeds_one :metadata, Pleroma.Instances.Metadata, primary_key: false do + field(:software_name, :string) + field(:software_version, :string) + field(:software_repository, :string) + end + + field(:metadata_updated_at, :utc_datetime) + timestamps() end @@ -31,11 +40,17 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) + |> cast(params, __schema__(:fields) -- [:metadata]) + |> cast_embed(:metadata, with: &metadata_changeset/2) |> validate_required([:host]) |> unique_constraint(:host) end + def metadata_changeset(struct, params \\ %{}) do + struct + |> cast(params, [:software_name, :software_version, :software_repository]) + end + def filter_reachable([]), do: %{} def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do @@ -198,6 +213,89 @@ defmodule Pleroma.Instances.Instance do end end + def get_or_update_metadata(%URI{host: host} = instance_uri) do + existing_record = Repo.get_by(Instance, %{host: host}) + now = NaiveDateTime.utc_now() + + if existing_record && existing_record.metadata_updated_at && + NaiveDateTime.diff(now, existing_record.metadata_updated_at) < 86_400 do + existing_record.metadata + else + metadata = scrape_metadata(instance_uri) + + if existing_record do + existing_record + |> changeset(%{metadata: metadata, metadata_updated_at: now}) + |> Repo.update() + else + %Instance{} + |> changeset(%{host: host, metadata: metadata, metadata_updated_at: now}) + |> Repo.insert() + end + + metadata + end + end + + defp get_nodeinfo_uri(well_known) do + links = Map.get(well_known, "links", []) + + nodeinfo21 = + Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.1"))["href"] + + nodeinfo20 = + Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))["href"] + + cond do + is_binary(nodeinfo21) -> {:ok, nodeinfo21} + is_binary(nodeinfo20) -> {:ok, nodeinfo20} + true -> {:error, :no_links} + end + end + + defp scrape_metadata(%URI{} = instance_uri) do + try do + with {_, true} <- {:reachable, reachable?(instance_uri.host)}, + {:ok, %Tesla.Env{body: well_known_body}} <- + instance_uri + |> URI.merge("/.well-known/nodeinfo") + |> to_string() + |> Pleroma.HTTP.get([{"accept", "application/json"}]), + {:ok, well_known_json} <- Jason.decode(well_known_body), + {:ok, nodeinfo_uri} <- get_nodeinfo_uri(well_known_json), + {:ok, %Tesla.Env{body: nodeinfo_body}} <- + Pleroma.HTTP.get(nodeinfo_uri, [{"accept", "application/json"}]), + {:ok, nodeinfo} <- Jason.decode(nodeinfo_body) do + # Can extract more metadata from NodeInfo but need to be careful about it's size, + # can't just dump the entire thing + software = Map.get(nodeinfo, "software", %{}) + + %{ + software_name: software["name"], + software_version: software["version"] + } + |> Maps.put_if_present(:software_repository, software["repository"]) + else + {:reachable, false} -> + Logger.debug( + "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") ignored unreachable host" + ) + + nil + + _ -> + nil + end + rescue + e -> + Logger.warn( + "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" + ) + + nil + end + end + @doc """ Deletes all users from an instance in a background task, thus also deleting all of those users' activities and notifications. diff --git a/lib/pleroma/upload/filter.ex b/lib/pleroma/upload/filter.ex index 717f06621..809bc6e70 100644 --- a/lib/pleroma/upload/filter.ex +++ b/lib/pleroma/upload/filter.ex @@ -38,9 +38,9 @@ defmodule Pleroma.Upload.Filter do {:ok, :noop} -> filter(rest, upload) - error -> - Logger.error("#{__MODULE__}: Filter #{filter} failed: #{inspect(error)}") - error + {:error, e} -> + Logger.error("#{__MODULE__}: Filter #{filter} failed: #{inspect(e)}") + {:error, e} end end end diff --git a/lib/pleroma/upload/filter/only_media.ex b/lib/pleroma/upload/filter/only_media.ex new file mode 100644 index 000000000..a9caeba67 --- /dev/null +++ b/lib/pleroma/upload/filter/only_media.ex @@ -0,0 +1,20 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Upload.Filter.OnlyMedia do + @behaviour Pleroma.Upload.Filter + alias Pleroma.Upload + + def filter(%Upload{content_type: content_type}) do + [type, _subtype] = String.split(content_type, "/") + + if type in ["image", "video", "audio"] do + {:ok, :noop} + else + {:error, "Disallowed content-type: #{content_type}"} + end + end + + def filter(_), do: {:ok, :noop} +end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f6e30555c..ce125d608 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -124,7 +124,6 @@ defmodule Pleroma.User do field(:domain_blocks, {:array, :string}, default: []) field(:is_active, :boolean, default: true) field(:no_rich_text, :boolean, default: false) - field(:ap_enabled, :boolean, default: false) field(:is_moderator, :boolean, default: false) field(:is_admin, :boolean, default: false) field(:show_role, :boolean, default: true) @@ -488,7 +487,6 @@ defmodule Pleroma.User do :nickname, :public_key, :avatar, - :ap_enabled, :banner, :is_locked, :last_refreshed_at, @@ -1061,11 +1059,7 @@ defmodule Pleroma.User do end def maybe_direct_follow(%User{} = follower, %User{} = followed) do - if not ap_enabled?(followed) do - follow(follower, followed) - else - {:ok, follower, followed} - end + {:ok, follower, followed} end @doc "A mass follow for local users. Respects blocks in both directions but does not create activities." @@ -1898,7 +1892,6 @@ defmodule Pleroma.User do confirmation_token: nil, domain_blocks: [], is_active: false, - ap_enabled: false, is_moderator: false, is_admin: false, mascot: nil, @@ -2151,10 +2144,6 @@ defmodule Pleroma.User do end end - def ap_enabled?(%User{local: true}), do: true - def ap_enabled?(%User{ap_enabled: ap_enabled}), do: ap_enabled - def ap_enabled?(_), do: false - @doc "Gets or fetch a user by uri or nickname." @spec get_or_fetch(String.t()) :: {:ok, User.t()} | {:error, String.t()} def get_or_fetch("http://" <> _host = uri), do: get_or_fetch_by_ap_id(uri) diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 9df010605..447fca2a1 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -9,12 +9,14 @@ defmodule Pleroma.User.Backup do import Ecto.Query import Pleroma.Web.Gettext + require Logger require Pleroma.Constants alias Pleroma.Activity alias Pleroma.Bookmark alias Pleroma.Repo alias Pleroma.User + alias Pleroma.User.Backup.State alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView @@ -25,6 +27,8 @@ defmodule Pleroma.User.Backup do field(:file_name, :string) field(:file_size, :integer, default: 0) field(:processed, :boolean, default: false) + field(:state, State, default: :invalid) + field(:processed_number, :integer, default: 0) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) @@ -46,7 +50,8 @@ defmodule Pleroma.User.Backup do %__MODULE__{ user_id: user.id, content_type: "application/zip", - file_name: name + file_name: name, + state: :pending } end @@ -109,27 +114,108 @@ defmodule Pleroma.User.Backup do def get(id), do: Repo.get(__MODULE__, id) + defp set_state(backup, state, processed_number \\ nil) do + struct = + %{state: state} + |> Pleroma.Maps.put_if_present(:processed_number, processed_number) + + backup + |> cast(struct, [:state, :processed_number]) + |> Repo.update() + end + def process(%__MODULE__{} = backup) do - with {:ok, zip_file} <- export(backup), + set_state(backup, :running, 0) + + current_pid = self() + + task = + Task.Supervisor.async_nolink( + Pleroma.TaskSupervisor, + __MODULE__, + :do_process, + [backup, current_pid] + ) + + wait_backup(backup, backup.processed_number, task) + end + + def do_process(backup, current_pid) do + with {:ok, zip_file} <- export(backup, current_pid), {:ok, %{size: size}} <- File.stat(zip_file), {:ok, _upload} <- upload(backup, zip_file) do backup - |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) + |> cast( + %{ + file_size: size, + processed: true, + state: :complete + }, + [:file_size, :processed, :state] + ) |> Repo.update() end end + defp wait_backup(backup, current_processed, task) do + wait_time = Pleroma.Config.get([__MODULE__, :process_wait_time]) + + receive do + {:progress, new_processed} -> + total_processed = current_processed + new_processed + + set_state(backup, :running, total_processed) + wait_backup(backup, total_processed, task) + + {:DOWN, _ref, _proc, _pid, reason} -> + backup = get(backup.id) + + if reason != :normal do + Logger.error("Backup #{backup.id} process ended abnormally: #{inspect(reason)}") + + {:ok, backup} = set_state(backup, :failed) + + cleanup(backup) + + {:error, + %{ + backup: backup, + reason: :exit, + details: reason + }} + else + {:ok, backup} + end + after + wait_time -> + Logger.error( + "Backup #{backup.id} timed out after no response for #{wait_time}ms, terminating" + ) + + Task.Supervisor.terminate_child(Pleroma.TaskSupervisor, task.pid) + + {:ok, backup} = set_state(backup, :failed) + + cleanup(backup) + + {:error, + %{ + backup: backup, + reason: :timeout + }} + end + end + @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] - def export(%__MODULE__{} = backup) do + def export(%__MODULE__{} = backup, caller_pid) do backup = Repo.preload(backup, :user) - name = String.trim_trailing(backup.file_name, ".zip") - dir = dir(name) + dir = backup_tempdir(backup) with :ok <- File.mkdir(dir), - :ok <- actor(dir, backup.user), - :ok <- statuses(dir, backup.user), - :ok <- likes(dir, backup.user), - :ok <- bookmarks(dir, backup.user), + :ok <- actor(dir, backup.user, caller_pid), + :ok <- statuses(dir, backup.user, caller_pid), + :ok <- likes(dir, backup.user, caller_pid), + :ok <- bookmarks(dir, backup.user, caller_pid), {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), {:ok, _} <- File.rm_rf(dir) do {:ok, to_string(zip_path)} @@ -157,11 +243,12 @@ defmodule Pleroma.User.Backup do end end - defp actor(dir, user) do + defp actor(dir, user, caller_pid) do with {:ok, json} <- UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) |> Jason.encode() do + send(caller_pid, {:progress, 1}) File.write(Path.join(dir, "actor.json"), json) end end @@ -180,47 +267,80 @@ defmodule Pleroma.User.Backup do ) end - defp write(query, dir, name, fun) do + defp should_report?(num, chunk_size), do: rem(num, chunk_size) == 0 + + defp backup_tempdir(backup) do + name = String.trim_trailing(backup.file_name, ".zip") + dir(name) + end + + defp cleanup(backup) do + dir = backup_tempdir(backup) + File.rm_rf(dir) + end + + defp write(query, dir, name, fun, caller_pid) do path = Path.join(dir, "#{name}.json") + chunk_size = Pleroma.Config.get([__MODULE__, :process_chunk_size]) + with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do total = query - |> Pleroma.Repo.chunk_stream(100) + |> Pleroma.Repo.chunk_stream(chunk_size, _returns_as = :one, timeout: :infinity) |> Enum.reduce(0, fn i, acc -> - with {:ok, data} <- fun.(i), + with {:ok, data} <- + (try do + fun.(i) + rescue + e -> {:error, e} + end), {:ok, str} <- Jason.encode(data), :ok <- IO.write(file, str <> ",\n") do + if should_report?(acc + 1, chunk_size) do + send(caller_pid, {:progress, chunk_size}) + end + acc + 1 else - _ -> acc + {:error, e} -> + Logger.warn( + "Error processing backup item: #{inspect(e)}\n The item is: #{inspect(i)}" + ) + + acc + + _ -> + acc end end) + send(caller_pid, {:progress, rem(total, chunk_size)}) + with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do File.close(file) end end end - defp bookmarks(dir, %{id: user_id} = _user) do + defp bookmarks(dir, %{id: user_id} = _user, caller_pid) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) - |> write(dir, "bookmarks", fn a -> {:ok, a.object} end) + |> write(dir, "bookmarks", fn a -> {:ok, a.object} end, caller_pid) end - defp likes(dir, user) do + defp likes(dir, user, caller_pid) do user.ap_id |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) - |> write(dir, "likes", fn a -> {:ok, a.object} end) + |> write(dir, "likes", fn a -> {:ok, a.object} end, caller_pid) end - defp statuses(dir, user) do + defp statuses(dir, user, caller_pid) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) @@ -233,10 +353,15 @@ defmodule Pleroma.User.Backup do ] |> Enum.concat() |> ActivityPub.fetch_activities_query(opts) - |> write(dir, "outbox", fn a -> - with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do - {:ok, Map.delete(activity, "@context")} - end - end) + |> write( + dir, + "outbox", + fn a -> + with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do + {:ok, Map.delete(activity, "@context")} + end + end, + caller_pid + ) end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index f22756015..c93288b79 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1547,7 +1547,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do %{ ap_id: data["id"], uri: get_actor_url(data["url"]), - ap_enabled: true, banner: normalize_image(data["image"]), fields: fields, emoji: emojis, @@ -1668,7 +1667,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end end - def fetch_and_prepare_user_from_ap_id(ap_id, additional \\ []) do + defp fetch_and_prepare_user_from_ap_id(ap_id, additional) do with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id), {:ok, data} <- user_data_from_user_object(data, additional) do {:ok, maybe_update_follow_information(data)} @@ -1721,6 +1720,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) end + def pin_data_from_featured_collection(obj) do + Logger.error("Could not parse featured collection #{inspect(obj)}") + %{} + end + def fetch_and_prepare_featured_from_ap_id(nil) do {:ok, %{}} end @@ -1751,24 +1755,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do def make_user_from_ap_id(ap_id, additional \\ []) do user = User.get_cached_by_ap_id(ap_id) - if user && !User.ap_enabled?(user) do - Transmogrifier.upgrade_user_from_ap_id(ap_id) - else - with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do - {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) + with {:ok, data} <- fetch_and_prepare_user_from_ap_id(ap_id, additional) do + {:ok, _pid} = Task.start(fn -> pinned_fetch_task(data) end) - if user do - user - |> User.remote_user_changeset(data) - |> User.update_and_set_cache() - else - maybe_handle_clashing_nickname(data) + if user do + user + |> User.remote_user_changeset(data) + |> User.update_and_set_cache() + else + maybe_handle_clashing_nickname(data) - data - |> User.remote_user_changeset() - |> Repo.insert() - |> User.set_cache() - end + data + |> User.remote_user_changeset() + |> Repo.insert() + |> User.set_cache() end end end diff --git a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex index 70224561c..5532093cb 100644 --- a/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex +++ b/lib/pleroma/web/activity_pub/mrf/force_mentions_in_content.ex @@ -1,5 +1,5 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do @@ -95,11 +95,13 @@ defmodule Pleroma.Web.ActivityPub.MRF.ForceMentionsInContent do |> Enum.reject(&is_nil/1) |> sort_replied_user(replied_to_user) - explicitly_mentioned_uris = extract_mention_uris_from_content(content) + explicitly_mentioned_uris = + extract_mention_uris_from_content(content) + |> MapSet.new() added_mentions = - Enum.reduce(mention_users, "", fn %User{ap_id: uri} = user, acc -> - unless uri in explicitly_mentioned_uris do + Enum.reduce(mention_users, "", fn %User{ap_id: ap_id, uri: uri} = user, acc -> + if MapSet.disjoint?(MapSet.new([ap_id, uri]), explicitly_mentioned_uris) do acc <> Formatter.mention_from_user(user, %{mentions_format: :compact}) <> " " else acc diff --git a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex index 5202db7f1..db3259550 100644 --- a/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/add_remove_validator.ex @@ -73,6 +73,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AddRemoveValidator do end defp maybe_refetch_user(%User{ap_id: ap_id}) do - Pleroma.Web.ActivityPub.Transmogrifier.upgrade_user_from_ap_id(ap_id) + # Maybe it could use User.get_or_fetch_by_ap_id to avoid refreshing too often + User.fetch_by_ap_id(ap_id) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex index 7b60c139a..d580208df 100644 --- a/lib/pleroma/web/activity_pub/object_validators/common_fields.ex +++ b/lib/pleroma/web/activity_pub/object_validators/common_fields.ex @@ -58,7 +58,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CommonFields do field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) field(:inReplyTo, ObjectValidators.ObjectID) - field(:url, ObjectValidators.Uri) + field(:url, ObjectValidators.BareUri) field(:likes, {:array, ObjectValidators.ObjectID}, default: []) field(:announcements, {:array, ObjectValidators.ObjectID}, default: []) diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex index 6c1ba76a3..af6aa0781 100644 --- a/lib/pleroma/web/activity_pub/publisher.ex +++ b/lib/pleroma/web/activity_pub/publisher.ex @@ -199,7 +199,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do inboxes = recipients - |> Enum.filter(&User.ap_enabled?/1) |> Enum.map(fn actor -> actor.inbox end) |> Enum.filter(fn inbox -> should_federate?(inbox, public) end) |> Instances.filter_reachable() @@ -241,7 +240,6 @@ defmodule Pleroma.Web.ActivityPub.Publisher do json = Jason.encode!(data) recipients(actor, activity) - |> Enum.filter(fn user -> User.ap_enabled?(user) end) |> Enum.map(fn %User{} = user -> determine_inbox(activity, user) end) diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 3141f8437..0e6c429f9 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -20,7 +20,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.Federator - alias Pleroma.Workers.TransmogrifierWorker import Ecto.Query @@ -946,47 +945,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp strip_internal_tags(object), do: object - def perform(:user_upgrade, user) do - # we pass a fake user so that the followers collection is stripped away - old_follower_address = User.ap_followers(%User{nickname: user.nickname}) - - from( - a in Activity, - where: ^old_follower_address in a.recipients, - update: [ - set: [ - recipients: - fragment( - "array_replace(?,?,?)", - a.recipients, - ^old_follower_address, - ^user.follower_address - ) - ] - ] - ) - |> Repo.update_all([]) - end - - def upgrade_user_from_ap_id(ap_id) do - with %User{local: false} = user <- User.get_cached_by_ap_id(ap_id), - {:ok, data} <- ActivityPub.fetch_and_prepare_user_from_ap_id(ap_id), - {:ok, user} <- update_user(user, data) do - {:ok, _pid} = Task.start(fn -> ActivityPub.pinned_fetch_task(user) end) - TransmogrifierWorker.enqueue("user_upgrade", %{"user_id" => user.id}) - {:ok, user} - else - %User{} = user -> {:ok, user} - e -> e - end - end - - defp update_user(user, data) do - user - |> User.remote_user_changeset(data) - |> User.update_and_set_cache() - end - def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do Map.put(data, "url", url["href"]) end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex index 45fa2b058..400f3825d 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -64,7 +64,13 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do content_type: %Schema{type: :string}, file_name: %Schema{type: :string}, file_size: %Schema{type: :integer}, - processed: %Schema{type: :boolean} + processed: %Schema{type: :boolean, description: "whether this backup has succeeded"}, + state: %Schema{ + type: :string, + description: "the state of the backup", + enum: ["pending", "running", "complete", "failed"] + }, + processed_number: %Schema{type: :integer, description: "the number of records processed"} }, example: %{ "content_type" => "application/zip", @@ -72,7 +78,9 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do "https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip", "file_size" => 4105, "inserted_at" => "2020-09-08T16:42:07.000Z", - "processed" => true + "processed" => true, + "state" => "complete", + "processed_number" => 20 } } end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 89cc0d6fe..77b3fa5d2 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -142,7 +142,7 @@ defmodule Pleroma.Web.CommonAPI do def delete(activity_id, user) do with {_, %Activity{data: %{"object" => _, "type" => "Create"}} = activity} <- - {:find_activity, Activity.get_by_id(activity_id)}, + {:find_activity, Activity.get_by_id(activity_id, filter: [])}, {_, %Object{} = object, _} <- {:find_object, Object.normalize(activity, fetch: false), activity}, true <- User.privileged?(user, :messages_delete) || user.ap_id == object.data["actor"], @@ -583,7 +583,7 @@ defmodule Pleroma.Web.CommonAPI do end def update_report_state(activity_id, state) do - with %Activity{} = activity <- Activity.get_by_id(activity_id) do + with %Activity{} = activity <- Activity.get_by_id(activity_id, filter: []) do Utils.update_report_state(activity, state) else nil -> {:error, :not_found} diff --git a/lib/pleroma/web/federator.ex b/lib/pleroma/web/federator.ex index 318b6cb11..84b77cda1 100644 --- a/lib/pleroma/web/federator.ex +++ b/lib/pleroma/web/federator.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.Federator do alias Pleroma.Activity alias Pleroma.Object.Containment alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.Federator.Publisher @@ -80,7 +79,7 @@ defmodule Pleroma.Web.Federator do # NOTE: we use the actor ID to do the containment, this is fine because an # actor shouldn't be acting on objects outside their own AP server. - with {_, {:ok, _user}} <- {:actor, ap_enabled_actor(actor)}, + with {_, {:ok, _user}} <- {:actor, User.get_or_fetch_by_ap_id(actor)}, nil <- Activity.normalize(params["id"]), {_, :ok} <- {:correct_origin?, Containment.contain_origin_from_id(actor, params)}, @@ -110,14 +109,4 @@ defmodule Pleroma.Web.Federator do {:error, e} end end - - def ap_enabled_actor(id) do - user = User.get_cached_by_ap_id(id) - - if User.ap_enabled?(user) do - {:ok, user} - else - ActivityPub.make_user_from_ap_id(id) - end - end end diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index c313a0e97..9a4b56301 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -263,6 +263,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do {:error, %Ecto.Changeset{errors: [background: {"file is too large", _}]}} -> render_error(conn, :request_entity_too_large, "File is too large") + {:error, %Ecto.Changeset{errors: [{:bio, {_, _}} | _]}} -> + render_error(conn, :request_entity_too_large, "Bio is too long") + + {:error, %Ecto.Changeset{errors: [{:name, {_, _}} | _]}} -> + render_error(conn, :request_entity_too_large, "Name is too long") + + {:error, %Ecto.Changeset{errors: [{:fields, {"invalid", _}} | _]}} -> + render_error(conn, :request_entity_too_large, "One or more field entries are too long") + + {:error, %Ecto.Changeset{errors: [{:fields, {_, _}} | _]}} -> + render_error(conn, :request_entity_too_large, "Too many field entries") + _e -> render_error(conn, :forbidden, "Invalid request") end diff --git a/lib/pleroma/web/metadata/providers/twitter_card.ex b/lib/pleroma/web/metadata/providers/twitter_card.ex index 2dac22ee2..426022c65 100644 --- a/lib/pleroma/web/metadata/providers/twitter_card.ex +++ b/lib/pleroma/web/metadata/providers/twitter_card.ex @@ -76,9 +76,10 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do {:meta, [name: "twitter:card", content: "summary_large_image"], []}, {:meta, [ - name: "twitter:player", + name: "twitter:image", content: MediaProxy.url(url["href"]) - ], []} + ], []}, + {:meta, [name: "twitter:image:alt", content: truncate(attachment["name"])], []} | acc ] |> maybe_add_dimensions(url) @@ -130,4 +131,12 @@ defmodule Pleroma.Web.Metadata.Providers.TwitterCard do metadata end end + + defp truncate(nil), do: "" + + defp truncate(text) do + # truncate to 420 characters + # see https://developer.twitter.com/en/docs/twitter-for-websites/cards/overview/markup + Pleroma.Formatter.truncate(text, 420) + end end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index d778590f0..20403aeee 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -9,12 +9,22 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do alias Pleroma.Web.CommonAPI.Utils def render("show.json", %{backup: %Backup{} = backup}) do + # To deal with records before the migration + state = + if backup.state == :invalid do + if backup.processed, do: :complete, else: :failed + else + backup.state + end + %{ id: backup.id, content_type: backup.content_type, url: download_url(backup), file_size: backup.file_size, processed: backup.processed, + state: to_string(state), + processed_number: backup.processed_number, inserted_at: Utils.to_masto_date(backup.inserted_at) } end diff --git a/lib/pleroma/web/plugs/http_signature_plug.ex b/lib/pleroma/web/plugs/http_signature_plug.ex index 4bf325218..e814efc2c 100644 --- a/lib/pleroma/web/plugs/http_signature_plug.ex +++ b/lib/pleroma/web/plugs/http_signature_plug.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.Plugs.HTTPSignaturePlug do end def call(conn, _opts) do - if get_format(conn) == "activity+json" do + if get_format(conn) in ["json", "activity+json"] do conn |> maybe_assign_valid_signature() |> maybe_require_signature() diff --git a/lib/pleroma/web/preload.ex b/lib/pleroma/web/preload.ex index 4485383f9..6a4a8885e 100644 --- a/lib/pleroma/web/preload.ex +++ b/lib/pleroma/web/preload.ex @@ -11,7 +11,7 @@ defmodule Pleroma.Web.Preload do terms = params |> parser.generate_terms() - |> Enum.map(fn {k, v} -> {k, Base.encode64(Jason.encode!(v))} end) + |> Enum.map(fn {k, v} -> {k, Base.encode64(Jason.encode!(v, escape: :html_safe))} end) |> Enum.into(%{}) Map.merge(acc, terms) @@ -19,7 +19,7 @@ defmodule Pleroma.Web.Preload do rendered_html = preload_data - |> Jason.encode!() + |> Jason.encode!(escape: :html_safe) |> build_script_tag() |> HTML.safe_to_string() diff --git a/lib/pleroma/web/static_fe/static_fe_controller.ex b/lib/pleroma/web/static_fe/static_fe_controller.ex index 97c41c6f9..8019a218a 100644 --- a/lib/pleroma/web/static_fe/static_fe_controller.ex +++ b/lib/pleroma/web/static_fe/static_fe_controller.ex @@ -25,7 +25,15 @@ defmodule Pleroma.Web.StaticFE.StaticFEController do true <- Visibility.is_public?(activity.object), {_, true} <- {:visible?, Visibility.visible_for_user?(activity, _reading_user = nil)}, %User{} = user <- User.get_by_ap_id(activity.object.data["actor"]) do - meta = Metadata.build_tags(%{activity_id: notice_id, object: activity.object, user: user}) + url = Helpers.url(conn) <> conn.request_path + + meta = + Metadata.build_tags(%{ + activity_id: notice_id, + object: activity.object, + user: user, + url: url + }) timeline = activity.object.data["context"] diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 3c0da5c27..b9a04cc76 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -25,6 +25,7 @@ defmodule Pleroma.Web.Streamer do def registry, do: @registry @public_streams ["public", "public:local", "public:media", "public:local:media"] + @local_streams ["public:local", "public:local:media"] @user_streams ["user", "user:notification", "direct", "user:pleroma_chat"] @doc "Expands and authorizes a stream, and registers the process for streaming." @@ -41,14 +42,37 @@ defmodule Pleroma.Web.Streamer do end end + defp can_access_stream(user, oauth_token, kind) do + with {_, true} <- {:restrict?, Config.restrict_unauthenticated_access?(:timelines, kind)}, + {_, %User{id: user_id}, %Token{user_id: user_id}} <- {:user, user, oauth_token}, + {_, true} <- + {:scopes, + OAuthScopesPlug.filter_descendants(["read:statuses"], oauth_token.scopes) != []} do + true + else + {:restrict?, _} -> + true + + _ -> + false + end + end + @doc "Expand and authorizes a stream" @spec get_topic(stream :: String.t(), User.t() | nil, Token.t() | nil, Map.t()) :: {:ok, topic :: String.t()} | {:error, :bad_topic} def get_topic(stream, user, oauth_token, params \\ %{}) - # Allow all public steams. - def get_topic(stream, _user, _oauth_token, _params) when stream in @public_streams do - {:ok, stream} + # Allow all public steams if the instance allows unauthenticated access. + # Otherwise, only allow users with valid oauth tokens. + def get_topic(stream, user, oauth_token, _params) when stream in @public_streams do + kind = if stream in @local_streams, do: :local, else: :federated + + if can_access_stream(user, oauth_token, kind) do + {:ok, stream} + else + {:error, :unauthorized} + end end # Allow all hashtags streams. @@ -57,12 +81,20 @@ defmodule Pleroma.Web.Streamer do end # Allow remote instance streams. - def get_topic("public:remote", _user, _oauth_token, %{"instance" => instance} = _params) do - {:ok, "public:remote:" <> instance} + def get_topic("public:remote", user, oauth_token, %{"instance" => instance} = _params) do + if can_access_stream(user, oauth_token, :federated) do + {:ok, "public:remote:" <> instance} + else + {:error, :unauthorized} + end end - def get_topic("public:remote:media", _user, _oauth_token, %{"instance" => instance} = _params) do - {:ok, "public:remote:media:" <> instance} + def get_topic("public:remote:media", user, oauth_token, %{"instance" => instance} = _params) do + if can_access_stream(user, oauth_token, :federated) do + {:ok, "public:remote:media:" <> instance} + else + {:error, :unauthorized} + end end # Expand user streams. diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 12ee70f00..a485ddb4b 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -51,7 +51,7 @@ defmodule Pleroma.Workers.BackupWorker do end @impl Oban.Worker - def timeout(_job), do: :timer.seconds(900) + def timeout(_job), do: :infinity defp has_email?(user) do not is_nil(user.email) and user.email != "" diff --git a/lib/pleroma/workers/transmogrifier_worker.ex b/lib/pleroma/workers/transmogrifier_worker.ex deleted file mode 100644 index 1f3f5385e..000000000 --- a/lib/pleroma/workers/transmogrifier_worker.ex +++ /dev/null @@ -1,18 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Workers.TransmogrifierWorker do - alias Pleroma.User - - use Pleroma.Workers.WorkerHelper, queue: "transmogrifier" - - @impl Oban.Worker - def perform(%Job{args: %{"op" => "user_upgrade", "user_id" => user_id}}) do - user = User.get_cached_by_id(user_id) - Pleroma.Web.ActivityPub.Transmogrifier.perform(:user_upgrade, user) - end - - @impl Oban.Worker - def timeout(_job), do: :timer.seconds(5) -end |