summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/pleroma/ecto_enums.ex8
-rw-r--r--lib/pleroma/instances/instance.ex100
-rw-r--r--lib/pleroma/user/backup.ex175
-rw-r--r--lib/pleroma/web/activity_pub/activity_pub.ex5
-rw-r--r--lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex12
-rw-r--r--lib/pleroma/web/common_api.ex2
-rw-r--r--lib/pleroma/web/mastodon_api/controllers/account_controller.ex12
-rw-r--r--lib/pleroma/web/pleroma_api/views/backup_view.ex10
-rw-r--r--lib/pleroma/web/plugs/uploaded_media.ex22
-rw-r--r--lib/pleroma/workers/backup_worker.ex2
10 files changed, 297 insertions, 51 deletions
diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex
index a4890b489..b346b39d6 100644
--- a/lib/pleroma/ecto_enums.ex
+++ b/lib/pleroma/ecto_enums.ex
@@ -27,3 +27,11 @@ defenum(Pleroma.DataMigration.State,
failed: 4,
manual: 5
)
+
+defenum(Pleroma.User.Backup.State,
+ pending: 1,
+ running: 2,
+ complete: 3,
+ failed: 4,
+ invalid: 5
+)
diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex
index a5529ad44..9756c66dc 100644
--- a/lib/pleroma/instances/instance.ex
+++ b/lib/pleroma/instances/instance.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Instances.Instance do
alias Pleroma.Instances
alias Pleroma.Instances.Instance
+ alias Pleroma.Maps
alias Pleroma.Repo
alias Pleroma.User
alias Pleroma.Workers.BackgroundWorker
@@ -24,6 +25,14 @@ defmodule Pleroma.Instances.Instance do
field(:favicon, :string)
field(:favicon_updated_at, :naive_datetime)
+ embeds_one :metadata, Pleroma.Instances.Metadata, primary_key: false do
+ field(:software_name, :string)
+ field(:software_version, :string)
+ field(:software_repository, :string)
+ end
+
+ field(:metadata_updated_at, :utc_datetime)
+
timestamps()
end
@@ -31,11 +40,17 @@ defmodule Pleroma.Instances.Instance do
def changeset(struct, params \\ %{}) do
struct
- |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at])
+ |> cast(params, __schema__(:fields) -- [:metadata])
+ |> cast_embed(:metadata, with: &metadata_changeset/2)
|> validate_required([:host])
|> unique_constraint(:host)
end
+ def metadata_changeset(struct, params \\ %{}) do
+ struct
+ |> cast(params, [:software_name, :software_version, :software_repository])
+ end
+
def filter_reachable([]), do: %{}
def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do
@@ -198,6 +213,89 @@ defmodule Pleroma.Instances.Instance do
end
end
+ def get_or_update_metadata(%URI{host: host} = instance_uri) do
+ existing_record = Repo.get_by(Instance, %{host: host})
+ now = NaiveDateTime.utc_now()
+
+ if existing_record && existing_record.metadata_updated_at &&
+ NaiveDateTime.diff(now, existing_record.metadata_updated_at) < 86_400 do
+ existing_record.metadata
+ else
+ metadata = scrape_metadata(instance_uri)
+
+ if existing_record do
+ existing_record
+ |> changeset(%{metadata: metadata, metadata_updated_at: now})
+ |> Repo.update()
+ else
+ %Instance{}
+ |> changeset(%{host: host, metadata: metadata, metadata_updated_at: now})
+ |> Repo.insert()
+ end
+
+ metadata
+ end
+ end
+
+ defp get_nodeinfo_uri(well_known) do
+ links = Map.get(well_known, "links", [])
+
+ nodeinfo21 =
+ Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.1"))["href"]
+
+ nodeinfo20 =
+ Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))["href"]
+
+ cond do
+ is_binary(nodeinfo21) -> {:ok, nodeinfo21}
+ is_binary(nodeinfo20) -> {:ok, nodeinfo20}
+ true -> {:error, :no_links}
+ end
+ end
+
+ defp scrape_metadata(%URI{} = instance_uri) do
+ try do
+ with {_, true} <- {:reachable, reachable?(instance_uri.host)},
+ {:ok, %Tesla.Env{body: well_known_body}} <-
+ instance_uri
+ |> URI.merge("/.well-known/nodeinfo")
+ |> to_string()
+ |> Pleroma.HTTP.get([{"accept", "application/json"}]),
+ {:ok, well_known_json} <- Jason.decode(well_known_body),
+ {:ok, nodeinfo_uri} <- get_nodeinfo_uri(well_known_json),
+ {:ok, %Tesla.Env{body: nodeinfo_body}} <-
+ Pleroma.HTTP.get(nodeinfo_uri, [{"accept", "application/json"}]),
+ {:ok, nodeinfo} <- Jason.decode(nodeinfo_body) do
+ # Can extract more metadata from NodeInfo but need to be careful about it's size,
+ # can't just dump the entire thing
+ software = Map.get(nodeinfo, "software", %{})
+
+ %{
+ software_name: software["name"],
+ software_version: software["version"]
+ }
+ |> Maps.put_if_present(:software_repository, software["repository"])
+ else
+ {:reachable, false} ->
+ Logger.debug(
+ "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") ignored unreachable host"
+ )
+
+ nil
+
+ _ ->
+ nil
+ end
+ rescue
+ e ->
+ Logger.warn(
+ "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}"
+ )
+
+ nil
+ end
+ end
+
@doc """
Deletes all users from an instance in a background task, thus also deleting
all of those users' activities and notifications.
diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex
index 9df010605..447fca2a1 100644
--- a/lib/pleroma/user/backup.ex
+++ b/lib/pleroma/user/backup.ex
@@ -9,12 +9,14 @@ defmodule Pleroma.User.Backup do
import Ecto.Query
import Pleroma.Web.Gettext
+ require Logger
require Pleroma.Constants
alias Pleroma.Activity
alias Pleroma.Bookmark
alias Pleroma.Repo
alias Pleroma.User
+ alias Pleroma.User.Backup.State
alias Pleroma.Web.ActivityPub.ActivityPub
alias Pleroma.Web.ActivityPub.Transmogrifier
alias Pleroma.Web.ActivityPub.UserView
@@ -25,6 +27,8 @@ defmodule Pleroma.User.Backup do
field(:file_name, :string)
field(:file_size, :integer, default: 0)
field(:processed, :boolean, default: false)
+ field(:state, State, default: :invalid)
+ field(:processed_number, :integer, default: 0)
belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
@@ -46,7 +50,8 @@ defmodule Pleroma.User.Backup do
%__MODULE__{
user_id: user.id,
content_type: "application/zip",
- file_name: name
+ file_name: name,
+ state: :pending
}
end
@@ -109,27 +114,108 @@ defmodule Pleroma.User.Backup do
def get(id), do: Repo.get(__MODULE__, id)
+ defp set_state(backup, state, processed_number \\ nil) do
+ struct =
+ %{state: state}
+ |> Pleroma.Maps.put_if_present(:processed_number, processed_number)
+
+ backup
+ |> cast(struct, [:state, :processed_number])
+ |> Repo.update()
+ end
+
def process(%__MODULE__{} = backup) do
- with {:ok, zip_file} <- export(backup),
+ set_state(backup, :running, 0)
+
+ current_pid = self()
+
+ task =
+ Task.Supervisor.async_nolink(
+ Pleroma.TaskSupervisor,
+ __MODULE__,
+ :do_process,
+ [backup, current_pid]
+ )
+
+ wait_backup(backup, backup.processed_number, task)
+ end
+
+ def do_process(backup, current_pid) do
+ with {:ok, zip_file} <- export(backup, current_pid),
{:ok, %{size: size}} <- File.stat(zip_file),
{:ok, _upload} <- upload(backup, zip_file) do
backup
- |> cast(%{file_size: size, processed: true}, [:file_size, :processed])
+ |> cast(
+ %{
+ file_size: size,
+ processed: true,
+ state: :complete
+ },
+ [:file_size, :processed, :state]
+ )
|> Repo.update()
end
end
+ defp wait_backup(backup, current_processed, task) do
+ wait_time = Pleroma.Config.get([__MODULE__, :process_wait_time])
+
+ receive do
+ {:progress, new_processed} ->
+ total_processed = current_processed + new_processed
+
+ set_state(backup, :running, total_processed)
+ wait_backup(backup, total_processed, task)
+
+ {:DOWN, _ref, _proc, _pid, reason} ->
+ backup = get(backup.id)
+
+ if reason != :normal do
+ Logger.error("Backup #{backup.id} process ended abnormally: #{inspect(reason)}")
+
+ {:ok, backup} = set_state(backup, :failed)
+
+ cleanup(backup)
+
+ {:error,
+ %{
+ backup: backup,
+ reason: :exit,
+ details: reason
+ }}
+ else
+ {:ok, backup}
+ end
+ after
+ wait_time ->
+ Logger.error(
+ "Backup #{backup.id} timed out after no response for #{wait_time}ms, terminating"
+ )
+
+ Task.Supervisor.terminate_child(Pleroma.TaskSupervisor, task.pid)
+
+ {:ok, backup} = set_state(backup, :failed)
+
+ cleanup(backup)
+
+ {:error,
+ %{
+ backup: backup,
+ reason: :timeout
+ }}
+ end
+ end
+
@files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json']
- def export(%__MODULE__{} = backup) do
+ def export(%__MODULE__{} = backup, caller_pid) do
backup = Repo.preload(backup, :user)
- name = String.trim_trailing(backup.file_name, ".zip")
- dir = dir(name)
+ dir = backup_tempdir(backup)
with :ok <- File.mkdir(dir),
- :ok <- actor(dir, backup.user),
- :ok <- statuses(dir, backup.user),
- :ok <- likes(dir, backup.user),
- :ok <- bookmarks(dir, backup.user),
+ :ok <- actor(dir, backup.user, caller_pid),
+ :ok <- statuses(dir, backup.user, caller_pid),
+ :ok <- likes(dir, backup.user, caller_pid),
+ :ok <- bookmarks(dir, backup.user, caller_pid),
{:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir),
{:ok, _} <- File.rm_rf(dir) do
{:ok, to_string(zip_path)}
@@ -157,11 +243,12 @@ defmodule Pleroma.User.Backup do
end
end
- defp actor(dir, user) do
+ defp actor(dir, user, caller_pid) do
with {:ok, json} <-
UserView.render("user.json", %{user: user})
|> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"})
|> Jason.encode() do
+ send(caller_pid, {:progress, 1})
File.write(Path.join(dir, "actor.json"), json)
end
end
@@ -180,47 +267,80 @@ defmodule Pleroma.User.Backup do
)
end
- defp write(query, dir, name, fun) do
+ defp should_report?(num, chunk_size), do: rem(num, chunk_size) == 0
+
+ defp backup_tempdir(backup) do
+ name = String.trim_trailing(backup.file_name, ".zip")
+ dir(name)
+ end
+
+ defp cleanup(backup) do
+ dir = backup_tempdir(backup)
+ File.rm_rf(dir)
+ end
+
+ defp write(query, dir, name, fun, caller_pid) do
path = Path.join(dir, "#{name}.json")
+ chunk_size = Pleroma.Config.get([__MODULE__, :process_chunk_size])
+
with {:ok, file} <- File.open(path, [:write, :utf8]),
:ok <- write_header(file, name) do
total =
query
- |> Pleroma.Repo.chunk_stream(100)
+ |> Pleroma.Repo.chunk_stream(chunk_size, _returns_as = :one, timeout: :infinity)
|> Enum.reduce(0, fn i, acc ->
- with {:ok, data} <- fun.(i),
+ with {:ok, data} <-
+ (try do
+ fun.(i)
+ rescue
+ e -> {:error, e}
+ end),
{:ok, str} <- Jason.encode(data),
:ok <- IO.write(file, str <> ",\n") do
+ if should_report?(acc + 1, chunk_size) do
+ send(caller_pid, {:progress, chunk_size})
+ end
+
acc + 1
else
- _ -> acc
+ {:error, e} ->
+ Logger.warn(
+ "Error processing backup item: #{inspect(e)}\n The item is: #{inspect(i)}"
+ )
+
+ acc
+
+ _ ->
+ acc
end
end)
+ send(caller_pid, {:progress, rem(total, chunk_size)})
+
with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do
File.close(file)
end
end
end
- defp bookmarks(dir, %{id: user_id} = _user) do
+ defp bookmarks(dir, %{id: user_id} = _user, caller_pid) do
Bookmark
|> where(user_id: ^user_id)
|> join(:inner, [b], activity in assoc(b, :activity))
|> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)})
- |> write(dir, "bookmarks", fn a -> {:ok, a.object} end)
+ |> write(dir, "bookmarks", fn a -> {:ok, a.object} end, caller_pid)
end
- defp likes(dir, user) do
+ defp likes(dir, user, caller_pid) do
user.ap_id
|> Activity.Queries.by_actor()
|> Activity.Queries.by_type("Like")
|> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)})
- |> write(dir, "likes", fn a -> {:ok, a.object} end)
+ |> write(dir, "likes", fn a -> {:ok, a.object} end, caller_pid)
end
- defp statuses(dir, user) do
+ defp statuses(dir, user, caller_pid) do
opts =
%{}
|> Map.put(:type, ["Create", "Announce"])
@@ -233,10 +353,15 @@ defmodule Pleroma.User.Backup do
]
|> Enum.concat()
|> ActivityPub.fetch_activities_query(opts)
- |> write(dir, "outbox", fn a ->
- with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
- {:ok, Map.delete(activity, "@context")}
- end
- end)
+ |> write(
+ dir,
+ "outbox",
+ fn a ->
+ with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do
+ {:ok, Map.delete(activity, "@context")}
+ end
+ end,
+ caller_pid
+ )
end
end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 1a10aef1d..c93288b79 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1720,6 +1720,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end)
end
+ def pin_data_from_featured_collection(obj) do
+ Logger.error("Could not parse featured collection #{inspect(obj)}")
+ %{}
+ end
+
def fetch_and_prepare_featured_from_ap_id(nil) do
{:ok, %{}}
end
diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
index 45fa2b058..400f3825d 100644
--- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
+++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex
@@ -64,7 +64,13 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
content_type: %Schema{type: :string},
file_name: %Schema{type: :string},
file_size: %Schema{type: :integer},
- processed: %Schema{type: :boolean}
+ processed: %Schema{type: :boolean, description: "whether this backup has succeeded"},
+ state: %Schema{
+ type: :string,
+ description: "the state of the backup",
+ enum: ["pending", "running", "complete", "failed"]
+ },
+ processed_number: %Schema{type: :integer, description: "the number of records processed"}
},
example: %{
"content_type" => "application/zip",
@@ -72,7 +78,9 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do
"https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip",
"file_size" => 4105,
"inserted_at" => "2020-09-08T16:42:07.000Z",
- "processed" => true
+ "processed" => true,
+ "state" => "complete",
+ "processed_number" => 20
}
}
end
diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex
index 65d08de49..77b3fa5d2 100644
--- a/lib/pleroma/web/common_api.ex
+++ b/lib/pleroma/web/common_api.ex
@@ -583,7 +583,7 @@ defmodule Pleroma.Web.CommonAPI do
end
def update_report_state(activity_id, state) do
- with %Activity{} = activity <- Activity.get_by_id(activity_id) do
+ with %Activity{} = activity <- Activity.get_by_id(activity_id, filter: []) do
Utils.update_report_state(activity, state)
else
nil -> {:error, :not_found}
diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
index c313a0e97..9a4b56301 100644
--- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex
@@ -263,6 +263,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do
{:error, %Ecto.Changeset{errors: [background: {"file is too large", _}]}} ->
render_error(conn, :request_entity_too_large, "File is too large")
+ {:error, %Ecto.Changeset{errors: [{:bio, {_, _}} | _]}} ->
+ render_error(conn, :request_entity_too_large, "Bio is too long")
+
+ {:error, %Ecto.Changeset{errors: [{:name, {_, _}} | _]}} ->
+ render_error(conn, :request_entity_too_large, "Name is too long")
+
+ {:error, %Ecto.Changeset{errors: [{:fields, {"invalid", _}} | _]}} ->
+ render_error(conn, :request_entity_too_large, "One or more field entries are too long")
+
+ {:error, %Ecto.Changeset{errors: [{:fields, {_, _}} | _]}} ->
+ render_error(conn, :request_entity_too_large, "Too many field entries")
+
_e ->
render_error(conn, :forbidden, "Invalid request")
end
diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex
index d778590f0..20403aeee 100644
--- a/lib/pleroma/web/pleroma_api/views/backup_view.ex
+++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex
@@ -9,12 +9,22 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do
alias Pleroma.Web.CommonAPI.Utils
def render("show.json", %{backup: %Backup{} = backup}) do
+ # To deal with records before the migration
+ state =
+ if backup.state == :invalid do
+ if backup.processed, do: :complete, else: :failed
+ else
+ backup.state
+ end
+
%{
id: backup.id,
content_type: backup.content_type,
url: download_url(backup),
file_size: backup.file_size,
processed: backup.processed,
+ state: to_string(state),
+ processed_number: backup.processed_number,
inserted_at: Utils.to_masto_date(backup.inserted_at)
}
end
diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex
index 9dd5eb239..8b3bc9acb 100644
--- a/lib/pleroma/web/plugs/uploaded_media.ex
+++ b/lib/pleroma/web/plugs/uploaded_media.ex
@@ -46,32 +46,12 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do
config = Pleroma.Config.get(Pleroma.Upload)
- %{scheme: media_scheme, host: media_host, port: media_port} =
- Pleroma.Upload.base_url() |> URI.parse()
-
- with {:valid_host, true} <- {:valid_host, match?(^media_host, conn.host)},
- uploader <- Keyword.fetch!(config, :uploader),
+ with uploader <- Keyword.fetch!(config, :uploader),
proxy_remote = Keyword.get(config, :proxy_remote, false),
{:ok, get_method} <- uploader.get_file(file),
false <- media_is_banned(conn, get_method) do
get_media(conn, get_method, proxy_remote, opts)
else
- {:valid_host, false} ->
- redirect_url =
- %URI{
- scheme: media_scheme,
- host: media_host,
- port: media_port,
- path: conn.request_path,
- query: conn.query_string
- }
- |> URI.to_string()
- |> String.trim_trailing("?")
-
- conn
- |> Phoenix.Controller.redirect(external: redirect_url)
- |> halt()
-
_ ->
conn
|> send_resp(:internal_server_error, dgettext("errors", "Failed"))
diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex
index 12ee70f00..a485ddb4b 100644
--- a/lib/pleroma/workers/backup_worker.ex
+++ b/lib/pleroma/workers/backup_worker.ex
@@ -51,7 +51,7 @@ defmodule Pleroma.Workers.BackupWorker do
end
@impl Oban.Worker
- def timeout(_job), do: :timer.seconds(900)
+ def timeout(_job), do: :infinity
defp has_email?(user) do
not is_nil(user.email) and user.email != ""
generated by cgit v1.2.3 (git 2.25.1) at 2025-05-28 16:03:12 +0000