From ad605e3e16ba3f6ee3df7a0a3e6705036fef369f Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Tue, 20 Oct 2020 17:16:58 +0400 Subject: Rename `Pleroma.Backup` to `Pleroma.User.Backup` --- lib/pleroma/backup.ex | 258 --------------------- lib/pleroma/user/backup.ex | 258 +++++++++++++++++++++ .../admin_api/controllers/admin_api_controller.ex | 2 +- .../pleroma_api/controllers/backup_controller.ex | 7 +- lib/pleroma/web/pleroma_api/views/backup_view.ex | 2 +- lib/pleroma/workers/backup_worker.ex | 4 +- 6 files changed, 266 insertions(+), 265 deletions(-) delete mode 100644 lib/pleroma/backup.ex create mode 100644 lib/pleroma/user/backup.ex (limited to 'lib') diff --git a/lib/pleroma/backup.ex b/lib/pleroma/backup.ex deleted file mode 100644 index 629e879a7..000000000 --- a/lib/pleroma/backup.ex +++ /dev/null @@ -1,258 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Backup do - use Ecto.Schema - - import Ecto.Changeset - import Ecto.Query - import Pleroma.Web.Gettext - - require Pleroma.Constants - - alias Pleroma.Activity - alias Pleroma.Bookmark - alias Pleroma.Repo - alias Pleroma.User - alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.Transmogrifier - alias Pleroma.Web.ActivityPub.UserView - alias Pleroma.Workers.BackupWorker - - schema "backups" do - field(:content_type, :string) - field(:file_name, :string) - field(:file_size, :integer, default: 0) - field(:processed, :boolean, default: false) - - belongs_to(:user, User, type: FlakeId.Ecto.CompatType) - - timestamps() - end - - def create(user, admin_id \\ nil) do - with :ok <- validate_email_enabled(), - :ok <- validate_user_email(user), - :ok <- validate_limit(user, admin_id), - {:ok, backup} <- user |> new() |> Repo.insert() do - BackupWorker.process(backup, admin_id) - end - end - - def new(user) do - rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) - datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) - name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" - - %__MODULE__{ - user_id: user.id, - content_type: "application/zip", - file_name: name - } - end - - def delete(backup) do - uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - - with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do - Repo.delete(backup) - end - end - - defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok - - defp validate_limit(user, nil) do - case get_last(user.id) do - %__MODULE__{inserted_at: inserted_at} -> - days = Pleroma.Config.get([Pleroma.Backup, :limit_days]) - diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) - - if diff > days do - :ok - else - {:error, - dngettext( - "errors", - "Last export was less than a day ago", - "Last export was less than %{days} days ago", - days, - days: days - )} - end - - nil -> - :ok - end - end - - defp validate_email_enabled do - if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do - :ok - else - {:error, dgettext("errors", "Backups require enabled email")} - end - end - - defp validate_user_email(%User{email: nil}) do - {:error, dgettext("errors", "Email is required")} - end - - defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok - - def get_last(user_id) do - __MODULE__ - |> where(user_id: ^user_id) - |> order_by(desc: :id) - |> limit(1) - |> Repo.one() - end - - def list(%User{id: user_id}) do - __MODULE__ - |> where(user_id: ^user_id) - |> order_by(desc: :id) - |> Repo.all() - end - - def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do - __MODULE__ - |> where(user_id: ^user_id) - |> where([b], b.id != ^latest_id) - |> Repo.all() - |> Enum.each(&BackupWorker.delete/1) - end - - def get(id), do: Repo.get(__MODULE__, id) - - def process(%__MODULE__{} = backup) do - with {:ok, zip_file} <- export(backup), - {:ok, %{size: size}} <- File.stat(zip_file), - {:ok, _upload} <- upload(backup, zip_file) do - backup - |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) - |> Repo.update() - end - end - - @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] - def export(%__MODULE__{} = backup) do - backup = Repo.preload(backup, :user) - name = String.trim_trailing(backup.file_name, ".zip") - dir = dir(name) - - with :ok <- File.mkdir(dir), - :ok <- actor(dir, backup.user), - :ok <- statuses(dir, backup.user), - :ok <- likes(dir, backup.user), - :ok <- bookmarks(dir, backup.user), - {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), - {:ok, _} <- File.rm_rf(dir) do - {:ok, to_string(zip_path)} - end - end - - def dir(name) do - dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!() - Path.join(dir, name) - end - - def upload(%__MODULE__{} = backup, zip_path) do - uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - - upload = %Pleroma.Upload{ - name: backup.file_name, - tempfile: zip_path, - content_type: backup.content_type, - path: Path.join("backups", backup.file_name) - } - - with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), - :ok <- File.rm(zip_path) do - {:ok, upload} - end - end - - defp actor(dir, user) do - with {:ok, json} <- - UserView.render("user.json", %{user: user}) - |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) - |> Jason.encode() do - File.write(Path.join(dir, "actor.json"), json) - end - end - - defp write_header(file, name) do - IO.write( - file, - """ - { - "@context": "https://www.w3.org/ns/activitystreams", - "id": "#{name}.json", - "type": "OrderedCollection", - "orderedItems": [ - - """ - ) - end - - defp write(query, dir, name, fun) do - path = Path.join(dir, "#{name}.json") - - with {:ok, file} <- File.open(path, [:write, :utf8]), - :ok <- write_header(file, name) do - total = - query - |> Pleroma.Repo.chunk_stream(100) - |> Enum.reduce(0, fn i, acc -> - with {:ok, data} <- fun.(i), - {:ok, str} <- Jason.encode(data), - :ok <- IO.write(file, str <> ",\n") do - acc + 1 - else - _ -> acc - end - end) - - with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do - File.close(file) - end - end - end - - defp bookmarks(dir, %{id: user_id} = _user) do - Bookmark - |> where(user_id: ^user_id) - |> join(:inner, [b], activity in assoc(b, :activity)) - |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) - |> write(dir, "bookmarks", fn a -> {:ok, a.object} end) - end - - defp likes(dir, user) do - user.ap_id - |> Activity.Queries.by_actor() - |> Activity.Queries.by_type("Like") - |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) - |> write(dir, "likes", fn a -> {:ok, a.object} end) - end - - defp statuses(dir, user) do - opts = - %{} - |> Map.put(:type, ["Create", "Announce"]) - |> Map.put(:actor_id, user.ap_id) - - [ - [Pleroma.Constants.as_public(), user.ap_id], - User.following(user), - Pleroma.List.memberships(user) - ] - |> Enum.concat() - |> ActivityPub.fetch_activities_query(opts) - |> write(dir, "outbox", fn a -> - with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do - {:ok, Map.delete(activity, "@context")} - end - end) - end -end diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex new file mode 100644 index 000000000..a9041fd94 --- /dev/null +++ b/lib/pleroma/user/backup.ex @@ -0,0 +1,258 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.User.Backup do + use Ecto.Schema + + import Ecto.Changeset + import Ecto.Query + import Pleroma.Web.Gettext + + require Pleroma.Constants + + alias Pleroma.Activity + alias Pleroma.Bookmark + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.ActivityPub.Transmogrifier + alias Pleroma.Web.ActivityPub.UserView + alias Pleroma.Workers.BackupWorker + + schema "backups" do + field(:content_type, :string) + field(:file_name, :string) + field(:file_size, :integer, default: 0) + field(:processed, :boolean, default: false) + + belongs_to(:user, User, type: FlakeId.Ecto.CompatType) + + timestamps() + end + + def create(user, admin_id \\ nil) do + with :ok <- validate_email_enabled(), + :ok <- validate_user_email(user), + :ok <- validate_limit(user, admin_id), + {:ok, backup} <- user |> new() |> Repo.insert() do + BackupWorker.process(backup, admin_id) + end + end + + def new(user) do + rand_str = :crypto.strong_rand_bytes(32) |> Base.url_encode64(padding: false) + datetime = Calendar.NaiveDateTime.Format.iso8601_basic(NaiveDateTime.utc_now()) + name = "archive-#{user.nickname}-#{datetime}-#{rand_str}.zip" + + %__MODULE__{ + user_id: user.id, + content_type: "application/zip", + file_name: name + } + end + + def delete(backup) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + with :ok <- uploader.delete_file(Path.join("backups", backup.file_name)) do + Repo.delete(backup) + end + end + + defp validate_limit(_user, admin_id) when is_binary(admin_id), do: :ok + + defp validate_limit(user, nil) do + case get_last(user.id) do + %__MODULE__{inserted_at: inserted_at} -> + days = Pleroma.Config.get([__MODULE__, :limit_days]) + diff = Timex.diff(NaiveDateTime.utc_now(), inserted_at, :days) + + if diff > days do + :ok + else + {:error, + dngettext( + "errors", + "Last export was less than a day ago", + "Last export was less than %{days} days ago", + days, + days: days + )} + end + + nil -> + :ok + end + end + + defp validate_email_enabled do + if Pleroma.Config.get([Pleroma.Emails.Mailer, :enabled]) do + :ok + else + {:error, dgettext("errors", "Backups require enabled email")} + end + end + + defp validate_user_email(%User{email: nil}) do + {:error, dgettext("errors", "Email is required")} + end + + defp validate_user_email(%User{email: email}) when is_binary(email), do: :ok + + def get_last(user_id) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> limit(1) + |> Repo.one() + end + + def list(%User{id: user_id}) do + __MODULE__ + |> where(user_id: ^user_id) + |> order_by(desc: :id) + |> Repo.all() + end + + def remove_outdated(%__MODULE__{id: latest_id, user_id: user_id}) do + __MODULE__ + |> where(user_id: ^user_id) + |> where([b], b.id != ^latest_id) + |> Repo.all() + |> Enum.each(&BackupWorker.delete/1) + end + + def get(id), do: Repo.get(__MODULE__, id) + + def process(%__MODULE__{} = backup) do + with {:ok, zip_file} <- export(backup), + {:ok, %{size: size}} <- File.stat(zip_file), + {:ok, _upload} <- upload(backup, zip_file) do + backup + |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) + |> Repo.update() + end + end + + @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] + def export(%__MODULE__{} = backup) do + backup = Repo.preload(backup, :user) + name = String.trim_trailing(backup.file_name, ".zip") + dir = dir(name) + + with :ok <- File.mkdir(dir), + :ok <- actor(dir, backup.user), + :ok <- statuses(dir, backup.user), + :ok <- likes(dir, backup.user), + :ok <- bookmarks(dir, backup.user), + {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), + {:ok, _} <- File.rm_rf(dir) do + {:ok, to_string(zip_path)} + end + end + + def dir(name) do + dir = Pleroma.Config.get([__MODULE__, :dir]) || System.tmp_dir!() + Path.join(dir, name) + end + + def upload(%__MODULE__{} = backup, zip_path) do + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + upload = %Pleroma.Upload{ + name: backup.file_name, + tempfile: zip_path, + content_type: backup.content_type, + path: Path.join("backups", backup.file_name) + } + + with {:ok, _} <- Pleroma.Uploaders.Uploader.put_file(uploader, upload), + :ok <- File.rm(zip_path) do + {:ok, upload} + end + end + + defp actor(dir, user) do + with {:ok, json} <- + UserView.render("user.json", %{user: user}) + |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) + |> Jason.encode() do + File.write(Path.join(dir, "actor.json"), json) + end + end + + defp write_header(file, name) do + IO.write( + file, + """ + { + "@context": "https://www.w3.org/ns/activitystreams", + "id": "#{name}.json", + "type": "OrderedCollection", + "orderedItems": [ + + """ + ) + end + + defp write(query, dir, name, fun) do + path = Path.join(dir, "#{name}.json") + + with {:ok, file} <- File.open(path, [:write, :utf8]), + :ok <- write_header(file, name) do + total = + query + |> Pleroma.Repo.chunk_stream(100) + |> Enum.reduce(0, fn i, acc -> + with {:ok, data} <- fun.(i), + {:ok, str} <- Jason.encode(data), + :ok <- IO.write(file, str <> ",\n") do + acc + 1 + else + _ -> acc + end + end) + + with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do + File.close(file) + end + end + end + + defp bookmarks(dir, %{id: user_id} = _user) do + Bookmark + |> where(user_id: ^user_id) + |> join(:inner, [b], activity in assoc(b, :activity)) + |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) + |> write(dir, "bookmarks", fn a -> {:ok, a.object} end) + end + + defp likes(dir, user) do + user.ap_id + |> Activity.Queries.by_actor() + |> Activity.Queries.by_type("Like") + |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) + |> write(dir, "likes", fn a -> {:ok, a.object} end) + end + + defp statuses(dir, user) do + opts = + %{} + |> Map.put(:type, ["Create", "Announce"]) + |> Map.put(:actor_id, user.ap_id) + + [ + [Pleroma.Constants.as_public(), user.ap_id], + User.following(user), + Pleroma.List.memberships(user) + ] + |> Enum.concat() + |> ActivityPub.fetch_activities_query(opts) + |> write(dir, "outbox", fn a -> + with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do + {:ok, Map.delete(activity, "@context")} + end + end) + end +end diff --git a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex index a4f0d7d34..0a27c5861 100644 --- a/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/admin_api_controller.ex @@ -685,7 +685,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do def create_backup(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do with %User{} = user <- User.get_by_nickname(nickname), - {:ok, _} <- Pleroma.Backup.create(user, admin.id) do + {:ok, _} <- Pleroma.User.Backup.create(user, admin.id) do ModerationLog.insert_log(%{actor: admin, subject: user, action: "create_backup"}) json(conn, "") diff --git a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex index 8e3d081f3..bd7b36880 100644 --- a/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex +++ b/lib/pleroma/web/pleroma_api/controllers/backup_controller.ex @@ -6,6 +6,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do use Pleroma.Web, :controller alias Pleroma.Web.Plugs.OAuthScopesPlug + alias Pleroma.User.Backup action_fallback(Pleroma.Web.MastodonAPI.FallbackController) plug(OAuthScopesPlug, %{scopes: ["read:accounts"]} when action in [:index, :create]) @@ -14,13 +15,13 @@ defmodule Pleroma.Web.PleromaAPI.BackupController do defdelegate open_api_operation(action), to: Pleroma.Web.ApiSpec.PleromaBackupOperation def index(%{assigns: %{user: user}} = conn, _params) do - backups = Pleroma.Backup.list(user) + backups = Backup.list(user) render(conn, "index.json", backups: backups) end def create(%{assigns: %{user: user}} = conn, _params) do - with {:ok, _} <- Pleroma.Backup.create(user) do - backups = Pleroma.Backup.list(user) + with {:ok, _} <- Backup.create(user) do + backups = Backup.list(user) render(conn, "index.json", backups: backups) end end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index bf40a001e..af75876aa 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -5,7 +5,7 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do use Pleroma.Web, :view - alias Pleroma.Backup + alias Pleroma.User.Backup alias Pleroma.Web.CommonAPI.Utils def render("show.json", %{backup: %Backup{} = backup}) do diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 65754b6a2..5b4985983 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Workers.BackupWorker do use Oban.Worker, queue: :backup, max_attempts: 1 alias Oban.Job - alias Pleroma.Backup + alias Pleroma.User.Backup def process(backup, admin_user_id \\ nil) do %{"op" => "process", "backup_id" => backup.id, "admin_user_id" => admin_user_id} @@ -15,7 +15,7 @@ defmodule Pleroma.Workers.BackupWorker do end def schedule_deletion(backup) do - days = Pleroma.Config.get([Pleroma.Backup, :purge_after_days]) + days = Pleroma.Config.get([Backup, :purge_after_days]) time = 60 * 60 * 24 * days scheduled_at = Calendar.NaiveDateTime.add!(backup.inserted_at, time) -- cgit v1.2.3