diff options
| -rw-r--r-- | config/config.exs | 4 | ||||
| -rw-r--r-- | config/description.exs | 15 | ||||
| -rw-r--r-- | lib/pleroma/user/backup.ex | 37 |
3 files changed, 44 insertions, 12 deletions
diff --git a/config/config.exs b/config/config.exs index e41ec2f91..ecb592b9c 100644 --- a/config/config.exs +++ b/config/config.exs @@ -874,7 +874,9 @@ config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthent config :pleroma, Pleroma.User.Backup, purge_after_days: 30, limit_days: 7, - dir: nil + dir: nil, + process_wait_time: 30_000, + process_chunk_size: 100 config :pleroma, ConcurrentLimiter, [ {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, diff --git a/config/description.exs b/config/description.exs index bf4734426..996267558 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3394,6 +3394,21 @@ config :pleroma, :config_description, [ type: :integer, description: "Limit user to export not more often than once per N days", suggestions: [7] + }, + %{ + key: :process_wait_time, + type: :integer, + label: "Process Wait Time", + description: + "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed.", + suggestions: [30_000] + }, + %{ + key: :process_chunk_size, + type: :integer, + label: "Process Chunk Size", + description: "The number of activities to fetch in the backup job for each chunk.", + suggestions: [100] } ] }, diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index cb9a40ba1..447fca2a1 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -35,8 +35,6 @@ defmodule Pleroma.User.Backup do timestamps() end - @report_every 100 - def create(user, admin_id \\ nil) do with :ok <- validate_limit(user, admin_id), {:ok, backup} <- user |> new() |> Repo.insert() do @@ -160,6 +158,8 @@ defmodule Pleroma.User.Backup do end defp wait_backup(backup, current_processed, task) do + wait_time = Pleroma.Config.get([__MODULE__, :process_wait_time]) + receive do {:progress, new_processed} -> total_processed = current_processed + new_processed @@ -175,6 +175,8 @@ defmodule Pleroma.User.Backup do {:ok, backup} = set_state(backup, :failed) + cleanup(backup) + {:error, %{ backup: backup, @@ -185,15 +187,17 @@ defmodule Pleroma.User.Backup do {:ok, backup} end after - 30_000 -> + wait_time -> Logger.error( - "Backup #{backup.id} timed out after no response for 30 seconds, terminating" + "Backup #{backup.id} timed out after no response for #{wait_time}ms, terminating" ) Task.Supervisor.terminate_child(Pleroma.TaskSupervisor, task.pid) {:ok, backup} = set_state(backup, :failed) + cleanup(backup) + {:error, %{ backup: backup, @@ -205,8 +209,7 @@ defmodule Pleroma.User.Backup do @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] def export(%__MODULE__{} = backup, caller_pid) do backup = Repo.preload(backup, :user) - name = String.trim_trailing(backup.file_name, ".zip") - dir = dir(name) + dir = backup_tempdir(backup) with :ok <- File.mkdir(dir), :ok <- actor(dir, backup.user, caller_pid), @@ -264,16 +267,28 @@ defmodule Pleroma.User.Backup do ) end - defp should_report?(num), do: rem(num, @report_every) == 0 + defp should_report?(num, chunk_size), do: rem(num, chunk_size) == 0 + + defp backup_tempdir(backup) do + name = String.trim_trailing(backup.file_name, ".zip") + dir(name) + end + + defp cleanup(backup) do + dir = backup_tempdir(backup) + File.rm_rf(dir) + end defp write(query, dir, name, fun, caller_pid) do path = Path.join(dir, "#{name}.json") + chunk_size = Pleroma.Config.get([__MODULE__, :process_chunk_size]) + with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do total = query - |> Pleroma.Repo.chunk_stream(100) + |> Pleroma.Repo.chunk_stream(chunk_size, _returns_as = :one, timeout: :infinity) |> Enum.reduce(0, fn i, acc -> with {:ok, data} <- (try do @@ -283,8 +298,8 @@ defmodule Pleroma.User.Backup do end), {:ok, str} <- Jason.encode(data), :ok <- IO.write(file, str <> ",\n") do - if should_report?(acc + 1) do - send(caller_pid, {:progress, @report_every}) + if should_report?(acc + 1, chunk_size) do + send(caller_pid, {:progress, chunk_size}) end acc + 1 @@ -301,7 +316,7 @@ defmodule Pleroma.User.Backup do end end) - send(caller_pid, {:progress, rem(total, @report_every)}) + send(caller_pid, {:progress, rem(total, chunk_size)}) with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do File.close(file) |