Merge branch 'feature/account-export' into 'develop'

Add account export

Closes #847

See merge request pleroma/pleroma!2918

4 files changed, 399 insertions, 0 deletions

diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs
new file mode 100644
index 000000000..f68e4a029
--- /dev/null
+++ b/test/pleroma/user/backup_test.exs
@@ -0,0 +1,244 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.User.BackupTest do
+  use Oban.Testing, repo: Pleroma.Repo
+  use Pleroma.DataCase
+
+  import Mock
+  import Pleroma.Factory
+  import Swoosh.TestAssertions
+
+  alias Pleroma.Bookmark
+  alias Pleroma.Tests.ObanHelpers
+  alias Pleroma.User.Backup
+  alias Pleroma.Web.CommonAPI
+  alias Pleroma.Workers.BackupWorker
+
+  setup do
+    clear_config([Pleroma.Upload, :uploader])
+    clear_config([Backup, :limit_days])
+    clear_config([Pleroma.Emails.Mailer, :enabled], true)
+  end
+
+  test "it requries enabled email" do
+    Pleroma.Config.put([Pleroma.Emails.Mailer, :enabled], false)
+    user = insert(:user)
+    assert {:error, "Backups require enabled email"} == Backup.create(user)
+  end
+
+  test "it requries user's email" do
+    user = insert(:user, %{email: nil})
+    assert {:error, "Email is required"} == Backup.create(user)
+  end
+
+  test "it creates a backup record and an Oban job" do
+    %{id: user_id} = user = insert(:user)
+    assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
+    assert_enqueued(worker: BackupWorker, args: args)
+
+    backup = Backup.get(args["backup_id"])
+    assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+  end
+
+  test "it return an error if the export limit is over" do
+    %{id: user_id} = user = insert(:user)
+    limit_days = Pleroma.Config.get([Backup, :limit_days])
+    assert {:ok, %Oban.Job{args: args}} = Backup.create(user)
+    backup = Backup.get(args["backup_id"])
+    assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup
+
+    assert Backup.create(user) == {:error, "Last export was less than #{limit_days} days ago"}
+  end
+
+  test "it process a backup record" do
+    Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+    %{id: user_id} = user = insert(:user)
+
+    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user)
+    assert {:ok, backup} = perform_job(BackupWorker, args)
+    assert backup.file_size > 0
+    assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup
+
+    delete_job_args = %{"op" => "delete", "backup_id" => backup_id}
+
+    assert_enqueued(worker: BackupWorker, args: delete_job_args)
+    assert {:ok, backup} = perform_job(BackupWorker, delete_job_args)
+    refute Backup.get(backup_id)
+
+    email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup)
+
+    assert_email_sent(
+      to: {user.name, user.email},
+      html_body: email.html_body
+    )
+  end
+
+  test "it removes outdated backups after creating a fresh one" do
+    Pleroma.Config.put([Backup, :limit_days], -1)
+    Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+    user = insert(:user)
+
+    assert {:ok, job1} = Backup.create(user)
+
+    assert {:ok, %Backup{}} = ObanHelpers.perform(job1)
+    assert {:ok, job2} = Backup.create(user)
+    assert Pleroma.Repo.aggregate(Backup, :count) == 2
+    assert {:ok, backup2} = ObanHelpers.perform(job2)
+
+    ObanHelpers.perform_all()
+
+    assert [^backup2] = Pleroma.Repo.all(Backup)
+  end
+
+  test "it creates a zip archive with user data" do
+    user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+
+    {:ok, %{object: %{data: %{"id" => id1}}} = status1} =
+      CommonAPI.post(user, %{status: "status1"})
+
+    {:ok, %{object: %{data: %{"id" => id2}}} = status2} =
+      CommonAPI.post(user, %{status: "status2"})
+
+    {:ok, %{object: %{data: %{"id" => id3}}} = status3} =
+      CommonAPI.post(user, %{status: "status3"})
+
+    CommonAPI.favorite(user, status1.id)
+    CommonAPI.favorite(user, status2.id)
+
+    Bookmark.create(user.id, status2.id)
+    Bookmark.create(user.id, status3.id)
+
+    assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+    assert {:ok, path} = Backup.export(backup)
+    assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory])
+    assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile)
+
+    assert %{
+             "@context" => [
+               "https://www.w3.org/ns/activitystreams",
+               "http://localhost:4001/schemas/litepub-0.1.jsonld",
+               %{"@language" => "und"}
+             ],
+             "bookmarks" => "bookmarks.json",
+             "followers" => "http://cofe.io/users/cofe/followers",
+             "following" => "http://cofe.io/users/cofe/following",
+             "id" => "http://cofe.io/users/cofe",
+             "inbox" => "http://cofe.io/users/cofe/inbox",
+             "likes" => "likes.json",
+             "name" => "Cofe",
+             "outbox" => "http://cofe.io/users/cofe/outbox",
+             "preferredUsername" => "cofe",
+             "publicKey" => %{
+               "id" => "http://cofe.io/users/cofe#main-key",
+               "owner" => "http://cofe.io/users/cofe"
+             },
+             "type" => "Person",
+             "url" => "http://cofe.io/users/cofe"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'outbox.json', json}} = :zip.zip_get('outbox.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "outbox.json",
+             "orderedItems" => [
+               %{
+                 "object" => %{
+                   "actor" => "http://cofe.io/users/cofe",
+                   "content" => "status1",
+                   "type" => "Note"
+                 },
+                 "type" => "Create"
+               },
+               %{
+                 "object" => %{
+                   "actor" => "http://cofe.io/users/cofe",
+                   "content" => "status2"
+                 }
+               },
+               %{
+                 "actor" => "http://cofe.io/users/cofe",
+                 "object" => %{
+                   "content" => "status3"
+                 }
+               }
+             ],
+             "totalItems" => 3,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'likes.json', json}} = :zip.zip_get('likes.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "likes.json",
+             "orderedItems" => [^id1, ^id2],
+             "totalItems" => 2,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    assert {:ok, {'bookmarks.json', json}} = :zip.zip_get('bookmarks.json', zipfile)
+
+    assert %{
+             "@context" => "https://www.w3.org/ns/activitystreams",
+             "id" => "bookmarks.json",
+             "orderedItems" => [^id2, ^id3],
+             "totalItems" => 2,
+             "type" => "OrderedCollection"
+           } = Jason.decode!(json)
+
+    :zip.zip_close(zipfile)
+    File.rm!(path)
+  end
+
+  describe "it uploads and deletes a backup archive" do
+    setup do
+      clear_config(Pleroma.Uploaders.S3,
+        bucket: "test_bucket",
+        public_endpoint: "https://s3.amazonaws.com"
+      )
+
+      clear_config([Pleroma.Upload, :uploader])
+
+      user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"})
+
+      {:ok, status1} = CommonAPI.post(user, %{status: "status1"})
+      {:ok, status2} = CommonAPI.post(user, %{status: "status2"})
+      {:ok, status3} = CommonAPI.post(user, %{status: "status3"})
+      CommonAPI.favorite(user, status1.id)
+      CommonAPI.favorite(user, status2.id)
+      Bookmark.create(user.id, status2.id)
+      Bookmark.create(user.id, status3.id)
+
+      assert {:ok, backup} = user |> Backup.new() |> Repo.insert()
+      assert {:ok, path} = Backup.export(backup)
+
+      [path: path, backup: backup]
+    end
+
+    test "S3", %{path: path, backup: backup} do
+      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.S3)
+
+      with_mock ExAws,
+        request: fn
+          %{http_method: :put} -> {:ok, :ok}
+          %{http_method: :delete} -> {:ok, %{status_code: 204}}
+        end do
+        assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
+        assert {:ok, _backup} = Backup.delete(backup)
+      end
+
+      with_mock ExAws, request: fn %{http_method: :delete} -> {:ok, %{status_code: 204}} end do
+      end
+    end
+
+    test "Local", %{path: path, backup: backup} do
+      Pleroma.Config.put([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local)
+
+      assert {:ok, %Pleroma.Upload{}} = Backup.upload(backup, path)
+      assert {:ok, _backup} = Backup.delete(backup)
+    end
+  end
+end
diff --git a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
index e2d6d5f6e..74140b7bc 100644
--- a/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
+++ b/test/pleroma/web/admin_api/controllers/admin_api_controller_test.exs
@@ -977,6 +977,73 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do
                response["status_visibility"]
     end
   end
+
+  describe "/api/pleroma/backups" do
+    test "it creates a backup", %{conn: conn} do
+      admin = %{id: admin_id, nickname: admin_nickname} = insert(:user, is_admin: true)
+      token = insert(:oauth_admin_token, user: admin)
+      user = %{id: user_id, nickname: user_nickname} = insert(:user)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert [backup] = Repo.all(Pleroma.User.Backup)
+
+      ObanHelpers.perform_all()
+
+      email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup, admin.id)
+
+      assert String.contains?(email.html_body, "Admin @#{admin.nickname} requested a full backup")
+      assert_email_sent(to: {user.name, user.email}, html_body: email.html_body)
+
+      log_message = "@#{admin_nickname} requested account backup for @#{user_nickname}"
+
+      assert [
+               %{
+                 data: %{
+                   "action" => "create_backup",
+                   "actor" => %{
+                     "id" => ^admin_id,
+                     "nickname" => ^admin_nickname
+                   },
+                   "message" => ^log_message,
+                   "subject" => %{
+                     "id" => ^user_id,
+                     "nickname" => ^user_nickname
+                   }
+                 }
+               }
+             ] = Pleroma.ModerationLog |> Repo.all()
+    end
+
+    test "it doesn't limit admins", %{conn: conn} do
+      admin = insert(:user, is_admin: true)
+      token = insert(:oauth_admin_token, user: admin)
+      user = insert(:user)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert [_backup] = Repo.all(Pleroma.User.Backup)
+
+      assert "" ==
+               conn
+               |> assign(:user, admin)
+               |> assign(:token, token)
+               |> post("/api/pleroma/admin/backups", %{nickname: user.nickname})
+               |> json_response(200)
+
+      assert Repo.aggregate(Pleroma.User.Backup, :count) == 2
+    end
+  end
 end
 
 # Needed for testing
diff --git a/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
new file mode 100644
index 000000000..f1941f6dd
--- /dev/null
+++ b/test/pleroma/web/pleroma_api/controllers/backup_controller_test.exs
@@ -0,0 +1,85 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.PleromaAPI.BackupControllerTest do
+  use Pleroma.Web.ConnCase
+
+  alias Pleroma.User.Backup
+  alias Pleroma.Web.PleromaAPI.BackupView
+
+  setup do
+    clear_config([Pleroma.Upload, :uploader])
+    clear_config([Backup, :limit_days])
+    oauth_access(["read:accounts"])
+  end
+
+  test "GET /api/v1/pleroma/backups", %{user: user, conn: conn} do
+    assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id}}} = Backup.create(user)
+
+    backup = Backup.get(backup_id)
+
+    response =
+      conn
+      |> get("/api/v1/pleroma/backups")
+      |> json_response_and_validate_schema(:ok)
+
+    assert [
+             %{
+               "content_type" => "application/zip",
+               "url" => url,
+               "file_size" => 0,
+               "processed" => false,
+               "inserted_at" => _
+             }
+           ] = response
+
+    assert url == BackupView.download_url(backup)
+
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert [
+             %{
+               "url" => ^url,
+               "processed" => true
+             }
+           ] =
+             conn
+             |> get("/api/v1/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+  end
+
+  test "POST /api/v1/pleroma/backups", %{user: _user, conn: conn} do
+    assert [
+             %{
+               "content_type" => "application/zip",
+               "url" => url,
+               "file_size" => 0,
+               "processed" => false,
+               "inserted_at" => _
+             }
+           ] =
+             conn
+             |> post("/api/v1/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+
+    Pleroma.Tests.ObanHelpers.perform_all()
+
+    assert [
+             %{
+               "url" => ^url,
+               "processed" => true
+             }
+           ] =
+             conn
+             |> get("/api/v1/pleroma/backups")
+             |> json_response_and_validate_schema(:ok)
+
+    days = Pleroma.Config.get([Backup, :limit_days])
+
+    assert %{"error" => "Last export was less than #{days} days ago"} ==
+             conn
+             |> post("/api/v1/pleroma/backups")
+             |> json_response_and_validate_schema(400)
+  end
+end
diff --git a/test/support/oban_helpers.ex b/test/support/oban_helpers.ex
index 9f90a821c..2468f66dc 100644
--- a/test/support/oban_helpers.ex
+++ b/test/support/oban_helpers.ex
@@ -7,6 +7,8 @@ defmodule Pleroma.Tests.ObanHelpers do
   Oban test helpers.
   """
 
+  require Ecto.Query
+
   alias Pleroma.Repo
 
   def wipe_all do
@@ -15,6 +17,7 @@ defmodule Pleroma.Tests.ObanHelpers do
 
   def perform_all do
     Oban.Job
+    |> Ecto.Query.where(state: "available")
     |> Repo.all()
     |> perform()
   end