diff options
32 files changed, 644 insertions, 94 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8daa9f434..148cbe405 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -65,13 +65,21 @@ check-changelog: - "**/*.exs" - "mix.lock" +.using-ci-base: + tags: + - amd64 + build: - extends: .build_changes_policy + extends: + - .build_changes_policy + - .using-ci-base stage: build script: - mix compile --force spec-build: + extends: + - .using-ci-base stage: test rules: - changes: @@ -85,6 +93,8 @@ spec-build: - mix pleroma.openapi_spec spec.json benchmark: + extends: + - .using-ci-base stage: benchmark when: manual variables: @@ -99,7 +109,9 @@ benchmark: - mix pleroma.load_testing unit-testing: - extends: .build_changes_policy + extends: + - .build_changes_policy + - .using-ci-base stage: test cache: &testing_cache_policy <<: *global_cache_policy @@ -121,7 +133,9 @@ unit-testing: path: coverage.xml unit-testing-erratic: - extends: .build_changes_policy + extends: + - .build_changes_policy + - .using-ci-base stage: test retry: 2 allow_failure: true @@ -155,7 +169,9 @@ unit-testing-erratic: # - mix test --trace --only federated unit-testing-rum: - extends: .build_changes_policy + extends: + - .build_changes_policy + - .using-ci-base stage: test cache: *testing_cache_policy services: @@ -186,7 +202,9 @@ lint: - mix format --check-formatted analysis: - extends: .build_changes_policy + extends: + - .build_changes_policy + - .using-ci-base stage: test cache: *testing_cache_policy script: diff --git a/changelog.d/3896.add b/changelog.d/3896.add deleted file mode 100644 index e8fde4c8a..000000000 --- a/changelog.d/3896.add +++ /dev/null @@ -1 +0,0 @@ -Validate Host header for Uploads and return a 302 if the base_url has changed diff --git a/changelog.d/3909.skip b/changelog.d/3909.skip new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/changelog.d/3909.skip diff --git a/changelog.d/amd64-runner.skip b/changelog.d/amd64-runner.skip new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/changelog.d/amd64-runner.skip diff --git a/changelog.d/featured-collection-shouldnt-break-user-fetch.fix b/changelog.d/featured-collection-shouldnt-break-user-fetch.fix new file mode 100644 index 000000000..e8ce288cc --- /dev/null +++ b/changelog.d/featured-collection-shouldnt-break-user-fetch.fix @@ -0,0 +1 @@ +Fix user fetch completely broken if featured collection is not in a supported form diff --git a/changelog.d/handle-report-from-deactivated-user.fix b/changelog.d/handle-report-from-deactivated-user.fix new file mode 100644 index 000000000..6692d1aa8 --- /dev/null +++ b/changelog.d/handle-report-from-deactivated-user.fix @@ -0,0 +1 @@ +Fix handling report from a deactivated user diff --git a/changelog.d/update-credentials-limit-error.fix b/changelog.d/update-credentials-limit-error.fix new file mode 100644 index 000000000..7682f958e --- /dev/null +++ b/changelog.d/update-credentials-limit-error.fix @@ -0,0 +1 @@ +Show more informative errors when profile exceeds char limits diff --git a/config/config.exs b/config/config.exs index 178b6be99..a92ab574b 100644 --- a/config/config.exs +++ b/config/config.exs @@ -871,7 +871,9 @@ config :pleroma, Pleroma.Web.Auth.Authenticator, Pleroma.Web.Auth.PleromaAuthent config :pleroma, Pleroma.User.Backup, purge_after_days: 30, limit_days: 7, - dir: nil + dir: nil, + process_wait_time: 30_000, + process_chunk_size: 100 config :pleroma, ConcurrentLimiter, [ {Pleroma.Web.RichMedia.Helpers, [max_running: 5, max_waiting: 5]}, diff --git a/config/description.exs b/config/description.exs index 5f8add8a4..d18649ae8 100644 --- a/config/description.exs +++ b/config/description.exs @@ -3364,6 +3364,21 @@ config :pleroma, :config_description, [ type: :integer, description: "Limit user to export not more often than once per N days", suggestions: [7] + }, + %{ + key: :process_wait_time, + type: :integer, + label: "Process Wait Time", + description: + "The amount of time to wait for backup to report progress, in milliseconds. If no progress is received from the backup job for that much time, terminate it and deem it failed.", + suggestions: [30_000] + }, + %{ + key: :process_chunk_size, + type: :integer, + label: "Process Chunk Size", + description: "The number of activities to fetch in the backup job for each chunk.", + suggestions: [100] } ] }, diff --git a/lib/pleroma/ecto_enums.ex b/lib/pleroma/ecto_enums.ex index a4890b489..b346b39d6 100644 --- a/lib/pleroma/ecto_enums.ex +++ b/lib/pleroma/ecto_enums.ex @@ -27,3 +27,11 @@ defenum(Pleroma.DataMigration.State, failed: 4, manual: 5 ) + +defenum(Pleroma.User.Backup.State, + pending: 1, + running: 2, + complete: 3, + failed: 4, + invalid: 5 +) diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index a5529ad44..9756c66dc 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Instances.Instance do alias Pleroma.Instances alias Pleroma.Instances.Instance + alias Pleroma.Maps alias Pleroma.Repo alias Pleroma.User alias Pleroma.Workers.BackgroundWorker @@ -24,6 +25,14 @@ defmodule Pleroma.Instances.Instance do field(:favicon, :string) field(:favicon_updated_at, :naive_datetime) + embeds_one :metadata, Pleroma.Instances.Metadata, primary_key: false do + field(:software_name, :string) + field(:software_version, :string) + field(:software_repository, :string) + end + + field(:metadata_updated_at, :utc_datetime) + timestamps() end @@ -31,11 +40,17 @@ defmodule Pleroma.Instances.Instance do def changeset(struct, params \\ %{}) do struct - |> cast(params, [:host, :unreachable_since, :favicon, :favicon_updated_at]) + |> cast(params, __schema__(:fields) -- [:metadata]) + |> cast_embed(:metadata, with: &metadata_changeset/2) |> validate_required([:host]) |> unique_constraint(:host) end + def metadata_changeset(struct, params \\ %{}) do + struct + |> cast(params, [:software_name, :software_version, :software_repository]) + end + def filter_reachable([]), do: %{} def filter_reachable(urls_or_hosts) when is_list(urls_or_hosts) do @@ -198,6 +213,89 @@ defmodule Pleroma.Instances.Instance do end end + def get_or_update_metadata(%URI{host: host} = instance_uri) do + existing_record = Repo.get_by(Instance, %{host: host}) + now = NaiveDateTime.utc_now() + + if existing_record && existing_record.metadata_updated_at && + NaiveDateTime.diff(now, existing_record.metadata_updated_at) < 86_400 do + existing_record.metadata + else + metadata = scrape_metadata(instance_uri) + + if existing_record do + existing_record + |> changeset(%{metadata: metadata, metadata_updated_at: now}) + |> Repo.update() + else + %Instance{} + |> changeset(%{host: host, metadata: metadata, metadata_updated_at: now}) + |> Repo.insert() + end + + metadata + end + end + + defp get_nodeinfo_uri(well_known) do + links = Map.get(well_known, "links", []) + + nodeinfo21 = + Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.1"))["href"] + + nodeinfo20 = + Enum.find(links, &(&1["rel"] == "http://nodeinfo.diaspora.software/ns/schema/2.0"))["href"] + + cond do + is_binary(nodeinfo21) -> {:ok, nodeinfo21} + is_binary(nodeinfo20) -> {:ok, nodeinfo20} + true -> {:error, :no_links} + end + end + + defp scrape_metadata(%URI{} = instance_uri) do + try do + with {_, true} <- {:reachable, reachable?(instance_uri.host)}, + {:ok, %Tesla.Env{body: well_known_body}} <- + instance_uri + |> URI.merge("/.well-known/nodeinfo") + |> to_string() + |> Pleroma.HTTP.get([{"accept", "application/json"}]), + {:ok, well_known_json} <- Jason.decode(well_known_body), + {:ok, nodeinfo_uri} <- get_nodeinfo_uri(well_known_json), + {:ok, %Tesla.Env{body: nodeinfo_body}} <- + Pleroma.HTTP.get(nodeinfo_uri, [{"accept", "application/json"}]), + {:ok, nodeinfo} <- Jason.decode(nodeinfo_body) do + # Can extract more metadata from NodeInfo but need to be careful about it's size, + # can't just dump the entire thing + software = Map.get(nodeinfo, "software", %{}) + + %{ + software_name: software["name"], + software_version: software["version"] + } + |> Maps.put_if_present(:software_repository, software["repository"]) + else + {:reachable, false} -> + Logger.debug( + "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") ignored unreachable host" + ) + + nil + + _ -> + nil + end + rescue + e -> + Logger.warn( + "Instance.scrape_metadata(\"#{to_string(instance_uri)}\") error: #{inspect(e)}" + ) + + nil + end + end + @doc """ Deletes all users from an instance in a background task, thus also deleting all of those users' activities and notifications. diff --git a/lib/pleroma/user/backup.ex b/lib/pleroma/user/backup.ex index 9df010605..447fca2a1 100644 --- a/lib/pleroma/user/backup.ex +++ b/lib/pleroma/user/backup.ex @@ -9,12 +9,14 @@ defmodule Pleroma.User.Backup do import Ecto.Query import Pleroma.Web.Gettext + require Logger require Pleroma.Constants alias Pleroma.Activity alias Pleroma.Bookmark alias Pleroma.Repo alias Pleroma.User + alias Pleroma.User.Backup.State alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Transmogrifier alias Pleroma.Web.ActivityPub.UserView @@ -25,6 +27,8 @@ defmodule Pleroma.User.Backup do field(:file_name, :string) field(:file_size, :integer, default: 0) field(:processed, :boolean, default: false) + field(:state, State, default: :invalid) + field(:processed_number, :integer, default: 0) belongs_to(:user, User, type: FlakeId.Ecto.CompatType) @@ -46,7 +50,8 @@ defmodule Pleroma.User.Backup do %__MODULE__{ user_id: user.id, content_type: "application/zip", - file_name: name + file_name: name, + state: :pending } end @@ -109,27 +114,108 @@ defmodule Pleroma.User.Backup do def get(id), do: Repo.get(__MODULE__, id) + defp set_state(backup, state, processed_number \\ nil) do + struct = + %{state: state} + |> Pleroma.Maps.put_if_present(:processed_number, processed_number) + + backup + |> cast(struct, [:state, :processed_number]) + |> Repo.update() + end + def process(%__MODULE__{} = backup) do - with {:ok, zip_file} <- export(backup), + set_state(backup, :running, 0) + + current_pid = self() + + task = + Task.Supervisor.async_nolink( + Pleroma.TaskSupervisor, + __MODULE__, + :do_process, + [backup, current_pid] + ) + + wait_backup(backup, backup.processed_number, task) + end + + def do_process(backup, current_pid) do + with {:ok, zip_file} <- export(backup, current_pid), {:ok, %{size: size}} <- File.stat(zip_file), {:ok, _upload} <- upload(backup, zip_file) do backup - |> cast(%{file_size: size, processed: true}, [:file_size, :processed]) + |> cast( + %{ + file_size: size, + processed: true, + state: :complete + }, + [:file_size, :processed, :state] + ) |> Repo.update() end end + defp wait_backup(backup, current_processed, task) do + wait_time = Pleroma.Config.get([__MODULE__, :process_wait_time]) + + receive do + {:progress, new_processed} -> + total_processed = current_processed + new_processed + + set_state(backup, :running, total_processed) + wait_backup(backup, total_processed, task) + + {:DOWN, _ref, _proc, _pid, reason} -> + backup = get(backup.id) + + if reason != :normal do + Logger.error("Backup #{backup.id} process ended abnormally: #{inspect(reason)}") + + {:ok, backup} = set_state(backup, :failed) + + cleanup(backup) + + {:error, + %{ + backup: backup, + reason: :exit, + details: reason + }} + else + {:ok, backup} + end + after + wait_time -> + Logger.error( + "Backup #{backup.id} timed out after no response for #{wait_time}ms, terminating" + ) + + Task.Supervisor.terminate_child(Pleroma.TaskSupervisor, task.pid) + + {:ok, backup} = set_state(backup, :failed) + + cleanup(backup) + + {:error, + %{ + backup: backup, + reason: :timeout + }} + end + end + @files ['actor.json', 'outbox.json', 'likes.json', 'bookmarks.json'] - def export(%__MODULE__{} = backup) do + def export(%__MODULE__{} = backup, caller_pid) do backup = Repo.preload(backup, :user) - name = String.trim_trailing(backup.file_name, ".zip") - dir = dir(name) + dir = backup_tempdir(backup) with :ok <- File.mkdir(dir), - :ok <- actor(dir, backup.user), - :ok <- statuses(dir, backup.user), - :ok <- likes(dir, backup.user), - :ok <- bookmarks(dir, backup.user), + :ok <- actor(dir, backup.user, caller_pid), + :ok <- statuses(dir, backup.user, caller_pid), + :ok <- likes(dir, backup.user, caller_pid), + :ok <- bookmarks(dir, backup.user, caller_pid), {:ok, zip_path} <- :zip.create(String.to_charlist(dir <> ".zip"), @files, cwd: dir), {:ok, _} <- File.rm_rf(dir) do {:ok, to_string(zip_path)} @@ -157,11 +243,12 @@ defmodule Pleroma.User.Backup do end end - defp actor(dir, user) do + defp actor(dir, user, caller_pid) do with {:ok, json} <- UserView.render("user.json", %{user: user}) |> Map.merge(%{"likes" => "likes.json", "bookmarks" => "bookmarks.json"}) |> Jason.encode() do + send(caller_pid, {:progress, 1}) File.write(Path.join(dir, "actor.json"), json) end end @@ -180,47 +267,80 @@ defmodule Pleroma.User.Backup do ) end - defp write(query, dir, name, fun) do + defp should_report?(num, chunk_size), do: rem(num, chunk_size) == 0 + + defp backup_tempdir(backup) do + name = String.trim_trailing(backup.file_name, ".zip") + dir(name) + end + + defp cleanup(backup) do + dir = backup_tempdir(backup) + File.rm_rf(dir) + end + + defp write(query, dir, name, fun, caller_pid) do path = Path.join(dir, "#{name}.json") + chunk_size = Pleroma.Config.get([__MODULE__, :process_chunk_size]) + with {:ok, file} <- File.open(path, [:write, :utf8]), :ok <- write_header(file, name) do total = query - |> Pleroma.Repo.chunk_stream(100) + |> Pleroma.Repo.chunk_stream(chunk_size, _returns_as = :one, timeout: :infinity) |> Enum.reduce(0, fn i, acc -> - with {:ok, data} <- fun.(i), + with {:ok, data} <- + (try do + fun.(i) + rescue + e -> {:error, e} + end), {:ok, str} <- Jason.encode(data), :ok <- IO.write(file, str <> ",\n") do + if should_report?(acc + 1, chunk_size) do + send(caller_pid, {:progress, chunk_size}) + end + acc + 1 else - _ -> acc + {:error, e} -> + Logger.warn( + "Error processing backup item: #{inspect(e)}\n The item is: #{inspect(i)}" + ) + + acc + + _ -> + acc end end) + send(caller_pid, {:progress, rem(total, chunk_size)}) + with :ok <- :file.pwrite(file, {:eof, -2}, "\n],\n \"totalItems\": #{total}}") do File.close(file) end end end - defp bookmarks(dir, %{id: user_id} = _user) do + defp bookmarks(dir, %{id: user_id} = _user, caller_pid) do Bookmark |> where(user_id: ^user_id) |> join(:inner, [b], activity in assoc(b, :activity)) |> select([b, a], %{id: b.id, object: fragment("(?)->>'object'", a.data)}) - |> write(dir, "bookmarks", fn a -> {:ok, a.object} end) + |> write(dir, "bookmarks", fn a -> {:ok, a.object} end, caller_pid) end - defp likes(dir, user) do + defp likes(dir, user, caller_pid) do user.ap_id |> Activity.Queries.by_actor() |> Activity.Queries.by_type("Like") |> select([like], %{id: like.id, object: fragment("(?)->>'object'", like.data)}) - |> write(dir, "likes", fn a -> {:ok, a.object} end) + |> write(dir, "likes", fn a -> {:ok, a.object} end, caller_pid) end - defp statuses(dir, user) do + defp statuses(dir, user, caller_pid) do opts = %{} |> Map.put(:type, ["Create", "Announce"]) @@ -233,10 +353,15 @@ defmodule Pleroma.User.Backup do ] |> Enum.concat() |> ActivityPub.fetch_activities_query(opts) - |> write(dir, "outbox", fn a -> - with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do - {:ok, Map.delete(activity, "@context")} - end - end) + |> write( + dir, + "outbox", + fn a -> + with {:ok, activity} <- Transmogrifier.prepare_outgoing(a.data) do + {:ok, Map.delete(activity, "@context")} + end + end, + caller_pid + ) end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 1a10aef1d..c93288b79 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -1720,6 +1720,11 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do end) end + def pin_data_from_featured_collection(obj) do + Logger.error("Could not parse featured collection #{inspect(obj)}") + %{} + end + def fetch_and_prepare_featured_from_ap_id(nil) do {:ok, %{}} end diff --git a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex index 45fa2b058..400f3825d 100644 --- a/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex +++ b/lib/pleroma/web/api_spec/operations/pleroma_backup_operation.ex @@ -64,7 +64,13 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do content_type: %Schema{type: :string}, file_name: %Schema{type: :string}, file_size: %Schema{type: :integer}, - processed: %Schema{type: :boolean} + processed: %Schema{type: :boolean, description: "whether this backup has succeeded"}, + state: %Schema{ + type: :string, + description: "the state of the backup", + enum: ["pending", "running", "complete", "failed"] + }, + processed_number: %Schema{type: :integer, description: "the number of records processed"} }, example: %{ "content_type" => "application/zip", @@ -72,7 +78,9 @@ defmodule Pleroma.Web.ApiSpec.PleromaBackupOperation do "https://cofe.fe:4000/media/backups/archive-foobar-20200908T164207-Yr7vuT5Wycv-sN3kSN2iJ0k-9pMo60j9qmvRCdDqIew.zip", "file_size" => 4105, "inserted_at" => "2020-09-08T16:42:07.000Z", - "processed" => true + "processed" => true, + "state" => "complete", + "processed_number" => 20 } } end diff --git a/lib/pleroma/web/common_api.ex b/lib/pleroma/web/common_api.ex index 65d08de49..77b3fa5d2 100644 --- a/lib/pleroma/web/common_api.ex +++ b/lib/pleroma/web/common_api.ex @@ -583,7 +583,7 @@ defmodule Pleroma.Web.CommonAPI do end def update_report_state(activity_id, state) do - with %Activity{} = activity <- Activity.get_by_id(activity_id) do + with %Activity{} = activity <- Activity.get_by_id(activity_id, filter: []) do Utils.update_report_state(activity, state) else nil -> {:error, :not_found} diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index c313a0e97..9a4b56301 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -263,6 +263,18 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do {:error, %Ecto.Changeset{errors: [background: {"file is too large", _}]}} -> render_error(conn, :request_entity_too_large, "File is too large") + {:error, %Ecto.Changeset{errors: [{:bio, {_, _}} | _]}} -> + render_error(conn, :request_entity_too_large, "Bio is too long") + + {:error, %Ecto.Changeset{errors: [{:name, {_, _}} | _]}} -> + render_error(conn, :request_entity_too_large, "Name is too long") + + {:error, %Ecto.Changeset{errors: [{:fields, {"invalid", _}} | _]}} -> + render_error(conn, :request_entity_too_large, "One or more field entries are too long") + + {:error, %Ecto.Changeset{errors: [{:fields, {_, _}} | _]}} -> + render_error(conn, :request_entity_too_large, "Too many field entries") + _e -> render_error(conn, :forbidden, "Invalid request") end diff --git a/lib/pleroma/web/pleroma_api/views/backup_view.ex b/lib/pleroma/web/pleroma_api/views/backup_view.ex index d778590f0..20403aeee 100644 --- a/lib/pleroma/web/pleroma_api/views/backup_view.ex +++ b/lib/pleroma/web/pleroma_api/views/backup_view.ex @@ -9,12 +9,22 @@ defmodule Pleroma.Web.PleromaAPI.BackupView do alias Pleroma.Web.CommonAPI.Utils def render("show.json", %{backup: %Backup{} = backup}) do + # To deal with records before the migration + state = + if backup.state == :invalid do + if backup.processed, do: :complete, else: :failed + else + backup.state + end + %{ id: backup.id, content_type: backup.content_type, url: download_url(backup), file_size: backup.file_size, processed: backup.processed, + state: to_string(state), + processed_number: backup.processed_number, inserted_at: Utils.to_masto_date(backup.inserted_at) } end diff --git a/lib/pleroma/web/plugs/uploaded_media.ex b/lib/pleroma/web/plugs/uploaded_media.ex index 9dd5eb239..8b3bc9acb 100644 --- a/lib/pleroma/web/plugs/uploaded_media.ex +++ b/lib/pleroma/web/plugs/uploaded_media.ex @@ -46,32 +46,12 @@ defmodule Pleroma.Web.Plugs.UploadedMedia do config = Pleroma.Config.get(Pleroma.Upload) - %{scheme: media_scheme, host: media_host, port: media_port} = - Pleroma.Upload.base_url() |> URI.parse() - - with {:valid_host, true} <- {:valid_host, match?(^media_host, conn.host)}, - uploader <- Keyword.fetch!(config, :uploader), + with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), {:ok, get_method} <- uploader.get_file(file), false <- media_is_banned(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else - {:valid_host, false} -> - redirect_url = - %URI{ - scheme: media_scheme, - host: media_host, - port: media_port, - path: conn.request_path, - query: conn.query_string - } - |> URI.to_string() - |> String.trim_trailing("?") - - conn - |> Phoenix.Controller.redirect(external: redirect_url) - |> halt() - _ -> conn |> send_resp(:internal_server_error, dgettext("errors", "Failed")) diff --git a/lib/pleroma/workers/backup_worker.ex b/lib/pleroma/workers/backup_worker.ex index 12ee70f00..a485ddb4b 100644 --- a/lib/pleroma/workers/backup_worker.ex +++ b/lib/pleroma/workers/backup_worker.ex @@ -51,7 +51,7 @@ defmodule Pleroma.Workers.BackupWorker do end @impl Oban.Worker - def timeout(_job), do: :timer.seconds(900) + def timeout(_job), do: :infinity defp has_email?(user) do not is_nil(user.email) and user.email != "" diff --git a/priv/repo/migrations/20221216052127_add_state_to_backups.exs b/priv/repo/migrations/20221216052127_add_state_to_backups.exs new file mode 100644 index 000000000..73b30fc35 --- /dev/null +++ b/priv/repo/migrations/20221216052127_add_state_to_backups.exs @@ -0,0 +1,21 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.AddStateToBackups do + use Ecto.Migration + + def up do + alter table(:backups) do + add(:state, :integer, default: 5) + add(:processed_number, :integer, default: 0) + end + end + + def down do + alter table(:backups) do + remove(:state) + remove(:processed_number) + end + end +end diff --git a/priv/repo/migrations/20230306112859_instances_add_metadata.exs b/priv/repo/migrations/20230306112859_instances_add_metadata.exs new file mode 100644 index 000000000..898f5220e --- /dev/null +++ b/priv/repo/migrations/20230306112859_instances_add_metadata.exs @@ -0,0 +1,14 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2023 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.InstancesAddMetadata do + use Ecto.Migration + + def change do + alter table(:instances) do + add(:metadata, :map) + add(:metadata_updated_at, :utc_datetime) + end + end +end diff --git a/test/fixtures/mastodon-nodeinfo20.json b/test/fixtures/mastodon-nodeinfo20.json new file mode 100644 index 000000000..35010fdf0 --- /dev/null +++ b/test/fixtures/mastodon-nodeinfo20.json @@ -0,0 +1 @@ +{"version":"2.0","software":{"name":"mastodon","version":"4.1.0"},"protocols":["activitypub"],"services":{"outbound":[],"inbound":[]},"usage":{"users":{"total":971090,"activeMonth":167218,"activeHalfyear":384808},"localPosts":52071541},"openRegistrations":true,"metadata":{}}
\ No newline at end of file diff --git a/test/fixtures/mastodon-well-known-nodeinfo.json b/test/fixtures/mastodon-well-known-nodeinfo.json new file mode 100644 index 000000000..237d5462a --- /dev/null +++ b/test/fixtures/mastodon-well-known-nodeinfo.json @@ -0,0 +1 @@ +{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://mastodon.example.org/nodeinfo/2.0"}]}
\ No newline at end of file diff --git a/test/fixtures/wildebeest-nodeinfo21.json b/test/fixtures/wildebeest-nodeinfo21.json new file mode 100644 index 000000000..c6af474bf --- /dev/null +++ b/test/fixtures/wildebeest-nodeinfo21.json @@ -0,0 +1 @@ +{"version":"2.1","software":{"name":"wildebeest","version":"0.0.1","repository":"https://github.com/cloudflare/wildebeest"},"protocols":["activitypub"],"usage":{"users":{"total":1,"activeMonth":1,"activeHalfyear":1}},"openRegistrations":false,"metadata":{"upstream":{"name":"mastodon","version":"3.5.1"}}}
\ No newline at end of file diff --git a/test/fixtures/wildebeest-well-known-nodeinfo.json b/test/fixtures/wildebeest-well-known-nodeinfo.json new file mode 100644 index 000000000..c7ddb43af --- /dev/null +++ b/test/fixtures/wildebeest-well-known-nodeinfo.json @@ -0,0 +1 @@ +{"links":[{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.0","href":"https://wildebeest.example.org/nodeinfo/2.0"},{"rel":"http://nodeinfo.diaspora.software/ns/schema/2.1","href":"https://wildebeest.example.org/nodeinfo/2.1"}]}
\ No newline at end of file diff --git a/test/pleroma/instances/instance_test.exs b/test/pleroma/instances/instance_test.exs index 861519bce..a769f9362 100644 --- a/test/pleroma/instances/instance_test.exs +++ b/test/pleroma/instances/instance_test.exs @@ -161,6 +161,66 @@ defmodule Pleroma.Instances.InstanceTest do end end + describe "get_or_update_metadata/1" do + test "Scrapes Wildebeest NodeInfo" do + Tesla.Mock.mock(fn + %{url: "https://wildebeest.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/wildebeest-well-known-nodeinfo.json") + } + + %{url: "https://wildebeest.example.org/nodeinfo/2.1"} -> + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/wildebeest-nodeinfo21.json") + } + end) + + expected = %{ + software_name: "wildebeest", + software_repository: "https://github.com/cloudflare/wildebeest", + software_version: "0.0.1" + } + + assert expected == + Instance.get_or_update_metadata(URI.parse("https://wildebeest.example.org/")) + + expected = %Pleroma.Instances.Instance.Pleroma.Instances.Metadata{ + software_name: "wildebeest", + software_repository: "https://github.com/cloudflare/wildebeest", + software_version: "0.0.1" + } + + assert expected == + Repo.get_by(Pleroma.Instances.Instance, %{host: "wildebeest.example.org"}).metadata + end + + test "Scrapes Mastodon NodeInfo" do + Tesla.Mock.mock(fn + %{url: "https://mastodon.example.org/.well-known/nodeinfo"} -> + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/mastodon-well-known-nodeinfo.json") + } + + %{url: "https://mastodon.example.org/nodeinfo/2.0"} -> + %Tesla.Env{ + status: 200, + body: File.read!("test/fixtures/mastodon-nodeinfo20.json") + } + end) + + expected = %{ + software_name: "mastodon", + software_version: "4.1.0" + } + + assert expected == + Instance.get_or_update_metadata(URI.parse("https://mastodon.example.org/")) + end + end + test "delete_users_and_activities/1 deletes remote instance users and activities" do [mario, luigi, _peach, wario] = users = [ diff --git a/test/pleroma/user/backup_test.exs b/test/pleroma/user/backup_test.exs index 5c9b94000..066bf6ba8 100644 --- a/test/pleroma/user/backup_test.exs +++ b/test/pleroma/user/backup_test.exs @@ -39,7 +39,7 @@ defmodule Pleroma.User.BackupTest do assert_enqueued(worker: BackupWorker, args: args) backup = Backup.get(args["backup_id"]) - assert %Backup{user_id: ^user_id, processed: false, file_size: 0} = backup + assert %Backup{user_id: ^user_id, processed: false, file_size: 0, state: :pending} = backup end test "it return an error if the export limit is over" do @@ -59,7 +59,30 @@ defmodule Pleroma.User.BackupTest do assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user) assert {:ok, backup} = perform_job(BackupWorker, args) assert backup.file_size > 0 - assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id} = backup + assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id, state: :complete} = backup + + delete_job_args = %{"op" => "delete", "backup_id" => backup_id} + + assert_enqueued(worker: BackupWorker, args: delete_job_args) + assert {:ok, backup} = perform_job(BackupWorker, delete_job_args) + refute Backup.get(backup_id) + + email = Pleroma.Emails.UserEmail.backup_is_ready_email(backup) + + assert_email_sent( + to: {user.name, user.email}, + html_body: email.html_body + ) + end + + test "it updates states of the backup" do + clear_config([Pleroma.Upload, :uploader], Pleroma.Uploaders.Local) + %{id: user_id} = user = insert(:user) + + assert {:ok, %Oban.Job{args: %{"backup_id" => backup_id} = args}} = Backup.create(user) + assert {:ok, backup} = perform_job(BackupWorker, args) + assert backup.file_size > 0 + assert %Backup{id: ^backup_id, processed: true, user_id: ^user_id, state: :complete} = backup delete_job_args = %{"op" => "delete", "backup_id" => backup_id} @@ -148,7 +171,7 @@ defmodule Pleroma.User.BackupTest do Bookmark.create(user.id, status3.id) assert {:ok, backup} = user |> Backup.new() |> Repo.insert() - assert {:ok, path} = Backup.export(backup) + assert {:ok, path} = Backup.export(backup, self()) assert {:ok, zipfile} = :zip.zip_open(String.to_charlist(path), [:memory]) assert {:ok, {'actor.json', json}} = :zip.zip_get('actor.json', zipfile) @@ -230,6 +253,73 @@ defmodule Pleroma.User.BackupTest do File.rm!(path) end + test "it counts the correct number processed" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + Enum.map(1..120, fn i -> + {:ok, status} = CommonAPI.post(user, %{status: "status #{i}"}) + CommonAPI.favorite(user, status.id) + Bookmark.create(user.id, status.id) + end) + + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + {:ok, backup} = Backup.process(backup) + + assert backup.processed_number == 1 + 120 + 120 + 120 + + Backup.delete(backup) + end + + test "it handles errors" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + Enum.map(1..120, fn i -> + {:ok, _status} = CommonAPI.post(user, %{status: "status #{i}"}) + end) + + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + + with_mock Pleroma.Web.ActivityPub.Transmogrifier, + [:passthrough], + prepare_outgoing: fn data -> + object = + data["object"] + |> Pleroma.Object.normalize(fetch: false) + |> Map.get(:data) + + data = data |> Map.put("object", object) + + if String.contains?(data["object"]["content"], "119"), + do: raise(%Postgrex.Error{}), + else: {:ok, data} + end do + {:ok, backup} = Backup.process(backup) + assert backup.processed + assert backup.state == :complete + assert backup.processed_number == 1 + 119 + + Backup.delete(backup) + end + end + + test "it handles unrecoverable exceptions" do + user = insert(:user, %{nickname: "cofe", name: "Cofe", ap_id: "http://cofe.io/users/cofe"}) + + assert {:ok, backup} = user |> Backup.new() |> Repo.insert() + + with_mock Backup, [:passthrough], do_process: fn _, _ -> raise "mock exception" end do + {:error, %{backup: backup, reason: :exit}} = Backup.process(backup) + + assert backup.state == :failed + end + + with_mock Backup, [:passthrough], do_process: fn _, _ -> Process.sleep(:timer.seconds(32)) end do + {:error, %{backup: backup, reason: :timeout}} = Backup.process(backup) + + assert backup.state == :failed + end + end + describe "it uploads and deletes a backup archive" do setup do clear_config([Pleroma.Upload, :base_url], "https://s3.amazonaws.com") @@ -246,7 +336,7 @@ defmodule Pleroma.User.BackupTest do Bookmark.create(user.id, status3.id) assert {:ok, backup} = user |> Backup.new() |> Repo.insert() - assert {:ok, path} = Backup.export(backup) + assert {:ok, path} = Backup.export(backup, self()) [path: path, backup: backup] end diff --git a/test/pleroma/web/activity_pub/activity_pub_test.exs b/test/pleroma/web/activity_pub/activity_pub_test.exs index 54fc6ef0a..1e8c14043 100644 --- a/test/pleroma/web/activity_pub/activity_pub_test.exs +++ b/test/pleroma/web/activity_pub/activity_pub_test.exs @@ -2652,4 +2652,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubTest do {:ok, user} = ActivityPub.make_user_from_ap_id("https://princess.cat/users/mewmew") assert user.name == " " end + + test "pin_data_from_featured_collection will ignore unsupported values" do + assert %{} == + ActivityPub.pin_data_from_featured_collection(%{ + "type" => "OrderedCollection", + "first" => "https://social.example/users/alice/collections/featured?page=true" + }) + end end diff --git a/test/pleroma/web/admin_api/controllers/report_controller_test.exs b/test/pleroma/web/admin_api/controllers/report_controller_test.exs index c141cf69d..fb2579a3d 100644 --- a/test/pleroma/web/admin_api/controllers/report_controller_test.exs +++ b/test/pleroma/web/admin_api/controllers/report_controller_test.exs @@ -123,6 +123,7 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do }) %{ + reporter: reporter, id: report_id, second_report_id: second_report_id } @@ -266,6 +267,26 @@ defmodule Pleroma.Web.AdminAPI.ReportControllerTest do assert ModerationLog.get_log_entry_message(second_log_entry) == "@#{admin.nickname} updated report ##{second_report_id} (on user @#{second_activity.user_actor.nickname}) with 'closed' state" end + + test "works if reporter is deactivated", %{ + conn: conn, + id: id, + reporter: reporter + } do + Pleroma.User.set_activation(reporter, false) + + conn + |> put_req_header("content-type", "application/json") + |> patch("/api/pleroma/admin/reports", %{ + "reports" => [ + %{"state" => "resolved", "id" => id} + ] + }) + |> json_response_and_validate_schema(:no_content) + + activity = Activity.get_by_id_with_user_actor(id) + assert activity.data["state"] == "resolved" + end end describe "GET /api/pleroma/admin/reports" do diff --git a/test/pleroma/web/mastodon_api/update_credentials_test.exs b/test/pleroma/web/mastodon_api/update_credentials_test.exs index 6c63d53c2..45412bb34 100644 --- a/test/pleroma/web/mastodon_api/update_credentials_test.exs +++ b/test/pleroma/web/mastodon_api/update_credentials_test.exs @@ -97,6 +97,42 @@ defmodule Pleroma.Web.MastodonAPI.UpdateCredentialsTest do assert user.raw_bio == raw_bio end + test "updating bio honours bio limit", %{conn: conn} do + bio_limit = Config.get([:instance, :user_bio_length], 5000) + + raw_bio = String.duplicate(".", bio_limit + 1) + + conn = patch(conn, "/api/v1/accounts/update_credentials", %{"note" => raw_bio}) + + assert %{"error" => "Bio is too long"} = json_response_and_validate_schema(conn, 413) + end + + test "updating name honours name limit", %{conn: conn} do + name_limit = Config.get([:instance, :user_name_length], 100) + + name = String.duplicate(".", name_limit + 1) + + conn = patch(conn, "/api/v1/accounts/update_credentials", %{"display_name" => name}) + + assert %{"error" => "Name is too long"} = json_response_and_validate_schema(conn, 413) + end + + test "when both name and bio exceeds the limit, display name error", %{conn: conn} do + name_limit = Config.get([:instance, :user_name_length], 100) + bio_limit = Config.get([:instance, :user_bio_length], 5000) + + name = String.duplicate(".", name_limit + 1) + raw_bio = String.duplicate(".", bio_limit + 1) + + conn = + patch(conn, "/api/v1/accounts/update_credentials", %{ + "display_name" => name, + "note" => raw_bio + }) + + assert %{"error" => "Name is too long"} = json_response_and_validate_schema(conn, 413) + end + test "updates the user's locking status", %{conn: conn} do conn = patch(conn, "/api/v1/accounts/update_credentials", %{locked: "true"}) @@ -595,17 +631,17 @@ defmodule Pleroma.Web.MastodonAPI.UpdateCredentialsTest do fields = [%{"name" => "foo", "value" => long_value}] - assert %{"error" => "Invalid request"} == + assert %{"error" => "One or more field entries are too long"} == conn |> patch("/api/v1/accounts/update_credentials", %{"fields_attributes" => fields}) - |> json_response_and_validate_schema(403) + |> json_response_and_validate_schema(413) fields = [%{"name" => long_name, "value" => "bar"}] - assert %{"error" => "Invalid request"} == + assert %{"error" => "One or more field entries are too long"} == conn |> patch("/api/v1/accounts/update_credentials", %{"fields_attributes" => fields}) - |> json_response_and_validate_schema(403) + |> json_response_and_validate_schema(413) clear_config([:instance, :max_account_fields], 1) @@ -614,10 +650,10 @@ defmodule Pleroma.Web.MastodonAPI.UpdateCredentialsTest do %{"name" => "link", "value" => "cofe.io"} ] - assert %{"error" => "Invalid request"} == + assert %{"error" => "Too many field entries"} == conn |> patch("/api/v1/accounts/update_credentials", %{"fields_attributes" => fields}) - |> json_response_and_validate_schema(403) + |> json_response_and_validate_schema(413) end end diff --git a/test/pleroma/web/pleroma_api/views/backup_view_test.exs b/test/pleroma/web/pleroma_api/views/backup_view_test.exs index a86688bc4..6908463d6 100644 --- a/test/pleroma/web/pleroma_api/views/backup_view_test.exs +++ b/test/pleroma/web/pleroma_api/views/backup_view_test.exs @@ -15,4 +15,43 @@ defmodule Pleroma.Web.PleromaAPI.BackupViewTest do result = BackupView.render("show.json", backup: backup) assert result.id == backup.id end + + test "it renders the state and processed_number" do + user = insert(:user) + backup = Backup.new(user) + + result = BackupView.render("show.json", backup: backup) + assert result.state == to_string(backup.state) + assert result.processed_number == backup.processed_number + end + + test "it renders failed state with legacy records" do + backup = %Backup{ + id: 0, + content_type: "application/zip", + file_name: "dummy", + file_size: 1, + state: :invalid, + processed: true, + processed_number: 1, + inserted_at: NaiveDateTime.utc_now() + } + + result = BackupView.render("show.json", backup: backup) + assert result.state == "complete" + + backup = %Backup{ + id: 0, + content_type: "application/zip", + file_name: "dummy", + file_size: 1, + state: :invalid, + processed: false, + processed_number: 1, + inserted_at: NaiveDateTime.utc_now() + } + + result = BackupView.render("show.json", backup: backup) + assert result.state == "failed" + end end diff --git a/test/pleroma/web/plugs/uploaded_media_plug_test.exs b/test/pleroma/web/plugs/uploaded_media_plug_test.exs index dbf8ca5ec..8323ff6ab 100644 --- a/test/pleroma/web/plugs/uploaded_media_plug_test.exs +++ b/test/pleroma/web/plugs/uploaded_media_plug_test.exs @@ -40,30 +40,4 @@ defmodule Pleroma.Web.Plugs.UploadedMediaPlugTest do &(&1 == {"content-disposition", ~s[inline; filename="\\"cofe\\".gif"]}) ) end - - test "denies access to media if wrong Host", %{ - attachment_url: attachment_url - } do - conn = get(build_conn(), attachment_url) - - assert conn.status == 200 - - new_media_base = "http://media.localhost:8080" - - %{scheme: new_media_scheme, host: new_media_host, port: new_media_port} = - URI.parse(new_media_base) - - clear_config([Pleroma.Upload, :base_url], new_media_base) - - conn = get(build_conn(), attachment_url) - - expected_url = - URI.parse(attachment_url) - |> Map.put(:host, new_media_host) - |> Map.put(:port, new_media_port) - |> Map.put(:scheme, new_media_scheme) - |> URI.to_string() - - assert redirected_to(conn, 302) == expected_url - end end |