diff options
22 files changed, 183 insertions, 40 deletions
diff --git a/config/config.exs b/config/config.exs index 7385fb6c3..d15998715 100644 --- a/config/config.exs +++ b/config/config.exs @@ -274,7 +274,7 @@ config :pleroma, :markup, config :pleroma, :frontend_configurations, pleroma_fe: %{ alwaysShowSubjectInput: true, - background: "/static/aurora_borealis.jpg", + background: "/images/city.jpg", collapseMessageWithSubject: false, disableChat: false, greentext: false, diff --git a/docs/configuration/postgresql.md b/docs/configuration/postgresql.md new file mode 100644 index 000000000..6983fb459 --- /dev/null +++ b/docs/configuration/postgresql.md @@ -0,0 +1,31 @@ +# Optimizing your PostgreSQL performance + +Pleroma performance depends to a large extent on good database performance. The default PostgreSQL settings are mostly fine, but often you can get better performance by changing a few settings. + +You can use [PGTune](https://pgtune.leopard.in.ua) to get recommendations for your setup. If you do, set the "Number of Connections" field to 20, as Pleroma will only use 10 concurrent connections anyway. If you don't, it will give you advice that might even hurt your performance. + +We also recommend not using the "Network Storage" option. + +## Example configurations + +Here are some configuration suggestions for PostgreSQL 10+. + +### 1GB RAM, 1 CPU +``` +shared_buffers = 256MB +effective_cache_size = 768MB +maintenance_work_mem = 64MB +work_mem = 13107kB +``` + +### 2GB RAM, 2 CPU +``` +shared_buffers = 512MB +effective_cache_size = 1536MB +maintenance_work_mem = 128MB +work_mem = 26214kB +max_worker_processes = 2 +max_parallel_workers_per_gather = 1 +max_parallel_workers = 2 +``` + diff --git a/docs/installation/otp_en.md b/docs/installation/otp_en.md index fb99af699..86135cd20 100644 --- a/docs/installation/otp_en.md +++ b/docs/installation/otp_en.md @@ -63,7 +63,7 @@ apt install postgresql-11-rum ``` #### (Optional) Performance configuration -For optimal performance, you may use [PGTune](https://pgtune.leopard.in.ua), don't forget to restart postgresql after editing the configuration +It is encouraged to check [Optimizing your PostgreSQL performance](../configuration/postgresql.md) document, for tips on PostgreSQL tuning. ```sh tab="Alpine" rc-service postgresql restart diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index a34c20343..c99aae44b 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -24,10 +24,7 @@ defmodule Pleroma.Activity.Queries do @spec by_actor(query, String.t()) :: query def by_actor(query \\ Activity, actor) do - from( - activity in query, - where: fragment("(?)->>'actor' = ?", activity.data, ^actor) - ) + from(a in query, where: a.actor == ^actor) end @spec by_author(query, User.t()) :: query diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 557961e94..7eca55ac9 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -92,8 +92,9 @@ defmodule Pleroma.Notification do |> join(:left, [n, a], object in Object, on: fragment( - "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + "(?->>'id') = COALESCE(?->'object'->>'id', ?->>'object')", object.data, + a.data, a.data ) ) @@ -224,18 +225,8 @@ defmodule Pleroma.Notification do |> Marker.multi_set_last_read_id(user, "notifications") |> Repo.transaction() - Notification + for_user_query(user) |> where([n], n.id in ^notification_ids) - |> join(:inner, [n], activity in assoc(n, :activity)) - |> join(:left, [n, a], object in Object, - on: - fragment( - "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", - object.data, - a.data - ) - ) - |> preload([n, a, o], activity: {a, object: o}) |> Repo.all() end diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index f57cd3e74..42c4c4e3e 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1619,12 +1619,19 @@ defmodule Pleroma.User do def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id) def get_or_fetch_by_ap_id(ap_id) do - user = get_cached_by_ap_id(ap_id) + cached_user = get_cached_by_ap_id(ap_id) - if !is_nil(user) and !needs_update?(user) do - {:ok, user} - else - fetch_by_ap_id(ap_id) + maybe_fetched_user = needs_update?(cached_user) && fetch_by_ap_id(ap_id) + + case {cached_user, maybe_fetched_user} do + {_, {:ok, %User{} = user}} -> + {:ok, user} + + {%User{} = user, _} -> + {:ok, user} + + _ -> + {:error, :not_found} end end diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 2cea55285..b8a2873d8 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -538,14 +538,27 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.one() end - @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()] - def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do + @spec fetch_public_or_unlisted_activities(map(), Pagination.type()) :: [Activity.t()] + def fetch_public_or_unlisted_activities(opts \\ %{}, pagination \\ :keyset) do opts = Map.drop(opts, ["user"]) - [Constants.as_public()] - |> fetch_activities_query(opts) - |> restrict_unlisted() - |> Pagination.fetch_paginated(opts, pagination) + query = fetch_activities_query([Constants.as_public()], opts) + + query = + if opts["restrict_unlisted"] do + restrict_unlisted(query) + else + query + end + + Pagination.fetch_paginated(query, opts, pagination) + end + + @spec fetch_public_activities(map(), Pagination.type()) :: [Activity.t()] + def fetch_public_activities(opts \\ %{}, pagination \\ :keyset) do + opts + |> Map.put("restrict_unlisted", true) + |> fetch_public_or_unlisted_activities(pagination) end @valid_visibilities ~w[direct unlisted public private] @@ -1145,7 +1158,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Activity.with_joined_object() |> Object.with_joined_activity() |> select([_like, object, activity], %{activity | object: object}) - |> order_by([like, _, _], desc: like.id) + |> order_by([like, _, _], desc_nulls_last: like.id) |> Pagination.fetch_paginated( Map.merge(params, %{"skip_order" => true}), pagination, diff --git a/lib/pleroma/web/feed/user_controller.ex b/lib/pleroma/web/feed/user_controller.ex index 1b72e23dc..5a6fc9de0 100644 --- a/lib/pleroma/web/feed/user_controller.ex +++ b/lib/pleroma/web/feed/user_controller.ex @@ -56,7 +56,7 @@ defmodule Pleroma.Web.Feed.UserController do "actor_id" => user.ap_id } |> put_if_exist("max_id", params["max_id"]) - |> ActivityPub.fetch_public_activities() + |> ActivityPub.fetch_public_or_unlisted_activities() conn |> put_resp_content_type("application/#{format}+xml") diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 75512442d..47649d41d 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -81,7 +81,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do plug( RateLimiter, - [name: :relation_id_action, params: ["id", "uri"]] when action in @relationship_actions + [name: :relation_id_action, params: [:id, :uri]] when action in @relationship_actions ) plug(RateLimiter, [name: :relations_actions] when action in @relationship_actions) diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex index 83d997abd..f20157a5f 100644 --- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex @@ -84,13 +84,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do plug( RateLimiter, - [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: ["id"]] + [name: :status_id_action, bucket_name: "status_id_action:reblog_unreblog", params: [:id]] when action in ~w(reblog unreblog)a ) plug( RateLimiter, - [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: ["id"]] + [name: :status_id_action, bucket_name: "status_id_action:fav_unfav", params: [:id]] when action in ~w(favourite unfavourite)a ) diff --git a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex index 341eff054..a4c3b9516 100644 --- a/lib/pleroma/workers/cron/clear_oauth_token_worker.ex +++ b/lib/pleroma/workers/cron/clear_oauth_token_worker.ex @@ -16,6 +16,8 @@ defmodule Pleroma.Workers.Cron.ClearOauthTokenWorker do def perform(_opts, _job) do if Config.get([:oauth2, :clean_expired_tokens], false) do Token.delete_expired_tokens() + else + :ok end end end diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index dd13c3b17..7f09ff3cf 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -37,6 +37,8 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do ) |> Repo.all() |> send_emails + else + :ok end end diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 9bd0a5621..5c816b3fe 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -55,7 +55,11 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do |> Repo.all() |> Enum.map(&Pleroma.Emails.NewUsersDigestEmail.new_users(&1, users_and_statuses)) |> Enum.each(&Pleroma.Emails.Mailer.deliver/1) + else + :ok end + else + :ok end end end diff --git a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex index b8953dd7f..84b3b84de 100644 --- a/lib/pleroma/workers/cron/purge_expired_activities_worker.ex +++ b/lib/pleroma/workers/cron/purge_expired_activities_worker.ex @@ -23,6 +23,8 @@ defmodule Pleroma.Workers.Cron.PurgeExpiredActivitiesWorker do def perform(_opts, _job) do if Config.get([ActivityExpiration, :enabled]) do Enum.each(ActivityExpiration.due_expirations(@interval), &delete_activity/1) + else + :ok end end diff --git a/priv/repo/migrations/20200526144426_add_apps_indexes.exs b/priv/repo/migrations/20200526144426_add_apps_indexes.exs new file mode 100644 index 000000000..5cb6a0473 --- /dev/null +++ b/priv/repo/migrations/20200526144426_add_apps_indexes.exs @@ -0,0 +1,7 @@ +defmodule Pleroma.Repo.Migrations.AddAppsIndexes do + use Ecto.Migration + + def change do + create(index(:apps, [:client_id, :client_secret])) + end +end diff --git a/priv/repo/migrations/20200527104138_change_notification_user_index.exs b/priv/repo/migrations/20200527104138_change_notification_user_index.exs new file mode 100644 index 000000000..4dcfe6de9 --- /dev/null +++ b/priv/repo/migrations/20200527104138_change_notification_user_index.exs @@ -0,0 +1,8 @@ +defmodule Pleroma.Repo.Migrations.ChangeNotificationUserIndex do + use Ecto.Migration + + def change do + drop_if_exists(index(:notifications, [:user_id])) + create_if_not_exists(index(:notifications, [:user_id, "id desc nulls last"])) + end +end diff --git a/test/notification_test.exs b/test/notification_test.exs index 3a96721fa..37c255fee 100644 --- a/test/notification_test.exs +++ b/test/notification_test.exs @@ -454,8 +454,7 @@ defmodule Pleroma.NotificationTest do status: "hey again @#{other_user.nickname}!" }) - [n2, n1] = notifs = Notification.for_user(other_user) - assert length(notifs) == 2 + [n2, n1] = Notification.for_user(other_user) assert n2.id > n1.id @@ -464,7 +463,9 @@ defmodule Pleroma.NotificationTest do status: "hey yet again @#{other_user.nickname}!" }) - Notification.set_read_up_to(other_user, n2.id) + [_, read_notification] = Notification.set_read_up_to(other_user, n2.id) + + assert read_notification.activity.object [n3, n2, n1] = Notification.for_user(other_user) @@ -972,7 +973,9 @@ defmodule Pleroma.NotificationTest do {:ok, _activity} = CommonAPI.post(muted, %{status: "hey @#{user.nickname}"}) - assert length(Notification.for_user(user)) == 1 + [notification] = Notification.for_user(user) + + assert notification.activity.object end test "it doesn't return notifications for muted user with notifications" do diff --git a/test/plugs/authentication_plug_test.exs b/test/plugs/authentication_plug_test.exs index 3c70c1747..777ae15ae 100644 --- a/test/plugs/authentication_plug_test.exs +++ b/test/plugs/authentication_plug_test.exs @@ -68,6 +68,7 @@ defmodule Pleroma.Plugs.AuthenticationPlugTest do assert "$pbkdf2" <> _ = user.password_hash end + @tag :skip_on_mac test "with a crypt hash, it updates to a pkbdf2 hash", %{conn: conn} do user = insert(:user, diff --git a/test/support/factory.ex b/test/support/factory.ex index d4284831c..6e3676aca 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -34,7 +34,8 @@ defmodule Pleroma.Factory do last_digest_emailed_at: NaiveDateTime.utc_now(), last_refreshed_at: NaiveDateTime.utc_now(), notification_settings: %Pleroma.User.NotificationSetting{}, - multi_factor_authentication_settings: %Pleroma.MFA.Settings{} + multi_factor_authentication_settings: %Pleroma.MFA.Settings{}, + ap_enabled: true } %{ diff --git a/test/user_test.exs b/test/user_test.exs index 45125f704..3556ef1b4 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -586,6 +586,26 @@ defmodule Pleroma.UserTest do refute user.last_refreshed_at == orig_user.last_refreshed_at end + + @tag capture_log: true + test "it returns the old user if stale, but unfetchable" do + a_week_ago = NaiveDateTime.add(NaiveDateTime.utc_now(), -604_800) + + orig_user = + insert( + :user, + local: false, + nickname: "admin@mastodon.example.org", + ap_id: "http://mastodon.example.org/users/raymoo", + last_refreshed_at: a_week_ago + ) + + assert orig_user.last_refreshed_at == a_week_ago + + {:ok, user} = User.get_or_fetch_by_ap_id("http://mastodon.example.org/users/raymoo") + + assert user.last_refreshed_at == orig_user.last_refreshed_at + end end test "returns an ap_id for a user" do diff --git a/test/web/activity_pub/activity_pub_controller_test.exs b/test/web/activity_pub/activity_pub_controller_test.exs index dd2a48a61..24edab41a 100644 --- a/test/web/activity_pub/activity_pub_controller_test.exs +++ b/test/web/activity_pub/activity_pub_controller_test.exs @@ -451,6 +451,36 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubControllerTest do assert Activity.get_by_ap_id(data["id"]) end + @tag capture_log: true + test "it inserts an incoming activity into the database" <> + "even if we can't fetch the user but have it in our db", + %{conn: conn} do + user = + insert(:user, + ap_id: "https://mastodon.example.org/users/raymoo", + ap_enabled: true, + local: false, + last_refreshed_at: nil + ) + + data = + File.read!("test/fixtures/mastodon-post-activity.json") + |> Poison.decode!() + |> Map.put("actor", user.ap_id) + |> put_in(["object", "attridbutedTo"], user.ap_id) + + conn = + conn + |> assign(:valid_signature, true) + |> put_req_header("content-type", "application/activity+json") + |> post("/inbox", data) + + assert "ok" == json_response(conn, 200) + + ObanHelpers.perform(all_enqueued(worker: ReceiverWorker)) + assert Activity.get_by_ap_id(data["id"]) + end + test "it clears `unreachable` federation status of the sender", %{conn: conn} do data = File.read!("test/fixtures/mastodon-post-activity.json") |> Poison.decode!() diff --git a/test/web/feed/user_controller_test.exs b/test/web/feed/user_controller_test.exs index 05ad427c2..fa2ed1ea5 100644 --- a/test/web/feed/user_controller_test.exs +++ b/test/web/feed/user_controller_test.exs @@ -11,13 +11,14 @@ defmodule Pleroma.Web.Feed.UserControllerTest do alias Pleroma.Config alias Pleroma.Object alias Pleroma.User + alias Pleroma.Web.CommonAPI setup do: clear_config([:instance, :federating], true) describe "feed" do setup do: clear_config([:feed]) - test "gets a feed", %{conn: conn} do + test "gets an atom feed", %{conn: conn} do Config.put( [:feed, :post_title], %{max_length: 10, omission: "..."} @@ -157,6 +158,29 @@ defmodule Pleroma.Web.Feed.UserControllerTest do assert response(conn, 404) end + + test "returns feed with public and unlisted activities", %{conn: conn} do + user = insert(:user) + + {:ok, _} = CommonAPI.post(user, %{status: "public", visibility: "public"}) + {:ok, _} = CommonAPI.post(user, %{status: "direct", visibility: "direct"}) + {:ok, _} = CommonAPI.post(user, %{status: "unlisted", visibility: "unlisted"}) + {:ok, _} = CommonAPI.post(user, %{status: "private", visibility: "private"}) + + resp = + conn + |> put_req_header("accept", "application/atom+xml") + |> get(user_feed_path(conn, :feed, user.nickname)) + |> response(200) + + activity_titles = + resp + |> SweetXml.parse() + |> SweetXml.xpath(~x"//entry/title/text()"l) + |> Enum.sort() + + assert activity_titles == ['public', 'unlisted'] + end end # Note: see ActivityPubControllerTest for JSON format tests |