From 3bd2829e5c125f961b7508bf40ef534a21070562 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 23 Mar 2020 18:56:01 +0100 Subject: Benchmarks: Add timeline benchmark --- benchmarks/load_testing/generator.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/generator.ex b/benchmarks/load_testing/generator.ex index 3f88fefd7..17e89c13c 100644 --- a/benchmarks/load_testing/generator.ex +++ b/benchmarks/load_testing/generator.ex @@ -22,9 +22,10 @@ defmodule Pleroma.LoadTesting.Generator do def generate_users(opts) do IO.puts("Starting generating #{opts[:users_max]} users...") - {time, _} = :timer.tc(fn -> do_generate_users(opts) end) + {time, users} = :timer.tc(fn -> do_generate_users(opts) end) IO.puts("Inserting users take #{to_sec(time)} sec.\n") + users end defp do_generate_users(opts) do -- cgit v1.2.3 From c8475cd5c63af18471864fe57504999ddd09e496 Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 25 Mar 2020 15:48:15 +0000 Subject: Apply suggestion to benchmarks/load_testing/generator.ex --- benchmarks/load_testing/generator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/generator.ex b/benchmarks/load_testing/generator.ex index 17e89c13c..e4673757c 100644 --- a/benchmarks/load_testing/generator.ex +++ b/benchmarks/load_testing/generator.ex @@ -24,7 +24,7 @@ defmodule Pleroma.LoadTesting.Generator do IO.puts("Starting generating #{opts[:users_max]} users...") {time, users} = :timer.tc(fn -> do_generate_users(opts) end) - IO.puts("Inserting users take #{to_sec(time)} sec.\n") + IO.puts("Inserting users took #{to_sec(time)} sec.\n") users end -- cgit v1.2.3 From eb9744cadea7191b088ddaadfbd5fa4d4fd45090 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 14 Jan 2020 14:42:30 +0300 Subject: activities generation tasks --- benchmarks/load_testing/activities.ex | 515 +++++++++++++++++++++++++ benchmarks/load_testing/fetcher.ex | 683 +++++++++++++++++++++++----------- benchmarks/load_testing/generator.ex | 410 -------------------- benchmarks/load_testing/helper.ex | 10 +- benchmarks/load_testing/users.ex | 161 ++++++++ 5 files changed, 1133 insertions(+), 646 deletions(-) create mode 100644 benchmarks/load_testing/activities.ex delete mode 100644 benchmarks/load_testing/generator.ex create mode 100644 benchmarks/load_testing/users.ex (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex new file mode 100644 index 000000000..db0e5a66f --- /dev/null +++ b/benchmarks/load_testing/activities.ex @@ -0,0 +1,515 @@ +defmodule Pleroma.LoadTesting.Activities do + @moduledoc """ + Module for generating different activities. + """ + import Ecto.Query + import Pleroma.LoadTesting.Helper, only: [to_sec: 1] + + alias Ecto.UUID + alias Pleroma.Constants + alias Pleroma.LoadTesting.Users + alias Pleroma.Repo + alias Pleroma.Web.CommonAPI + + require Constants + + @defaults [ + iterations: 170, + friends_used: 20, + non_friends_used: 20 + ] + + @max_concurrency 30 + + @visibility ~w(public private direct unlisted) + @types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote) + @groups ~w(user friends non_friends) + + @spec generate(User.t(), keyword()) :: :ok + def generate(user, opts \\ []) do + {:ok, _} = + Agent.start_link(fn -> %{} end, + name: :benchmark_state + ) + + opts = Keyword.merge(@defaults, opts) + + friends = + user + |> Users.get_users(limit: opts[:friends_used], local: :local, friends?: true) + |> Enum.shuffle() + + non_friends = + user + |> Users.get_users(limit: opts[:non_friends_used], local: :local, friends?: false) + |> Enum.shuffle() + + task_data = + for visibility <- @visibility, + type <- @types, + group <- @groups, + do: {visibility, type, group} + + IO.puts("Starting generating #{opts[:iterations]} iterations of activities...") + + friends_thread = Enum.take(friends, 5) + non_friends_thread = Enum.take(friends, 5) + + public_long_thread = fn -> + generate_long_thread("public", user, friends_thread, non_friends_thread, opts) + end + + private_long_thread = fn -> + generate_long_thread("private", user, friends_thread, non_friends_thread, opts) + end + + iterations = opts[:iterations] + + {time, _} = + :timer.tc(fn -> + Enum.each( + 1..iterations, + fn + i when i == iterations - 2 -> + spawn(public_long_thread) + spawn(private_long_thread) + generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) + + _ -> + generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) + end + ) + end) + + IO.puts("Generating iterations activities take #{to_sec(time)} sec.\n") + :ok + end + + defp generate_long_thread(visibility, user, friends, non_friends, _opts) do + group = + if visibility == "public", + do: "friends", + else: "user" + + tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Start of #{visibility} long thread", + "visibility" => visibility + }) + + Agent.update(:benchmark_state, fn state -> + key = + if visibility == "public", + do: :public_thread, + else: :private_thread + + Map.put(state, key, activity) + end) + + acc = {activity.id, ["@" <> user.nickname, "reply to long thread"]} + insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) + IO.puts("Generating #{visibility} long thread ended\n") + end + + defp insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) do + Enum.reduce(tasks, acc, fn + "friend", {id, data} -> + friend = Enum.random(friends) + insert_reply(friend, List.delete(data, "@" <> friend.nickname), id, visibility) + + "non_friend", {id, data} -> + non_friend = Enum.random(non_friends) + insert_reply(non_friend, List.delete(data, "@" <> non_friend.nickname), id, visibility) + + "user", {id, data} -> + insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility) + end) + end + + defp generate_activities(user, friends, non_friends, task_data, opts) do + Task.async_stream( + task_data, + fn {visibility, type, group} -> + insert_activity(type, visibility, group, user, friends, non_friends, opts) + end, + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end + + defp insert_activity("simple", visibility, group, user, friends, non_friends, _opts) do + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{"status" => "Simple status", "visibility" => visibility}) + end + + defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{ + "status" => "Simple status with emoji :firefox:", + "visibility" => visibility + }) + end + + defp insert_activity("mentions", visibility, group, user, friends, non_friends, _opts) do + user_mentions = + get_random_mentions(friends, Enum.random(0..3)) ++ + get_random_mentions(non_friends, Enum.random(0..3)) + + user_mentions = + if Enum.random([true, false]), + do: ["@" <> user.nickname | user_mentions], + else: user_mentions + + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{ + "status" => Enum.join(user_mentions, ", ") <> " simple status with mentions", + "visibility" => visibility + }) + end + + defp insert_activity("hell_thread", visibility, group, user, friends, non_friends, _opts) do + mentions = + with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do + cached = + ([user | Enum.take(friends, 10)] ++ Enum.take(non_friends, 10)) + |> Enum.map(&"@#{&1.nickname}") + |> Enum.join(", ") + + Cachex.put(:user_cache, "hell_thread_mentions", cached) + cached + else + {:ok, cached} -> cached + end + + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{ + "status" => mentions <> " hell thread status", + "visibility" => visibility + }) + end + + defp insert_activity("attachment", visibility, group, user, friends, non_friends, _opts) do + actor = get_actor(group, user, friends, non_friends) + + obj_data = %{ + "actor" => actor.ap_id, + "name" => "4467-11.jpg", + "type" => "Document", + "url" => [ + %{ + "href" => + "#{Pleroma.Web.base_url()}/media/b1b873552422a07bf53af01f3c231c841db4dfc42c35efde681abaf0f2a4eab7.jpg", + "mediaType" => "image/jpeg", + "type" => "Link" + } + ] + } + + object = Repo.insert!(%Pleroma.Object{data: obj_data}) + + {:ok, _activity} = + CommonAPI.post(actor, %{ + "status" => "Post with attachment", + "visibility" => visibility, + "media_ids" => [object.id] + }) + end + + defp insert_activity("tag", visibility, group, user, friends, non_friends, _opts) do + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{"status" => "Status with #tag", "visibility" => visibility}) + end + + defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do + actor = get_actor(group, user, friends, non_friends) + + with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), + {:ok, _activity, _object} <- CommonAPI.favorite(activity_id, actor) do + :ok + else + {:error, _} -> + insert_activity("like", visibility, group, user, friends, non_friends, opts) + + nil -> + Process.sleep(15) + insert_activity("like", visibility, group, user, friends, non_friends, opts) + end + end + + defp insert_activity("reblog", visibility, group, user, friends, non_friends, opts) do + actor = get_actor(group, user, friends, non_friends) + + with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), + {:ok, _activity, _object} <- CommonAPI.repeat(activity_id, actor) do + :ok + else + {:error, _} -> + insert_activity("reblog", visibility, group, user, friends, non_friends, opts) + + nil -> + Process.sleep(15) + insert_activity("reblog", visibility, group, user, friends, non_friends, opts) + end + end + + defp insert_activity("simple_thread", visibility, group, user, friends, non_friends, _opts) + when visibility in ["public", "unlisted", "private"] do + actor = get_actor(group, user, friends, non_friends) + tasks = get_reply_tasks(visibility, group) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "Simple status", "visibility" => "unlisted"}) + + acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} + insert_replies(tasks, visibility, user, friends, non_friends, acc) + end + + defp insert_activity("simple_thread", "direct", group, user, friends, non_friends, _opts) do + actor = get_actor(group, user, friends, non_friends) + tasks = get_reply_tasks("direct", group) + + list = + case group do + "non_friends" -> + Enum.take(non_friends, 3) + + _ -> + Enum.take(friends, 3) + end + + data = Enum.map(list, &("@" <> &1.nickname)) + + {:ok, activity} = + CommonAPI.post(actor, %{ + "status" => Enum.join(data, ", ") <> "simple status", + "visibility" => "direct" + }) + + acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]} + insert_direct_replies(tasks, user, list, acc) + end + + defp insert_activity("remote", _, "user", _, _, _, _), do: :ok + + defp insert_activity("remote", visibility, group, user, _friends, _non_friends, opts) do + remote_friends = + Users.get_users(user, limit: opts[:friends_used], local: :external, friends?: true) + + remote_non_friends = + Users.get_users(user, limit: opts[:non_friends_used], local: :external, friends?: false) + + actor = get_actor(group, user, remote_friends, remote_non_friends) + + {act_data, obj_data} = prepare_activity_data(actor, visibility, user) + {activity_data, object_data} = other_data(actor) + + activity_data + |> Map.merge(act_data) + |> Map.put("object", Map.merge(object_data, obj_data)) + |> Pleroma.Web.ActivityPub.ActivityPub.insert(false) + end + + defp get_actor("user", user, _friends, _non_friends), do: user + defp get_actor("friends", _user, friends, _non_friends), do: Enum.random(friends) + defp get_actor("non_friends", _user, _friends, non_friends), do: Enum.random(non_friends) + + defp other_data(actor) do + %{host: host} = URI.parse(actor.ap_id) + datetime = DateTime.utc_now() + context_id = "http://#{host}:4000/contexts/#{UUID.generate()}" + activity_id = "http://#{host}:4000/activities/#{UUID.generate()}" + object_id = "http://#{host}:4000/objects/#{UUID.generate()}" + + activity_data = %{ + "actor" => actor.ap_id, + "context" => context_id, + "id" => activity_id, + "published" => datetime, + "type" => "Create", + "directMessage" => false + } + + object_data = %{ + "actor" => actor.ap_id, + "attachment" => [], + "attributedTo" => actor.ap_id, + "bcc" => [], + "bto" => [], + "content" => "Remote post", + "context" => context_id, + "conversation" => context_id, + "emoji" => %{}, + "id" => object_id, + "published" => datetime, + "sensitive" => false, + "summary" => "", + "tag" => [], + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Note" + } + + {activity_data, object_data} + end + + defp prepare_activity_data(actor, "public", _mention) do + obj_data = %{ + "cc" => [actor.follower_address], + "to" => [Constants.as_public()] + } + + act_data = %{ + "cc" => [actor.follower_address], + "to" => [Constants.as_public()] + } + + {act_data, obj_data} + end + + defp prepare_activity_data(actor, "private", _mention) do + obj_data = %{ + "cc" => [], + "to" => [actor.follower_address] + } + + act_data = %{ + "cc" => [], + "to" => [actor.follower_address] + } + + {act_data, obj_data} + end + + defp prepare_activity_data(actor, "unlisted", _mention) do + obj_data = %{ + "cc" => [Constants.as_public()], + "to" => [actor.follower_address] + } + + act_data = %{ + "cc" => [Constants.as_public()], + "to" => [actor.follower_address] + } + + {act_data, obj_data} + end + + defp prepare_activity_data(_actor, "direct", mention) do + %{host: mentioned_host} = URI.parse(mention.ap_id) + + obj_data = %{ + "cc" => [], + "content" => + "@#{ + mention.nickname + } direct message", + "tag" => [ + %{ + "href" => mention.ap_id, + "name" => "@#{mention.nickname}@#{mentioned_host}", + "type" => "Mention" + } + ], + "to" => [mention.ap_id] + } + + act_data = %{ + "cc" => [], + "directMessage" => true, + "to" => [mention.ap_id] + } + + {act_data, obj_data} + end + + defp get_reply_tasks("public", "user"), do: ~w(friend non_friend user) + defp get_reply_tasks("public", "friends"), do: ~w(non_friend user friend) + defp get_reply_tasks("public", "non_friends"), do: ~w(user friend non_friend) + + defp get_reply_tasks(visibility, "user") when visibility in ["unlisted", "private"], + do: ~w(friend user friend) + + defp get_reply_tasks(visibility, "friends") when visibility in ["unlisted", "private"], + do: ~w(user friend user) + + defp get_reply_tasks(visibility, "non_friends") when visibility in ["unlisted", "private"], + do: [] + + defp get_reply_tasks("direct", "user"), do: ~w(friend user friend) + defp get_reply_tasks("direct", "friends"), do: ~w(user friend user) + defp get_reply_tasks("direct", "non_friends"), do: ~w(user non_friend user) + + defp insert_replies(tasks, visibility, user, friends, non_friends, acc) do + Enum.reduce(tasks, acc, fn + "friend", {id, data} -> + friend = Enum.random(friends) + insert_reply(friend, data, id, visibility) + + "non_friend", {id, data} -> + non_friend = Enum.random(non_friends) + insert_reply(non_friend, data, id, visibility) + + "user", {id, data} -> + insert_reply(user, data, id, visibility) + end) + end + + defp insert_direct_replies(tasks, user, list, acc) do + Enum.reduce(tasks, acc, fn + group, {id, data} when group in ["friend", "non_friend"] -> + actor = Enum.random(list) + + {reply_id, _} = + insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct") + + {reply_id, data} + + "user", {id, data} -> + {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct") + {reply_id, data} + end) + end + + defp insert_reply(actor, data, activity_id, visibility) do + {:ok, reply} = + CommonAPI.post(actor, %{ + "status" => Enum.join(data, ", "), + "visibility" => visibility, + "in_reply_to_status_id" => activity_id + }) + + {reply.id, ["@" <> actor.nickname | data]} + end + + defp get_random_mentions(_users, count) when count == 0, do: [] + + defp get_random_mentions(users, count) do + users + |> Enum.shuffle() + |> Enum.take(count) + |> Enum.map(&"@#{&1.nickname}") + end + + defp get_random_create_activity_id do + Repo.one( + from(a in Pleroma.Activity, + where: fragment("(?)->>'type' = ?", a.data, ^"Create"), + order_by: fragment("RANDOM()"), + limit: 1, + select: a.id + ) + ) + end +end diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index a45a71d4a..bd65ac84f 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -1,260 +1,489 @@ defmodule Pleroma.LoadTesting.Fetcher do - use Pleroma.LoadTesting.Helper - - def fetch_user(user) do - Benchee.run(%{ - "By id" => fn -> Repo.get_by(User, id: user.id) end, - "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end, - "By email" => fn -> Repo.get_by(User, email: user.email) end, - "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end - }) + alias Pleroma.Activity + alias Pleroma.Pagination + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.Web.ActivityPub.ActivityPub + alias Pleroma.Web.MastodonAPI.MastodonAPI + alias Pleroma.Web.MastodonAPI.StatusView + + @spec run_benchmarks(User.t()) :: any() + def run_benchmarks(user) do + fetch_user(user) + fetch_timelines(user) + render_views(user) end - def query_timelines(user) do - home_timeline_params = %{ - "count" => 20, - "with_muted" => true, - "type" => ["Create", "Announce"], + defp formatters do + [ + Benchee.Formatters.Console + ] + end + + defp fetch_user(user) do + Benchee.run( + %{ + "By id" => fn -> Repo.get_by(User, id: user.id) end, + "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end, + "By email" => fn -> Repo.get_by(User, email: user.email) end, + "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end + }, + formatters: formatters() + ) + end + + defp fetch_timelines(user) do + fetch_home_timeline(user) + fetch_direct_timeline(user) + fetch_public_timeline(user) + fetch_public_timeline(user, :local) + fetch_public_timeline(user, :tag) + fetch_notifications(user) + fetch_favourites(user) + fetch_long_thread(user) + end + + defp render_views(user) do + render_timelines(user) + render_long_thread(user) + end + + defp opts_for_home_timeline(user) do + %{ "blocking_user" => user, + "count" => "20", "muting_user" => user, - "user" => user + "type" => ["Create", "Announce"], + "user" => user, + "with_muted" => "true" } + end - mastodon_public_timeline_params = %{ - "count" => 20, - "local_only" => true, - "only_media" => "false", + defp fetch_home_timeline(user) do + opts = opts_for_home_timeline(user) + + recipients = [user.ap_id | User.following(user)] + + first_page_last = + ActivityPub.fetch_activities(recipients, opts) |> Enum.reverse() |> List.last() + + second_page_last = + ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", first_page_last.id)) + |> Enum.reverse() + |> List.last() + + third_page_last = + ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", second_page_last.id)) + |> Enum.reverse() + |> List.last() + + forth_page_last = + ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", third_page_last.id)) + |> Enum.reverse() + |> List.last() + + Benchee.run( + %{ + "home timeline" => fn opts -> ActivityPub.fetch_activities(recipients, opts) end + }, + inputs: %{ + "1 page" => opts, + "2 page" => Map.put(opts, "max_id", first_page_last.id), + "3 page" => Map.put(opts, "max_id", second_page_last.id), + "4 page" => Map.put(opts, "max_id", third_page_last.id), + "5 page" => Map.put(opts, "max_id", forth_page_last.id), + "1 page only media" => Map.put(opts, "only_media", "true"), + "2 page only media" => + Map.put(opts, "max_id", first_page_last.id) |> Map.put("only_media", "true"), + "3 page only media" => + Map.put(opts, "max_id", second_page_last.id) |> Map.put("only_media", "true"), + "4 page only media" => + Map.put(opts, "max_id", third_page_last.id) |> Map.put("only_media", "true"), + "5 page only media" => + Map.put(opts, "max_id", forth_page_last.id) |> Map.put("only_media", "true") + }, + formatters: formatters() + ) + end + + defp opts_for_direct_timeline(user) do + %{ + :visibility => "direct", + "blocking_user" => user, + "count" => "20", + "type" => "Create", + "user" => user, + "with_muted" => "true" + } + end + + defp fetch_direct_timeline(user) do + recipients = [user.ap_id] + + opts = opts_for_direct_timeline(user) + + first_page_last = + recipients + |> ActivityPub.fetch_activities_query(opts) + |> Pagination.fetch_paginated(opts) + |> List.last() + + opts2 = Map.put(opts, "max_id", first_page_last.id) + + second_page_last = + recipients + |> ActivityPub.fetch_activities_query(opts2) + |> Pagination.fetch_paginated(opts2) + |> List.last() + + opts3 = Map.put(opts, "max_id", second_page_last.id) + + third_page_last = + recipients + |> ActivityPub.fetch_activities_query(opts3) + |> Pagination.fetch_paginated(opts3) + |> List.last() + + opts4 = Map.put(opts, "max_id", third_page_last.id) + + forth_page_last = + recipients + |> ActivityPub.fetch_activities_query(opts4) + |> Pagination.fetch_paginated(opts4) + |> List.last() + + Benchee.run( + %{ + "direct timeline" => fn opts -> + ActivityPub.fetch_activities_query(recipients, opts) |> Pagination.fetch_paginated(opts) + end + }, + inputs: %{ + "1 page" => opts, + "2 page" => opts2, + "3 page" => opts3, + "4 page" => opts4, + "5 page" => Map.put(opts4, "max_id", forth_page_last.id) + }, + formatters: formatters() + ) + end + + defp opts_for_public_timeline(user) do + %{ "type" => ["Create", "Announce"], - "with_muted" => "true", + "local_only" => false, "blocking_user" => user, "muting_user" => user } + end - mastodon_federated_timeline_params = %{ - "count" => 20, - "only_media" => "false", + defp opts_for_public_timeline(user, :local) do + %{ "type" => ["Create", "Announce"], - "with_muted" => "true", + "local_only" => true, "blocking_user" => user, "muting_user" => user } + end - following = User.following(user) - - Benchee.run(%{ - "User home timeline" => fn -> - Pleroma.Web.ActivityPub.ActivityPub.fetch_activities( - following, - home_timeline_params - ) - end, - "User mastodon public timeline" => fn -> - Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( - mastodon_public_timeline_params - ) - end, - "User mastodon federated public timeline" => fn -> - Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( - mastodon_federated_timeline_params - ) - end - }) - - home_activities = - Pleroma.Web.ActivityPub.ActivityPub.fetch_activities( - following, - home_timeline_params - ) + defp opts_for_public_timeline(user, :tag) do + %{ + "blocking_user" => user, + "count" => "20", + "local_only" => nil, + "muting_user" => user, + "tag" => ["tag"], + "tag_all" => [], + "tag_reject" => [], + "type" => "Create", + "user" => user, + "with_muted" => "true" + } + end - public_activities = - Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params) + defp fetch_public_timeline(user) do + opts = opts_for_public_timeline(user) - public_federated_activities = - Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities( - mastodon_federated_timeline_params - ) + fetch_public_timeline(opts, "public timeline") + end + + defp fetch_public_timeline(user, :local) do + opts = opts_for_public_timeline(user, :local) - Benchee.run(%{ - "Rendering home timeline" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: home_activities, - for: user, - as: :activity - }) - end, - "Rendering public timeline" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: public_activities, - for: user, - as: :activity - }) - end, - "Rendering public federated timeline" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: public_federated_activities, - for: user, - as: :activity - }) - end, - "Rendering favorites timeline" => fn -> - conn = Phoenix.ConnTest.build_conn(:get, "http://localhost:4001/api/v1/favourites", nil) - Pleroma.Web.MastodonAPI.StatusController.favourites( - %Plug.Conn{conn | - assigns: %{user: user}, - query_params: %{"limit" => "0"}, - body_params: %{}, - cookies: %{}, - params: %{}, - path_params: %{}, - private: %{ - Pleroma.Web.Router => {[], %{}}, - phoenix_router: Pleroma.Web.Router, - phoenix_action: :favourites, - phoenix_controller: Pleroma.Web.MastodonAPI.StatusController, - phoenix_endpoint: Pleroma.Web.Endpoint, - phoenix_format: "json", - phoenix_layout: {Pleroma.Web.LayoutView, "app.html"}, - phoenix_recycled: true, - - phoenix_view: Pleroma.Web.MastodonAPI.StatusView, - plug_session: %{"user_id" => user.id}, - plug_session_fetch: :done, - plug_session_info: :write, - plug_skip_csrf_protection: true - } - }, - %{}) - end, - }) + fetch_public_timeline(opts, "public timeline only local") end - def query_notifications(user) do - without_muted_params = %{"count" => "20", "with_muted" => "false"} - with_muted_params = %{"count" => "20", "with_muted" => "true"} - - Benchee.run(%{ - "Notifications without muted" => fn -> - Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params) - end, - "Notifications with muted" => fn -> - Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) - end - }) - - without_muted_notifications = - Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params) - - with_muted_notifications = - Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params) - - Benchee.run(%{ - "Render notifications without muted" => fn -> - Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ - notifications: without_muted_notifications, - for: user - }) - end, - "Render notifications with muted" => fn -> - Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ - notifications: with_muted_notifications, - for: user - }) - end - }) + defp fetch_public_timeline(user, :tag) do + opts = opts_for_public_timeline(user, :tag) + + fetch_public_timeline(opts, "hashtag timeline") end - def query_dms(user) do - params = %{ - "count" => "20", - "with_muted" => "true", - "type" => "Create", + defp fetch_public_timeline(user, :only_media) do + opts = opts_for_public_timeline(user) |> Map.put("only_media", "true") + + fetch_public_timeline(opts, "public timeline only media") + end + + defp fetch_public_timeline(opts, title) when is_binary(title) do + first_page_last = ActivityPub.fetch_public_activities(opts) |> List.last() + + second_page_last = + ActivityPub.fetch_public_activities(Map.put(opts, "max_id", first_page_last.id)) + |> List.last() + + third_page_last = + ActivityPub.fetch_public_activities(Map.put(opts, "max_id", second_page_last.id)) + |> List.last() + + forth_page_last = + ActivityPub.fetch_public_activities(Map.put(opts, "max_id", third_page_last.id)) + |> List.last() + + Benchee.run( + %{ + title => fn opts -> + ActivityPub.fetch_public_activities(opts) + end + }, + inputs: %{ + "1 page" => opts, + "2 page" => Map.put(opts, "max_id", first_page_last.id), + "3 page" => Map.put(opts, "max_id", second_page_last.id), + "4 page" => Map.put(opts, "max_id", third_page_last.id), + "5 page" => Map.put(opts, "max_id", forth_page_last.id) + }, + formatters: formatters() + ) + end + + defp opts_for_notifications do + %{"count" => "20", "with_muted" => "true"} + end + + defp fetch_notifications(user) do + opts = opts_for_notifications() + + first_page_last = MastodonAPI.get_notifications(user, opts) |> List.last() + + second_page_last = + MastodonAPI.get_notifications(user, Map.put(opts, "max_id", first_page_last.id)) + |> List.last() + + third_page_last = + MastodonAPI.get_notifications(user, Map.put(opts, "max_id", second_page_last.id)) + |> List.last() + + forth_page_last = + MastodonAPI.get_notifications(user, Map.put(opts, "max_id", third_page_last.id)) + |> List.last() + + Benchee.run( + %{ + "Notifications" => fn opts -> + MastodonAPI.get_notifications(user, opts) + end + }, + inputs: %{ + "1 page" => opts, + "2 page" => Map.put(opts, "max_id", first_page_last.id), + "3 page" => Map.put(opts, "max_id", second_page_last.id), + "4 page" => Map.put(opts, "max_id", third_page_last.id), + "5 page" => Map.put(opts, "max_id", forth_page_last.id) + }, + formatters: formatters() + ) + end + + defp fetch_favourites(user) do + first_page_last = ActivityPub.fetch_favourites(user) |> List.last() + + second_page_last = + ActivityPub.fetch_favourites(user, %{"max_id" => first_page_last.id}) |> List.last() + + third_page_last = + ActivityPub.fetch_favourites(user, %{"max_id" => second_page_last.id}) |> List.last() + + forth_page_last = + ActivityPub.fetch_favourites(user, %{"max_id" => third_page_last.id}) |> List.last() + + Benchee.run( + %{ + "Favourites" => fn opts -> + ActivityPub.fetch_favourites(user, opts) + end + }, + inputs: %{ + "1 page" => %{}, + "2 page" => %{"max_id" => first_page_last.id}, + "3 page" => %{"max_id" => second_page_last.id}, + "4 page" => %{"max_id" => third_page_last.id}, + "5 page" => %{"max_id" => forth_page_last.id} + }, + formatters: formatters() + ) + end + + defp opts_for_long_thread(user) do + %{ "blocking_user" => user, - "user" => user, - visibility: "direct" + "user" => user } + end + + defp fetch_long_thread(user) do + %{public_thread: public, private_thread: private} = + Agent.get(:benchmark_state, fn state -> state end) + + opts = opts_for_long_thread(user) + + private_input = {private.data["context"], Map.put(opts, "exclude_id", private.id)} - Benchee.run(%{ - "Direct messages with muted" => fn -> - Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) - |> Pleroma.Pagination.fetch_paginated(params) - end, - "Direct messages without muted" => fn -> - Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) - |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) - end - }) - - dms_with_muted = - Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) - |> Pleroma.Pagination.fetch_paginated(params) - - dms_without_muted = - Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params) - |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false)) - - Benchee.run(%{ - "Rendering dms with muted" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: dms_with_muted, - for: user, - as: :activity - }) - end, - "Rendering dms without muted" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: dms_without_muted, - for: user, - as: :activity - }) - end - }) + public_input = {public.data["context"], Map.put(opts, "exclude_id", public.id)} + + Benchee.run( + %{ + "fetch context" => fn {context, opts} -> + ActivityPub.fetch_activities_for_context(context, opts) + end + }, + inputs: %{ + "Private long thread" => private_input, + "Public long thread" => public_input + }, + formatters: formatters() + ) end - def query_long_thread(user, activity) do - Benchee.run(%{ - "Fetch main post" => fn -> - Pleroma.Activity.get_by_id_with_object(activity.id) - end, - "Fetch context of main post" => fn -> - Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( - activity.data["context"], - %{ - "blocking_user" => user, - "user" => user, - "exclude_id" => activity.id - } - ) - end - }) - - activity = Pleroma.Activity.get_by_id_with_object(activity.id) - - context = - Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context( - activity.data["context"], - %{ - "blocking_user" => user, - "user" => user, - "exclude_id" => activity.id - } + defp render_timelines(user) do + opts = opts_for_home_timeline(user) + + recipients = [user.ap_id | User.following(user)] + + home_activities = ActivityPub.fetch_activities(recipients, opts) |> Enum.reverse() + + recipients = [user.ap_id] + + opts = opts_for_direct_timeline(user) + + direct_activities = + recipients + |> ActivityPub.fetch_activities_query(opts) + |> Pagination.fetch_paginated(opts) + + opts = opts_for_public_timeline(user) + + public_activities = ActivityPub.fetch_public_activities(opts) + + opts = opts_for_public_timeline(user, :tag) + + tag_activities = ActivityPub.fetch_public_activities(opts) + + opts = opts_for_notifications() + + notifications = MastodonAPI.get_notifications(user, opts) + + favourites = ActivityPub.fetch_favourites(user) + + Benchee.run( + %{ + "Rendering home timeline" => fn -> + StatusView.render("index.json", %{ + activities: home_activities, + for: user, + as: :activity + }) + end, + "Rendering direct timeline" => fn -> + StatusView.render("index.json", %{ + activities: direct_activities, + for: user, + as: :activity + }) + end, + "Rendering public timeline" => fn -> + StatusView.render("index.json", %{ + activities: public_activities, + for: user, + as: :activity + }) + end, + "Rendering tag timeline" => fn -> + StatusView.render("index.json", %{ + activities: tag_activities, + for: user, + as: :activity + }) + end, + "Rendering notifications" => fn -> + Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ + notifications: notifications, + for: user + }) + end, + "Rendering favourites timeline" => fn -> + StatusView.render("index.json", %{ + activities: favourites, + for: user, + as: :activity + }) + end + }, + formatters: formatters() + ) + end + + defp render_long_thread(user) do + %{public_thread: public, private_thread: private} = + Agent.get(:benchmark_state, fn state -> state end) + + opts = %{for: user} + public_activity = Activity.get_by_id_with_object(public.id) + private_activity = Activity.get_by_id_with_object(private.id) + + Benchee.run( + %{ + "render" => fn opts -> + StatusView.render("show.json", opts) + end + }, + inputs: %{ + "Public root" => Map.put(opts, :activity, public_activity), + "Private root" => Map.put(opts, :activity, private_activity) + }, + formatters: formatters() + ) + + fetch_opts = opts_for_long_thread(user) + + public_context = + ActivityPub.fetch_activities_for_context( + public.data["context"], + Map.put(fetch_opts, "exclude_id", public.id) ) - Benchee.run(%{ - "Render status" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render("show.json", %{ - activity: activity, - for: user - }) - end, - "Render context" => fn -> - Pleroma.Web.MastodonAPI.StatusView.render( - "index.json", - for: user, - activities: context, - as: :activity - ) - |> Enum.reverse() - end - }) + private_context = + ActivityPub.fetch_activities_for_context( + private.data["context"], + Map.put(fetch_opts, "exclude_id", private.id) + ) + + Benchee.run( + %{ + "render" => fn opts -> + StatusView.render("context.json", opts) + end + }, + inputs: %{ + "Public context" => %{user: user, activity: public_activity, activities: public_context}, + "Private context" => %{ + user: user, + activity: private_activity, + activities: private_context + } + }, + formatters: formatters() + ) end end diff --git a/benchmarks/load_testing/generator.ex b/benchmarks/load_testing/generator.ex deleted file mode 100644 index e4673757c..000000000 --- a/benchmarks/load_testing/generator.ex +++ /dev/null @@ -1,410 +0,0 @@ -defmodule Pleroma.LoadTesting.Generator do - use Pleroma.LoadTesting.Helper - alias Pleroma.Web.CommonAPI - - def generate_like_activities(user, posts) do - count_likes = Kernel.trunc(length(posts) / 4) - IO.puts("Starting generating #{count_likes} like activities...") - - {time, _} = - :timer.tc(fn -> - Task.async_stream( - Enum.take_random(posts, count_likes), - fn post -> {:ok, _, _} = CommonAPI.favorite(post.id, user) end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() - end) - - IO.puts("Inserting like activities take #{to_sec(time)} sec.\n") - end - - def generate_users(opts) do - IO.puts("Starting generating #{opts[:users_max]} users...") - {time, users} = :timer.tc(fn -> do_generate_users(opts) end) - - IO.puts("Inserting users took #{to_sec(time)} sec.\n") - users - end - - defp do_generate_users(opts) do - max = Keyword.get(opts, :users_max) - - Task.async_stream( - 1..max, - &generate_user_data(&1), - max_concurrency: 10, - timeout: 30_000 - ) - |> Enum.to_list() - end - - defp generate_user_data(i) do - remote = Enum.random([true, false]) - - user = %User{ - name: "Test テスト User #{i}", - email: "user#{i}@example.com", - nickname: "nick#{i}", - password_hash: - "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg", - bio: "Tester Number #{i}", - local: remote - } - - user_urls = - if remote do - base_url = - Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"]) - - ap_id = "#{base_url}/users/#{user.nickname}" - - %{ - ap_id: ap_id, - follower_address: ap_id <> "/followers", - following_address: ap_id <> "/following" - } - else - %{ - ap_id: User.ap_id(user), - follower_address: User.ap_followers(user), - following_address: User.ap_following(user) - } - end - - user = Map.merge(user, user_urls) - - Repo.insert!(user) - end - - def generate_activities(user, users) do - do_generate_activities(user, users) - end - - defp do_generate_activities(user, users) do - IO.puts("Starting generating 20000 common activities...") - - {time, _} = - :timer.tc(fn -> - Task.async_stream( - 1..20_000, - fn _ -> - do_generate_activity([user | users]) - end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() - end) - - IO.puts("Inserting common activities take #{to_sec(time)} sec.\n") - - IO.puts("Starting generating 20000 activities with mentions...") - - {time, _} = - :timer.tc(fn -> - Task.async_stream( - 1..20_000, - fn _ -> - do_generate_activity_with_mention(user, users) - end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() - end) - - IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n") - - IO.puts("Starting generating 10000 activities with threads...") - - {time, _} = - :timer.tc(fn -> - Task.async_stream( - 1..10_000, - fn _ -> - do_generate_threads([user | users]) - end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() - end) - - IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n") - end - - defp do_generate_activity(users) do - post = %{ - "status" => "Some status without mention with random user" - } - - CommonAPI.post(Enum.random(users), post) - end - - def generate_power_intervals(opts \\ []) do - count = Keyword.get(opts, :count, 20) - power = Keyword.get(opts, :power, 2) - IO.puts("Generating #{count} intervals for a power #{power} series...") - counts = Enum.map(1..count, fn n -> :math.pow(n, power) end) - sum = Enum.sum(counts) - - densities = - Enum.map(counts, fn c -> - c / sum - end) - - densities - |> Enum.reduce(0, fn density, acc -> - if acc == 0 do - [{0, density}] - else - [{_, lower} | _] = acc - [{lower, lower + density} | acc] - end - end) - |> Enum.reverse() - end - - def generate_tagged_activities(opts \\ []) do - tag_count = Keyword.get(opts, :tag_count, 20) - users = Keyword.get(opts, :users, Repo.all(User)) - activity_count = Keyword.get(opts, :count, 200_000) - - intervals = generate_power_intervals(count: tag_count) - - IO.puts( - "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0" - ) - - Enum.each(1..activity_count, fn _ -> - random = :rand.uniform() - i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end) - CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"}) - end) - end - - defp do_generate_activity_with_mention(user, users) do - mentions_cnt = Enum.random([2, 3, 4, 5]) - with_user = Enum.random([true, false]) - users = Enum.shuffle(users) - mentions_users = Enum.take(users, mentions_cnt) - mentions_users = if with_user, do: [user | mentions_users], else: mentions_users - - mentions_str = - Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ") - - post = %{ - "status" => mentions_str <> "some status with mentions random users" - } - - CommonAPI.post(Enum.random(users), post) - end - - defp do_generate_threads(users) do - thread_length = Enum.random([2, 3, 4, 5]) - actor = Enum.random(users) - - post = %{ - "status" => "Start of the thread" - } - - {:ok, activity} = CommonAPI.post(actor, post) - - Enum.each(1..thread_length, fn _ -> - user = Enum.random(users) - - post = %{ - "status" => "@#{actor.nickname} reply to thread", - "in_reply_to_status_id" => activity.id - } - - CommonAPI.post(user, post) - end) - end - - def generate_remote_activities(user, users) do - do_generate_remote_activities(user, users) - end - - defp do_generate_remote_activities(user, users) do - IO.puts("Starting generating 10000 remote activities...") - - {time, _} = - :timer.tc(fn -> - Task.async_stream( - 1..10_000, - fn i -> - do_generate_remote_activity(i, user, users) - end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() - end) - - IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n") - end - - defp do_generate_remote_activity(i, user, users) do - actor = Enum.random(users) - %{host: host} = URI.parse(actor.ap_id) - date = Date.utc_today() - datetime = DateTime.utc_now() - - map = %{ - "actor" => actor.ap_id, - "cc" => [actor.follower_address, user.ap_id], - "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", - "id" => actor.ap_id <> "/statuses/#{i}/activity", - "object" => %{ - "actor" => actor.ap_id, - "atomUri" => actor.ap_id <> "/statuses/#{i}", - "attachment" => [], - "attributedTo" => actor.ap_id, - "bcc" => [], - "bto" => [], - "cc" => [actor.follower_address, user.ap_id], - "content" => - "

- user.ap_id <> - "\" class=\"u-url mention\">@" <> user.nickname <> "

", - "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", - "conversation" => - "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation", - "emoji" => %{}, - "id" => actor.ap_id <> "/statuses/#{i}", - "inReplyTo" => nil, - "inReplyToAtomUri" => nil, - "published" => datetime, - "sensitive" => true, - "summary" => "cw", - "tag" => [ - %{ - "href" => user.ap_id, - "name" => "@#{user.nickname}@#{host}", - "type" => "Mention" - } - ], - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "type" => "Note", - "url" => "http://#{host}/@#{actor.nickname}/#{i}" - }, - "published" => datetime, - "to" => ["https://www.w3.org/ns/activitystreams#Public"], - "type" => "Create" - } - - Pleroma.Web.ActivityPub.ActivityPub.insert(map, false) - end - - def generate_dms(user, users, opts) do - IO.puts("Starting generating #{opts[:dms_max]} DMs") - {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end) - IO.puts("Inserting dms take #{to_sec(time)} sec.\n") - end - - defp do_generate_dms(user, users, opts) do - Task.async_stream( - 1..opts[:dms_max], - fn _ -> - do_generate_dm(user, users) - end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() - end - - defp do_generate_dm(user, users) do - post = %{ - "status" => "@#{user.nickname} some direct message", - "visibility" => "direct" - } - - CommonAPI.post(Enum.random(users), post) - end - - def generate_long_thread(user, users, opts) do - IO.puts("Starting generating long thread with #{opts[:thread_length]} replies") - {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end) - IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n") - {:ok, activity} - end - - defp do_generate_long_thread(user, users, opts) do - {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"}) - - Task.async_stream( - 1..opts[:thread_length], - fn _ -> do_generate_thread(users, id) end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() - - activity - end - - defp do_generate_thread(users, activity_id) do - CommonAPI.post(Enum.random(users), %{ - "status" => "reply to main post", - "in_reply_to_status_id" => activity_id - }) - end - - def generate_non_visible_message(user, users) do - IO.puts("Starting generating 1000 non visible posts") - - {time, _} = - :timer.tc(fn -> - do_generate_non_visible_posts(user, users) - end) - - IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n") - end - - defp do_generate_non_visible_posts(user, users) do - [not_friend | users] = users - - make_friends(user, users) - - Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end, - max_concurrency: 10, - timeout: 30_000 - ) - |> Stream.run() - end - - defp make_friends(_user, []), do: nil - - defp make_friends(user, [friend | users]) do - {:ok, _} = User.follow(user, friend) - {:ok, _} = User.follow(friend, user) - make_friends(user, users) - end - - defp do_generate_non_visible_post(not_friend, users) do - post = %{ - "status" => "some non visible post", - "visibility" => "private" - } - - {:ok, activity} = CommonAPI.post(not_friend, post) - - thread_length = Enum.random([2, 3, 4, 5]) - - Enum.each(1..thread_length, fn _ -> - user = Enum.random(users) - - post = %{ - "status" => "@#{not_friend.nickname} reply to non visible post", - "in_reply_to_status_id" => activity.id, - "visibility" => "private" - } - - CommonAPI.post(user, post) - end) - end -end diff --git a/benchmarks/load_testing/helper.ex b/benchmarks/load_testing/helper.ex index 47b25c65f..23bbb1cec 100644 --- a/benchmarks/load_testing/helper.ex +++ b/benchmarks/load_testing/helper.ex @@ -1,11 +1,3 @@ defmodule Pleroma.LoadTesting.Helper do - defmacro __using__(_) do - quote do - import Ecto.Query - alias Pleroma.Repo - alias Pleroma.User - - defp to_sec(microseconds), do: microseconds / 1_000_000 - end - end + def to_sec(microseconds), do: microseconds / 1_000_000 end diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex new file mode 100644 index 000000000..951b30d91 --- /dev/null +++ b/benchmarks/load_testing/users.ex @@ -0,0 +1,161 @@ +defmodule Pleroma.LoadTesting.Users do + @moduledoc """ + Module for generating users with friends. + """ + import Ecto.Query + import Pleroma.LoadTesting.Helper, only: [to_sec: 1] + + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.User.Query + + @defaults [ + users: 20_000, + friends: 100 + ] + + @max_concurrency 30 + + @spec generate(keyword()) :: User.t() + def generate(opts \\ []) do + opts = Keyword.merge(@defaults, opts) + + IO.puts("Starting generating #{opts[:users]} users...") + + {time, _} = :timer.tc(fn -> generate_users(opts[:users]) end) + + IO.puts("Generating users take #{to_sec(time)} sec.\n") + + main_user = + Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)) + + IO.puts("Starting making friends for #{opts[:friends]} users...") + {time, _} = :timer.tc(fn -> make_friends(main_user, opts[:friends]) end) + + IO.puts("Making friends take #{to_sec(time)} sec.\n") + + Repo.get(User, main_user.id) + end + + defp generate_users(max) do + Task.async_stream( + 1..max, + &generate_user(&1), + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end + + defp generate_user(i) do + remote = Enum.random([true, false]) + + %User{ + name: "Test テスト User #{i}", + email: "user#{i}@example.com", + nickname: "nick#{i}", + password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + bio: "Tester Number #{i}", + local: !remote + } + |> user_urls() + |> Repo.insert!() + end + + defp user_urls(%{local: true} = user) do + urls = %{ + ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following_address: User.ap_following(user) + } + + Map.merge(user, urls) + end + + defp user_urls(%{local: false} = user) do + base_domain = Enum.random(["domain1.com", "domain2.com", "domain3.com"]) + + ap_id = "https://#{base_domain}/users/#{user.nickname}" + + urls = %{ + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following" + } + + Map.merge(user, urls) + end + + defp make_friends(main_user, max) when is_integer(max) do + number_of_users = + (max / 2) + |> Kernel.trunc() + + main_user + |> get_users(%{limit: number_of_users, local: :local}) + |> run_stream(main_user) + + main_user + |> get_users(%{limit: number_of_users, local: :external}) + |> run_stream(main_user) + end + + defp make_friends(%User{} = main_user, %User{} = user) do + {:ok, _} = User.follow(main_user, user) + {:ok, _} = User.follow(user, main_user) + end + + @spec get_users(User.t(), keyword()) :: [User.t()] + def get_users(user, opts) do + criteria = %{limit: opts[:limit]} + + criteria = + if opts[:local] do + Map.put(criteria, opts[:local], true) + else + criteria + end + + criteria = + if opts[:friends?] do + Map.put(criteria, :friends, user) + else + criteria + end + + query = + criteria + |> Query.build() + |> random_without_user(user) + + query = + if opts[:friends?] == false do + friends_ids = + %{friends: user} + |> Query.build() + |> Repo.all() + |> Enum.map(& &1.id) + + from(u in query, where: u.id not in ^friends_ids) + else + query + end + + Repo.all(query) + end + + defp random_without_user(query, user) do + from(u in query, + where: u.id != ^user.id, + order_by: fragment("RANDOM()") + ) + end + + defp run_stream(users, main_user) do + Task.async_stream(users, &make_friends(main_user, &1), + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end +end -- cgit v1.2.3 From 1f29ecdcd7ecdc4ad8d6bc8fc4c34efbc9b7fe1d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 18 Feb 2020 12:19:10 +0300 Subject: sync with develop --- benchmarks/load_testing/activities.ex | 42 ++++++++++++++++++++++++ benchmarks/load_testing/helper.ex | 11 +++++++ benchmarks/load_testing/users.ex | 61 +++++++++++++++++++---------------- 3 files changed, 87 insertions(+), 27 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index db0e5a66f..121d5c500 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -85,6 +85,48 @@ defmodule Pleroma.LoadTesting.Activities do :ok end + def generate_power_intervals(opts \\ []) do + count = Keyword.get(opts, :count, 20) + power = Keyword.get(opts, :power, 2) + IO.puts("Generating #{count} intervals for a power #{power} series...") + counts = Enum.map(1..count, fn n -> :math.pow(n, power) end) + sum = Enum.sum(counts) + + densities = + Enum.map(counts, fn c -> + c / sum + end) + + densities + |> Enum.reduce(0, fn density, acc -> + if acc == 0 do + [{0, density}] + else + [{_, lower} | _] = acc + [{lower, lower + density} | acc] + end + end) + |> Enum.reverse() + end + + def generate_tagged_activities(opts \\ []) do + tag_count = Keyword.get(opts, :tag_count, 20) + users = Keyword.get(opts, :users, Repo.all(Pleroma.User)) + activity_count = Keyword.get(opts, :count, 200_000) + + intervals = generate_power_intervals(count: tag_count) + + IO.puts( + "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0" + ) + + Enum.each(1..activity_count, fn _ -> + random = :rand.uniform() + i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end) + CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"}) + end) + end + defp generate_long_thread(visibility, user, friends, non_friends, _opts) do group = if visibility == "public", diff --git a/benchmarks/load_testing/helper.ex b/benchmarks/load_testing/helper.ex index 23bbb1cec..cab60acb4 100644 --- a/benchmarks/load_testing/helper.ex +++ b/benchmarks/load_testing/helper.ex @@ -1,3 +1,14 @@ defmodule Pleroma.LoadTesting.Helper do + alias Ecto.Adapters.SQL + alias Pleroma.Repo + def to_sec(microseconds), do: microseconds / 1_000_000 + + def clean_tables do + IO.puts("Deleting old data...\n") + SQL.query!(Repo, "TRUNCATE users CASCADE;") + SQL.query!(Repo, "TRUNCATE activities CASCADE;") + SQL.query!(Repo, "TRUNCATE objects CASCADE;") + SQL.query!(Repo, "TRUNCATE oban_jobs CASCADE;") + end end diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index 951b30d91..bc31dc08b 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -20,31 +20,31 @@ defmodule Pleroma.LoadTesting.Users do def generate(opts \\ []) do opts = Keyword.merge(@defaults, opts) - IO.puts("Starting generating #{opts[:users]} users...") - - {time, _} = :timer.tc(fn -> generate_users(opts[:users]) end) - - IO.puts("Generating users take #{to_sec(time)} sec.\n") + generate_users(opts[:users]) main_user = Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)) - IO.puts("Starting making friends for #{opts[:friends]} users...") - {time, _} = :timer.tc(fn -> make_friends(main_user, opts[:friends]) end) - - IO.puts("Making friends take #{to_sec(time)} sec.\n") + make_friends(main_user, opts[:friends]) Repo.get(User, main_user.id) end - defp generate_users(max) do - Task.async_stream( - 1..max, - &generate_user(&1), - max_concurrency: @max_concurrency, - timeout: 30_000 - ) - |> Stream.run() + def generate_users(max) do + IO.puts("Starting generating #{opts[:users]} users...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..max, + &generate_user(&1), + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Generating users take #{to_sec(time)} sec.\n") end defp generate_user(i) do @@ -86,18 +86,25 @@ defmodule Pleroma.LoadTesting.Users do Map.merge(user, urls) end - defp make_friends(main_user, max) when is_integer(max) do - number_of_users = - (max / 2) - |> Kernel.trunc() + def make_friends(main_user, max) when is_integer(max) do + IO.puts("Starting making friends for #{opts[:friends]} users...") + + {time, _} = + :timer.tc(fn -> + number_of_users = + (max / 2) + |> Kernel.trunc() - main_user - |> get_users(%{limit: number_of_users, local: :local}) - |> run_stream(main_user) + main_user + |> get_users(%{limit: number_of_users, local: :local}) + |> run_stream(main_user) - main_user - |> get_users(%{limit: number_of_users, local: :external}) - |> run_stream(main_user) + main_user + |> get_users(%{limit: number_of_users, local: :external}) + |> run_stream(main_user) + end) + + IO.puts("Making friends take #{to_sec(time)} sec.\n") end defp make_friends(%User{} = main_user, %User{} = user) do -- cgit v1.2.3 From 56503c385e8412a1189748bcf3fdfd4090be9f56 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 17 Mar 2020 13:47:13 +0300 Subject: fix --- benchmarks/load_testing/activities.ex | 4 ++-- benchmarks/load_testing/users.ex | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 121d5c500..24c6b5531 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -19,7 +19,7 @@ defmodule Pleroma.LoadTesting.Activities do non_friends_used: 20 ] - @max_concurrency 30 + @max_concurrency 10 @visibility ~w(public private direct unlisted) @types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote) @@ -81,7 +81,7 @@ defmodule Pleroma.LoadTesting.Activities do ) end) - IO.puts("Generating iterations activities take #{to_sec(time)} sec.\n") + IO.puts("Generating iterations of activities take #{to_sec(time)} sec.\n") :ok end diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index bc31dc08b..b73ac8651 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -14,7 +14,7 @@ defmodule Pleroma.LoadTesting.Users do friends: 100 ] - @max_concurrency 30 + @max_concurrency 10 @spec generate(keyword()) :: User.t() def generate(opts \\ []) do @@ -31,7 +31,7 @@ defmodule Pleroma.LoadTesting.Users do end def generate_users(max) do - IO.puts("Starting generating #{opts[:users]} users...") + IO.puts("Starting generating #{max} users...") {time, _} = :timer.tc(fn -> @@ -87,7 +87,7 @@ defmodule Pleroma.LoadTesting.Users do end def make_friends(main_user, max) when is_integer(max) do - IO.puts("Starting making friends for #{opts[:friends]} users...") + IO.puts("Starting making friends for #{max} users...") {time, _} = :timer.tc(fn -> @@ -107,7 +107,7 @@ defmodule Pleroma.LoadTesting.Users do IO.puts("Making friends take #{to_sec(time)} sec.\n") end - defp make_friends(%User{} = main_user, %User{} = user) do + def make_friends(%User{} = main_user, %User{} = user) do {:ok, _} = User.follow(main_user, user) {:ok, _} = User.follow(user, main_user) end -- cgit v1.2.3 From 96e279655763fedcb701e59c500023a70568c4c6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 11:59:14 +0300 Subject: use in timelines benchmark new user generator --- benchmarks/load_testing/activities.ex | 2 +- benchmarks/load_testing/users.ex | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 24c6b5531..23ee2b987 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -81,7 +81,7 @@ defmodule Pleroma.LoadTesting.Activities do ) end) - IO.puts("Generating iterations of activities take #{to_sec(time)} sec.\n") + IO.puts("Generating iterations of activities took #{to_sec(time)} sec.\n") :ok end diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index b73ac8651..1a8c6e22f 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -33,7 +33,7 @@ defmodule Pleroma.LoadTesting.Users do def generate_users(max) do IO.puts("Starting generating #{max} users...") - {time, _} = + {time, users} = :timer.tc(fn -> Task.async_stream( 1..max, @@ -41,10 +41,11 @@ defmodule Pleroma.LoadTesting.Users do max_concurrency: @max_concurrency, timeout: 30_000 ) - |> Stream.run() + |> Enum.to_list() end) - IO.puts("Generating users take #{to_sec(time)} sec.\n") + IO.puts("Generating users took #{to_sec(time)} sec.\n") + users end defp generate_user(i) do @@ -104,7 +105,7 @@ defmodule Pleroma.LoadTesting.Users do |> run_stream(main_user) end) - IO.puts("Making friends take #{to_sec(time)} sec.\n") + IO.puts("Making friends took #{to_sec(time)} sec.\n") end def make_friends(%User{} = main_user, %User{} = user) do -- cgit v1.2.3 From 2f2bd7fe72f474b7177c751a2dc3af716622ba91 Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Wed, 1 Apr 2020 19:49:09 +0300 Subject: Ability to control the output of account/pleroma/relationship in statuses in order to improve the rendering performance. See `[:extensions, output_relationships_in_statuses_by_default]` setting and `with_relationships` param. --- benchmarks/load_testing/fetcher.ex | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index bd65ac84f..786929ace 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -386,47 +386,56 @@ defmodule Pleroma.LoadTesting.Fetcher do favourites = ActivityPub.fetch_favourites(user) + output_relationships = + !!Pleroma.Config.get([:extensions, :output_relationships_in_statuses_by_default]) + Benchee.run( %{ "Rendering home timeline" => fn -> StatusView.render("index.json", %{ activities: home_activities, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end, "Rendering direct timeline" => fn -> StatusView.render("index.json", %{ activities: direct_activities, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end, "Rendering public timeline" => fn -> StatusView.render("index.json", %{ activities: public_activities, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end, "Rendering tag timeline" => fn -> StatusView.render("index.json", %{ activities: tag_activities, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end, "Rendering notifications" => fn -> Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ notifications: notifications, - for: user + for: user, + skip_relationships: !output_relationships }) end, "Rendering favourites timeline" => fn -> StatusView.render("index.json", %{ activities: favourites, for: user, - as: :activity + as: :activity, + skip_relationships: !output_relationships }) end }, -- cgit v1.2.3 From 6e625a427cdc829714ad0365560d79aa4ee9c2e5 Mon Sep 17 00:00:00 2001 From: Alexander Date: Wed, 4 Dec 2019 09:49:17 +0300 Subject: reply filtering --- benchmarks/load_testing/fetcher.ex | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 786929ace..3aa82b48a 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -495,4 +495,57 @@ defmodule Pleroma.LoadTesting.Fetcher do formatters: formatters() ) end + + def query_replies(user) do + public_params = %{ + "type" => ["Create", "Announce"], + "local_only" => false, + "blocking_user" => user, + "muting_user" => user, + "count" => 20 + } + + Benchee.run(%{ + "Public timeline without reply filtering" => fn -> + ActivityPub.fetch_public_activities(public_params) + end, + "Public timeline with reply filtering - following" => fn -> + public_params + |> Map.put("reply_visibility", "following") + |> Map.put("user", user) + |> ActivityPub.fetch_public_activities() + end, + "Public timeline with reply filtering - self" => fn -> + public_params + |> Map.put("reply_visibility", "self") + |> Map.put("user", user) + |> ActivityPub.fetch_public_activities() + end + }) + + private_params = %{ + "type" => ["Create", "Announce"], + "blocking_user" => user, + "muting_user" => user, + "user" => user, + "count" => 20 + } + + recipients = [user.ap_id | User.following(user)] + + Benchee.run(%{ + "Home timeline without reply filtering" => fn -> + ActivityPub.fetch_activities(recipients, private_params) + end, + "Home timeline with reply filtering - following" => fn -> + private_params = Map.put(private_params, "reply_visibility", "following") + + ActivityPub.fetch_activities(recipients, private_params) + end, + "Home timeline with reply filtering - self" => fn -> + private_params = Map.put(private_params, "reply_visibility", "self") + ActivityPub.fetch_activities(recipients, private_params) + end + }) + end end -- cgit v1.2.3 From 1a75ef63b2f6fef96b9bf9d07b4963fb217d4017 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 24 Apr 2020 09:24:08 +0300 Subject: updating benchmarks --- benchmarks/load_testing/activities.ex | 2 +- benchmarks/load_testing/fetcher.ex | 89 +++++++++++++++++------------------ 2 files changed, 43 insertions(+), 48 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 23ee2b987..2b032943b 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -313,7 +313,7 @@ defmodule Pleroma.LoadTesting.Activities do tasks = get_reply_tasks(visibility, group) {:ok, activity} = - CommonAPI.post(user, %{"status" => "Simple status", "visibility" => "unlisted"}) + CommonAPI.post(user, %{"status" => "Simple status", "visibility" => visibility}) acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} insert_replies(tasks, visibility, user, friends, non_friends, acc) diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 3aa82b48a..6503deb41 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -41,6 +41,7 @@ defmodule Pleroma.LoadTesting.Fetcher do fetch_notifications(user) fetch_favourites(user) fetch_long_thread(user) + fetch_timelines_with_reply_filtering(user) end defp render_views(user) do @@ -496,56 +497,50 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end - def query_replies(user) do - public_params = %{ - "type" => ["Create", "Announce"], - "local_only" => false, - "blocking_user" => user, - "muting_user" => user, - "count" => 20 - } + defp fetch_timelines_with_reply_filtering(user) do + public_params = opts_for_public_timeline(user) - Benchee.run(%{ - "Public timeline without reply filtering" => fn -> - ActivityPub.fetch_public_activities(public_params) - end, - "Public timeline with reply filtering - following" => fn -> - public_params - |> Map.put("reply_visibility", "following") - |> Map.put("user", user) - |> ActivityPub.fetch_public_activities() - end, - "Public timeline with reply filtering - self" => fn -> - public_params - |> Map.put("reply_visibility", "self") - |> Map.put("user", user) - |> ActivityPub.fetch_public_activities() - end - }) - - private_params = %{ - "type" => ["Create", "Announce"], - "blocking_user" => user, - "muting_user" => user, - "user" => user, - "count" => 20 - } + Benchee.run( + %{ + "Public timeline without reply filtering" => fn -> + ActivityPub.fetch_public_activities(public_params) + end, + "Public timeline with reply filtering - following" => fn -> + public_params + |> Map.put("reply_visibility", "following") + |> Map.put("user", user) + |> ActivityPub.fetch_public_activities() + end, + "Public timeline with reply filtering - self" => fn -> + public_params + |> Map.put("reply_visibility", "self") + |> Map.put("user", user) + |> ActivityPub.fetch_public_activities() + end + }, + formatters: formatters() + ) + + private_params = opts_for_home_timeline(user) recipients = [user.ap_id | User.following(user)] - Benchee.run(%{ - "Home timeline without reply filtering" => fn -> - ActivityPub.fetch_activities(recipients, private_params) - end, - "Home timeline with reply filtering - following" => fn -> - private_params = Map.put(private_params, "reply_visibility", "following") - - ActivityPub.fetch_activities(recipients, private_params) - end, - "Home timeline with reply filtering - self" => fn -> - private_params = Map.put(private_params, "reply_visibility", "self") - ActivityPub.fetch_activities(recipients, private_params) - end - }) + Benchee.run( + %{ + "Home timeline without reply filtering" => fn -> + ActivityPub.fetch_activities(recipients, private_params) + end, + "Home timeline with reply filtering - following" => fn -> + private_params = Map.put(private_params, "reply_visibility", "following") + + ActivityPub.fetch_activities(recipients, private_params) + end, + "Home timeline with reply filtering - self" => fn -> + private_params = Map.put(private_params, "reply_visibility", "self") + ActivityPub.fetch_activities(recipients, private_params) + end + }, + formatters: formatters() + ) end end -- cgit v1.2.3 From 375ab05234a3590c161a2a5a4f715fbc61dafb34 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 24 Apr 2020 09:57:30 +0300 Subject: bench sync --- benchmarks/load_testing/activities.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 2b032943b..482e42fc1 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -279,7 +279,7 @@ defmodule Pleroma.LoadTesting.Activities do actor = get_actor(group, user, friends, non_friends) with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), - {:ok, _activity, _object} <- CommonAPI.favorite(activity_id, actor) do + {:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do :ok else {:error, _} -> -- cgit v1.2.3 From d89cd0a19733eec27b79b768df2e30a68bfc6d6b Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 24 Apr 2020 18:25:26 +0200 Subject: Reply Filtering: Refactor. --- benchmarks/load_testing/fetcher.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 6503deb41..12c30f6f5 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -508,13 +508,13 @@ defmodule Pleroma.LoadTesting.Fetcher do "Public timeline with reply filtering - following" => fn -> public_params |> Map.put("reply_visibility", "following") - |> Map.put("user", user) + |> Map.put("reply_filtering_user", user) |> ActivityPub.fetch_public_activities() end, "Public timeline with reply filtering - self" => fn -> public_params |> Map.put("reply_visibility", "self") - |> Map.put("user", user) + |> Map.put("reply_filtering_user", user) |> ActivityPub.fetch_public_activities() end }, @@ -531,12 +531,19 @@ defmodule Pleroma.LoadTesting.Fetcher do ActivityPub.fetch_activities(recipients, private_params) end, "Home timeline with reply filtering - following" => fn -> - private_params = Map.put(private_params, "reply_visibility", "following") + private_params = + private_params + |> Map.put("reply_filtering_user", user) + |> Map.put("reply_visibility", "following") ActivityPub.fetch_activities(recipients, private_params) end, "Home timeline with reply filtering - self" => fn -> - private_params = Map.put(private_params, "reply_visibility", "self") + private_params = + private_params + |> Map.put("reply_filtering_user", user) + |> Map.put("reply_visibility", "self") + ActivityPub.fetch_activities(recipients, private_params) end }, -- cgit v1.2.3 From d5cdc907e3fda14c2ce78ddbb124739441330ecc Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 1 May 2020 18:45:24 +0300 Subject: Restricted embedding of relationships where applicable (statuses / notifications / accounts rendering). Added support for :skip_notifications for accounts listing (index.json). Adjusted tests. --- benchmarks/load_testing/fetcher.ex | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 12c30f6f5..0de4924bc 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -387,56 +387,47 @@ defmodule Pleroma.LoadTesting.Fetcher do favourites = ActivityPub.fetch_favourites(user) - output_relationships = - !!Pleroma.Config.get([:extensions, :output_relationships_in_statuses_by_default]) - Benchee.run( %{ "Rendering home timeline" => fn -> StatusView.render("index.json", %{ activities: home_activities, for: user, - as: :activity, - skip_relationships: !output_relationships + as: :activity }) end, "Rendering direct timeline" => fn -> StatusView.render("index.json", %{ activities: direct_activities, for: user, - as: :activity, - skip_relationships: !output_relationships + as: :activity }) end, "Rendering public timeline" => fn -> StatusView.render("index.json", %{ activities: public_activities, for: user, - as: :activity, - skip_relationships: !output_relationships + as: :activity }) end, "Rendering tag timeline" => fn -> StatusView.render("index.json", %{ activities: tag_activities, for: user, - as: :activity, - skip_relationships: !output_relationships + as: :activity }) end, "Rendering notifications" => fn -> Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{ notifications: notifications, - for: user, - skip_relationships: !output_relationships + for: user }) end, "Rendering favourites timeline" => fn -> StatusView.render("index.json", %{ activities: favourites, for: user, - as: :activity, - skip_relationships: !output_relationships + as: :activity }) end }, -- cgit v1.2.3 From b46811a07444187e7765f439e933f214c0a0aeb3 Mon Sep 17 00:00:00 2001 From: Alex Gleason Date: Tue, 12 May 2020 16:42:24 -0500 Subject: Upgrade Comeonin to v5 https://github.com/riverrun/comeonin/blob/master/UPGRADE_v5.md --- benchmarks/load_testing/users.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index 1a8c6e22f..e4d0b22ff 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -55,7 +55,7 @@ defmodule Pleroma.LoadTesting.Users do name: "Test テスト User #{i}", email: "user#{i}@example.com", nickname: "nick#{i}", - password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + password_hash: Pbkdf2.hash_pwd_salt("test"), bio: "Tester Number #{i}", local: !remote } -- cgit v1.2.3 From 7b500d6b4fdad59c7242dc41646bd0bb5cc04e67 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 18 May 2020 13:56:58 +0200 Subject: Load Testing: adjust to new CommonAPI.post format. --- benchmarks/load_testing/activities.ex | 41 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 21 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 482e42fc1..ff0d481a8 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -123,7 +123,7 @@ defmodule Pleroma.LoadTesting.Activities do Enum.each(1..activity_count, fn _ -> random = :rand.uniform() i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end) - CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"}) + CommonAPI.post(Enum.random(users), %{status: "a post with the tag #tag_#{i}"}) end) end @@ -137,8 +137,8 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, activity} = CommonAPI.post(user, %{ - "status" => "Start of #{visibility} long thread", - "visibility" => visibility + status: "Start of #{visibility} long thread", + visibility: visibility }) Agent.update(:benchmark_state, fn state -> @@ -186,7 +186,7 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, _activity} = group |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{"status" => "Simple status", "visibility" => visibility}) + |> CommonAPI.post(%{status: "Simple status", visibility: visibility}) end defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do @@ -194,8 +194,8 @@ defmodule Pleroma.LoadTesting.Activities do group |> get_actor(user, friends, non_friends) |> CommonAPI.post(%{ - "status" => "Simple status with emoji :firefox:", - "visibility" => visibility + status: "Simple status with emoji :firefox:", + visibility: visibility }) end @@ -213,8 +213,8 @@ defmodule Pleroma.LoadTesting.Activities do group |> get_actor(user, friends, non_friends) |> CommonAPI.post(%{ - "status" => Enum.join(user_mentions, ", ") <> " simple status with mentions", - "visibility" => visibility + status: Enum.join(user_mentions, ", ") <> " simple status with mentions", + visibility: visibility }) end @@ -236,8 +236,8 @@ defmodule Pleroma.LoadTesting.Activities do group |> get_actor(user, friends, non_friends) |> CommonAPI.post(%{ - "status" => mentions <> " hell thread status", - "visibility" => visibility + status: mentions <> " hell thread status", + visibility: visibility }) end @@ -262,9 +262,9 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, _activity} = CommonAPI.post(actor, %{ - "status" => "Post with attachment", - "visibility" => visibility, - "media_ids" => [object.id] + status: "Post with attachment", + visibility: visibility, + media_ids: [object.id] }) end @@ -272,7 +272,7 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, _activity} = group |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{"status" => "Status with #tag", "visibility" => visibility}) + |> CommonAPI.post(%{status: "Status with #tag", visibility: visibility}) end defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do @@ -312,8 +312,7 @@ defmodule Pleroma.LoadTesting.Activities do actor = get_actor(group, user, friends, non_friends) tasks = get_reply_tasks(visibility, group) - {:ok, activity} = - CommonAPI.post(user, %{"status" => "Simple status", "visibility" => visibility}) + {:ok, activity} = CommonAPI.post(user, %{status: "Simple status", visibility: visibility}) acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} insert_replies(tasks, visibility, user, friends, non_friends, acc) @@ -336,8 +335,8 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, activity} = CommonAPI.post(actor, %{ - "status" => Enum.join(data, ", ") <> "simple status", - "visibility" => "direct" + status: Enum.join(data, ", ") <> "simple status", + visibility: "direct" }) acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]} @@ -527,9 +526,9 @@ defmodule Pleroma.LoadTesting.Activities do defp insert_reply(actor, data, activity_id, visibility) do {:ok, reply} = CommonAPI.post(actor, %{ - "status" => Enum.join(data, ", "), - "visibility" => visibility, - "in_reply_to_status_id" => activity_id + status: Enum.join(data, ", "), + visibility: visibility, + in_reply_to_status_id: activity_id }) {reply.id, ["@" <> actor.nickname | data]} -- cgit v1.2.3 From 7e6ec778d965419ed4083428d4d39b2a689f7619 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 20 May 2020 17:45:06 +0300 Subject: exclude replies on blocked domains --- benchmarks/load_testing/activities.ex | 361 ++++++++++++++++++---------------- benchmarks/load_testing/fetcher.ex | 71 +++++++ benchmarks/load_testing/users.ex | 22 ++- 3 files changed, 287 insertions(+), 167 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index ff0d481a8..074ded457 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -22,8 +22,21 @@ defmodule Pleroma.LoadTesting.Activities do @max_concurrency 10 @visibility ~w(public private direct unlisted) - @types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote) - @groups ~w(user friends non_friends) + @types [ + :simple, + :emoji, + :mentions, + :hell_thread, + :attachment, + :tag, + :like, + :reblog, + :simple_thread + ] + @groups [:friends_local, :friends_remote, :non_friends_local, :non_friends_local] + @remote_groups [:friends_remote, :non_friends_remote] + @friends_groups [:friends_local, :friends_remote] + @non_friends_groups [:non_friends_local, :non_friends_remote] @spec generate(User.t(), keyword()) :: :ok def generate(user, opts \\ []) do @@ -34,33 +47,24 @@ defmodule Pleroma.LoadTesting.Activities do opts = Keyword.merge(@defaults, opts) - friends = - user - |> Users.get_users(limit: opts[:friends_used], local: :local, friends?: true) - |> Enum.shuffle() + users = Users.prepare_users(user, opts) - non_friends = - user - |> Users.get_users(limit: opts[:non_friends_used], local: :local, friends?: false) - |> Enum.shuffle() + {:ok, _} = Agent.start_link(fn -> users[:non_friends_remote] end, name: :non_friends_remote) task_data = for visibility <- @visibility, type <- @types, - group <- @groups, + group <- [:user | @groups], do: {visibility, type, group} IO.puts("Starting generating #{opts[:iterations]} iterations of activities...") - friends_thread = Enum.take(friends, 5) - non_friends_thread = Enum.take(friends, 5) - public_long_thread = fn -> - generate_long_thread("public", user, friends_thread, non_friends_thread, opts) + generate_long_thread("public", users, opts) end private_long_thread = fn -> - generate_long_thread("private", user, friends_thread, non_friends_thread, opts) + generate_long_thread("private", users, opts) end iterations = opts[:iterations] @@ -73,10 +77,10 @@ defmodule Pleroma.LoadTesting.Activities do i when i == iterations - 2 -> spawn(public_long_thread) spawn(private_long_thread) - generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) + generate_activities(users, Enum.shuffle(task_data), opts) _ -> - generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) + generate_activities(users, Enum.shuffle(task_data), opts) end ) end) @@ -127,16 +131,16 @@ defmodule Pleroma.LoadTesting.Activities do end) end - defp generate_long_thread(visibility, user, friends, non_friends, _opts) do + defp generate_long_thread(visibility, users, _opts) do group = if visibility == "public", - do: "friends", - else: "user" + do: :friends_local, + else: :user tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50) {:ok, activity} = - CommonAPI.post(user, %{ + CommonAPI.post(users[:user], %{ status: "Start of #{visibility} long thread", visibility: visibility }) @@ -150,31 +154,28 @@ defmodule Pleroma.LoadTesting.Activities do Map.put(state, key, activity) end) - acc = {activity.id, ["@" <> user.nickname, "reply to long thread"]} - insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) + acc = {activity.id, ["@" <> users[:user].nickname, "reply to long thread"]} + insert_replies_for_long_thread(tasks, visibility, users, acc) IO.puts("Generating #{visibility} long thread ended\n") end - defp insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) do + defp insert_replies_for_long_thread(tasks, visibility, users, acc) do Enum.reduce(tasks, acc, fn - "friend", {id, data} -> - friend = Enum.random(friends) - insert_reply(friend, List.delete(data, "@" <> friend.nickname), id, visibility) - - "non_friend", {id, data} -> - non_friend = Enum.random(non_friends) - insert_reply(non_friend, List.delete(data, "@" <> non_friend.nickname), id, visibility) - - "user", {id, data} -> + :user, {id, data} -> + user = users[:user] insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility) + + group, {id, data} -> + replier = Enum.random(users[group]) + insert_reply(replier, List.delete(data, "@" <> replier.nickname), id, visibility) end) end - defp generate_activities(user, friends, non_friends, task_data, opts) do + defp generate_activities(users, task_data, opts) do Task.async_stream( task_data, fn {visibility, type, group} -> - insert_activity(type, visibility, group, user, friends, non_friends, opts) + insert_activity(type, visibility, group, users, opts) end, max_concurrency: @max_concurrency, timeout: 30_000 @@ -182,67 +183,104 @@ defmodule Pleroma.LoadTesting.Activities do |> Stream.run() end - defp insert_activity("simple", visibility, group, user, friends, non_friends, _opts) do - {:ok, _activity} = + defp insert_local_activity(visibility, group, users, status) do + {:ok, _} = group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{status: "Simple status", visibility: visibility}) + |> get_actor(users) + |> CommonAPI.post(%{status: status, visibility: visibility}) end - defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do - {:ok, _activity} = - group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{ - status: "Simple status with emoji :firefox:", - visibility: visibility - }) + defp insert_remote_activity(visibility, group, users, status) do + actor = get_actor(group, users) + {act_data, obj_data} = prepare_activity_data(actor, visibility, users[:user]) + {activity_data, object_data} = other_data(actor, status) + + activity_data + |> Map.merge(act_data) + |> Map.put("object", Map.merge(object_data, obj_data)) + |> Pleroma.Web.ActivityPub.ActivityPub.insert(false) end - defp insert_activity("mentions", visibility, group, user, friends, non_friends, _opts) do + defp user_mentions(users) do user_mentions = - get_random_mentions(friends, Enum.random(0..3)) ++ - get_random_mentions(non_friends, Enum.random(0..3)) + Enum.reduce( + @groups, + [], + fn group, acc -> + acc ++ get_random_mentions(users[group], Enum.random(0..2)) + end + ) - user_mentions = - if Enum.random([true, false]), - do: ["@" <> user.nickname | user_mentions], - else: user_mentions + if Enum.random([true, false]), + do: ["@" <> users[:user].nickname | user_mentions], + else: user_mentions + end - {:ok, _activity} = - group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{ - status: Enum.join(user_mentions, ", ") <> " simple status with mentions", - visibility: visibility - }) + defp hell_thread_mentions(users) do + with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do + cached = + @groups + |> Enum.reduce([users[:user]], fn group, acc -> + acc ++ Enum.take(users[group], 5) + end) + |> Enum.map(&"@#{&1.nickname}") + |> Enum.join(", ") + + Cachex.put(:user_cache, "hell_thread_mentions", cached) + cached + else + {:ok, cached} -> cached + end end - defp insert_activity("hell_thread", visibility, group, user, friends, non_friends, _opts) do - mentions = - with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do - cached = - ([user | Enum.take(friends, 10)] ++ Enum.take(non_friends, 10)) - |> Enum.map(&"@#{&1.nickname}") - |> Enum.join(", ") + defp insert_activity(:simple, visibility, group, users, _opts) + when group in @remote_groups do + insert_remote_activity(visibility, group, users, "Remote status") + end - Cachex.put(:user_cache, "hell_thread_mentions", cached) - cached - else - {:ok, cached} -> cached - end + defp insert_activity(:simple, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Simple status") + end - {:ok, _activity} = - group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{ - status: mentions <> " hell thread status", - visibility: visibility - }) + defp insert_activity(:emoji, visibility, group, users, _opts) + when group in @remote_groups do + insert_remote_activity(visibility, group, users, "Remote status with emoji :firefox:") + end + + defp insert_activity(:emoji, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Simple status with emoji :firefox:") + end + + defp insert_activity(:mentions, visibility, group, users, _opts) + when group in @remote_groups do + mentions = user_mentions(users) + + status = Enum.join(mentions, ", ") <> " remote status with mentions" + + insert_remote_activity(visibility, group, users, status) + end + + defp insert_activity(:mentions, visibility, group, users, _opts) do + mentions = user_mentions(users) + + status = Enum.join(mentions, ", ") <> " simple status with mentions" + insert_remote_activity(visibility, group, users, status) + end + + defp insert_activity(:hell_thread, visibility, group, users, _) + when group in @remote_groups do + mentions = hell_thread_mentions(users) + insert_remote_activity(visibility, group, users, mentions <> " remote hell thread status") + end + + defp insert_activity(:hell_thread, visibility, group, users, _opts) do + mentions = hell_thread_mentions(users) + + insert_local_activity(visibility, group, users, mentions <> " hell thread status") end - defp insert_activity("attachment", visibility, group, user, friends, non_friends, _opts) do - actor = get_actor(group, user, friends, non_friends) + defp insert_activity(:attachment, visibility, group, users, _opts) do + actor = get_actor(group, users) obj_data = %{ "actor" => actor.ap_id, @@ -268,67 +306,54 @@ defmodule Pleroma.LoadTesting.Activities do }) end - defp insert_activity("tag", visibility, group, user, friends, non_friends, _opts) do - {:ok, _activity} = - group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{status: "Status with #tag", visibility: visibility}) + defp insert_activity(:tag, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Status with #tag") end - defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do - actor = get_actor(group, user, friends, non_friends) + defp insert_activity(:like, visibility, group, users, opts) do + actor = get_actor(group, users) with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), {:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do :ok else {:error, _} -> - insert_activity("like", visibility, group, user, friends, non_friends, opts) + insert_activity(:like, visibility, group, users, opts) nil -> Process.sleep(15) - insert_activity("like", visibility, group, user, friends, non_friends, opts) + insert_activity(:like, visibility, group, users, opts) end end - defp insert_activity("reblog", visibility, group, user, friends, non_friends, opts) do - actor = get_actor(group, user, friends, non_friends) + defp insert_activity(:reblog, visibility, group, users, opts) do + actor = get_actor(group, users) with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), - {:ok, _activity, _object} <- CommonAPI.repeat(activity_id, actor) do + {:ok, _activity} <- CommonAPI.repeat(activity_id, actor) do :ok else {:error, _} -> - insert_activity("reblog", visibility, group, user, friends, non_friends, opts) + insert_activity(:reblog, visibility, group, users, opts) nil -> Process.sleep(15) - insert_activity("reblog", visibility, group, user, friends, non_friends, opts) + insert_activity(:reblog, visibility, group, users, opts) end end - defp insert_activity("simple_thread", visibility, group, user, friends, non_friends, _opts) - when visibility in ["public", "unlisted", "private"] do - actor = get_actor(group, user, friends, non_friends) - tasks = get_reply_tasks(visibility, group) - - {:ok, activity} = CommonAPI.post(user, %{status: "Simple status", visibility: visibility}) - - acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} - insert_replies(tasks, visibility, user, friends, non_friends, acc) - end - - defp insert_activity("simple_thread", "direct", group, user, friends, non_friends, _opts) do - actor = get_actor(group, user, friends, non_friends) + defp insert_activity(:simple_thread, "direct", group, users, _opts) do + actor = get_actor(group, users) tasks = get_reply_tasks("direct", group) list = case group do - "non_friends" -> - Enum.take(non_friends, 3) + :user -> + group = Enum.random(@friends_groups) + Enum.take(users[group], 3) _ -> - Enum.take(friends, 3) + Enum.take(users[group], 3) end data = Enum.map(list, &("@" <> &1.nickname)) @@ -339,40 +364,30 @@ defmodule Pleroma.LoadTesting.Activities do visibility: "direct" }) - acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]} - insert_direct_replies(tasks, user, list, acc) + acc = {activity.id, ["@" <> users[:user].nickname | data] ++ ["reply to status"]} + insert_direct_replies(tasks, users[:user], list, acc) end - defp insert_activity("remote", _, "user", _, _, _, _), do: :ok - - defp insert_activity("remote", visibility, group, user, _friends, _non_friends, opts) do - remote_friends = - Users.get_users(user, limit: opts[:friends_used], local: :external, friends?: true) - - remote_non_friends = - Users.get_users(user, limit: opts[:non_friends_used], local: :external, friends?: false) - - actor = get_actor(group, user, remote_friends, remote_non_friends) + defp insert_activity(:simple_thread, visibility, group, users, _opts) do + actor = get_actor(group, users) + tasks = get_reply_tasks(visibility, group) - {act_data, obj_data} = prepare_activity_data(actor, visibility, user) - {activity_data, object_data} = other_data(actor) + {:ok, activity} = + CommonAPI.post(users[:user], %{status: "Simple status", visibility: visibility}) - activity_data - |> Map.merge(act_data) - |> Map.put("object", Map.merge(object_data, obj_data)) - |> Pleroma.Web.ActivityPub.ActivityPub.insert(false) + acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} + insert_replies(tasks, visibility, users, acc) end - defp get_actor("user", user, _friends, _non_friends), do: user - defp get_actor("friends", _user, friends, _non_friends), do: Enum.random(friends) - defp get_actor("non_friends", _user, _friends, non_friends), do: Enum.random(non_friends) + defp get_actor(:user, %{user: user}), do: user + defp get_actor(group, users), do: Enum.random(users[group]) - defp other_data(actor) do + defp other_data(actor, content) do %{host: host} = URI.parse(actor.ap_id) datetime = DateTime.utc_now() - context_id = "http://#{host}:4000/contexts/#{UUID.generate()}" - activity_id = "http://#{host}:4000/activities/#{UUID.generate()}" - object_id = "http://#{host}:4000/objects/#{UUID.generate()}" + context_id = "https://#{host}/contexts/#{UUID.generate()}" + activity_id = "https://#{host}/activities/#{UUID.generate()}" + object_id = "https://#{host}/objects/#{UUID.generate()}" activity_data = %{ "actor" => actor.ap_id, @@ -389,7 +404,7 @@ defmodule Pleroma.LoadTesting.Activities do "attributedTo" => actor.ap_id, "bcc" => [], "bto" => [], - "content" => "Remote post", + "content" => content, "context" => context_id, "conversation" => context_id, "emoji" => %{}, @@ -475,51 +490,65 @@ defmodule Pleroma.LoadTesting.Activities do {act_data, obj_data} end - defp get_reply_tasks("public", "user"), do: ~w(friend non_friend user) - defp get_reply_tasks("public", "friends"), do: ~w(non_friend user friend) - defp get_reply_tasks("public", "non_friends"), do: ~w(user friend non_friend) + defp get_reply_tasks("public", :user) do + [:friends_local, :friends_remote, :non_friends_local, :non_friends_remote, :user] + end + + defp get_reply_tasks("public", group) when group in @friends_groups do + [:non_friends_local, :non_friends_remote, :user, :friends_local, :friends_remote] + end - defp get_reply_tasks(visibility, "user") when visibility in ["unlisted", "private"], - do: ~w(friend user friend) + defp get_reply_tasks("public", group) when group in @non_friends_groups do + [:user, :friends_local, :friends_remote, :non_friends_local, :non_friends_remote] + end - defp get_reply_tasks(visibility, "friends") when visibility in ["unlisted", "private"], - do: ~w(user friend user) + defp get_reply_tasks(visibility, :user) when visibility in ["unlisted", "private"] do + [:friends_local, :friends_remote, :user, :friends_local, :friends_remote] + end - defp get_reply_tasks(visibility, "non_friends") when visibility in ["unlisted", "private"], - do: [] + defp get_reply_tasks(visibility, group) + when visibility in ["unlisted", "private"] and group in @friends_groups do + [:user, :friends_remote, :friends_local, :user] + end - defp get_reply_tasks("direct", "user"), do: ~w(friend user friend) - defp get_reply_tasks("direct", "friends"), do: ~w(user friend user) - defp get_reply_tasks("direct", "non_friends"), do: ~w(user non_friend user) + defp get_reply_tasks(visibility, group) + when visibility in ["unlisted", "private"] and + group in @non_friends_groups, + do: [] - defp insert_replies(tasks, visibility, user, friends, non_friends, acc) do - Enum.reduce(tasks, acc, fn - "friend", {id, data} -> - friend = Enum.random(friends) - insert_reply(friend, data, id, visibility) + defp get_reply_tasks("direct", :user), do: [:friends_local, :user, :friends_remote] - "non_friend", {id, data} -> - non_friend = Enum.random(non_friends) - insert_reply(non_friend, data, id, visibility) + defp get_reply_tasks("direct", group) when group in @friends_groups, + do: [:user, group, :user] - "user", {id, data} -> - insert_reply(user, data, id, visibility) + defp get_reply_tasks("direct", group) when group in @non_friends_groups do + [:user, :non_friends_remote, :user, :non_friends_local] + end + + defp insert_replies(tasks, visibility, users, acc) do + Enum.reduce(tasks, acc, fn + :user, {id, data} -> + insert_reply(users[:user], data, id, visibility) + + group, {id, data} -> + replier = Enum.random(users[group]) + insert_reply(replier, data, id, visibility) end) end defp insert_direct_replies(tasks, user, list, acc) do Enum.reduce(tasks, acc, fn - group, {id, data} when group in ["friend", "non_friend"] -> + :user, {id, data} -> + {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct") + {reply_id, data} + + _, {id, data} -> actor = Enum.random(list) {reply_id, _} = insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct") {reply_id, data} - - "user", {id, data} -> - {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct") - {reply_id, data} end) end diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 0de4924bc..b278faf9f 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -36,6 +36,7 @@ defmodule Pleroma.LoadTesting.Fetcher do fetch_home_timeline(user) fetch_direct_timeline(user) fetch_public_timeline(user) + fetch_public_timeline(user, :with_blocks) fetch_public_timeline(user, :local) fetch_public_timeline(user, :tag) fetch_notifications(user) @@ -227,6 +228,76 @@ defmodule Pleroma.LoadTesting.Fetcher do fetch_public_timeline(opts, "public timeline only media") end + # TODO: remove using `:method` after benchmarks + defp fetch_public_timeline(user, :with_blocks) do + opts = opts_for_public_timeline(user) + + remote_non_friends = Agent.get(:non_friends_remote, & &1) + + Benchee.run( + %{ + "public timeline without blocks" => fn opts -> + ActivityPub.fetch_public_activities(opts) + end + }, + inputs: %{ + "old filtering" => Map.delete(opts, :method), + "with psql fun" => Map.put(opts, :method, :fun), + "with unnest" => Map.put(opts, :method, :unnest) + } + ) + + Enum.each(remote_non_friends, fn non_friend -> + {:ok, _} = User.block(user, non_friend) + end) + + user = User.get_by_id(user.id) + + opts = Map.put(opts, "blocking_user", user) + + Benchee.run( + %{ + "public timeline with user block" => fn opts -> + ActivityPub.fetch_public_activities(opts) + end + }, + inputs: %{ + "old filtering" => Map.delete(opts, :method), + "with psql fun" => Map.put(opts, :method, :fun), + "with unnest" => Map.put(opts, :method, :unnest) + } + ) + + domains = + Enum.reduce(remote_non_friends, [], fn non_friend, domains -> + {:ok, _user} = User.unblock(user, non_friend) + %{host: host} = URI.parse(non_friend.ap_id) + [host | domains] + end) + + domains = Enum.uniq(domains) + + Enum.each(domains, fn domain -> + {:ok, _} = User.block_domain(user, domain) + end) + + user = User.get_by_id(user.id) + opts = Map.put(opts, "blocking_user", user) + + Benchee.run( + %{ + "public timeline with domain block" => fn opts -> + ActivityPub.fetch_public_activities(opts) + end + }, + inputs: %{ + "old filtering" => Map.delete(opts, :method), + "with psql fun" => Map.put(opts, :method, :fun), + "with unnest" => Map.put(opts, :method, :unnest) + } + ) + end + defp fetch_public_timeline(opts, title) when is_binary(title) do first_page_last = ActivityPub.fetch_public_activities(opts) |> List.last() diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index e4d0b22ff..6cf3958c1 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -27,7 +27,7 @@ defmodule Pleroma.LoadTesting.Users do make_friends(main_user, opts[:friends]) - Repo.get(User, main_user.id) + User.get_by_id(main_user.id) end def generate_users(max) do @@ -166,4 +166,24 @@ defmodule Pleroma.LoadTesting.Users do ) |> Stream.run() end + + @spec prepare_users(User.t(), keyword()) :: map() + def prepare_users(user, opts) do + friends_limit = opts[:friends_used] + non_friends_limit = opts[:non_friends_used] + + %{ + user: user, + friends_local: fetch_users(user, friends_limit, :local, true), + friends_remote: fetch_users(user, friends_limit, :external, true), + non_friends_local: fetch_users(user, non_friends_limit, :local, false), + non_friends_remote: fetch_users(user, non_friends_limit, :external, false) + } + end + + defp fetch_users(user, limit, local, friends?) do + user + |> get_users(limit: limit, local: local, friends?: friends?) + |> Enum.shuffle() + end end -- cgit v1.2.3 From 19f468c5bc230d6790b00aa87e509a07e709aaa7 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 2 Jun 2020 08:50:24 +0300 Subject: replies filtering for blocked domains --- benchmarks/load_testing/fetcher.ex | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index b278faf9f..22a06e472 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -228,24 +228,16 @@ defmodule Pleroma.LoadTesting.Fetcher do fetch_public_timeline(opts, "public timeline only media") end - # TODO: remove using `:method` after benchmarks defp fetch_public_timeline(user, :with_blocks) do opts = opts_for_public_timeline(user) remote_non_friends = Agent.get(:non_friends_remote, & &1) - Benchee.run( - %{ - "public timeline without blocks" => fn opts -> - ActivityPub.fetch_public_activities(opts) - end - }, - inputs: %{ - "old filtering" => Map.delete(opts, :method), - "with psql fun" => Map.put(opts, :method, :fun), - "with unnest" => Map.put(opts, :method, :unnest) - } - ) + Benchee.run(%{ + "public timeline without blocks" => fn -> + ActivityPub.fetch_public_activities(opts) + end + }) Enum.each(remote_non_friends, fn non_friend -> {:ok, _} = User.block(user, non_friend) @@ -257,15 +249,10 @@ defmodule Pleroma.LoadTesting.Fetcher do Benchee.run( %{ - "public timeline with user block" => fn opts -> + "public timeline with user block" => fn -> ActivityPub.fetch_public_activities(opts) end }, - inputs: %{ - "old filtering" => Map.delete(opts, :method), - "with psql fun" => Map.put(opts, :method, :fun), - "with unnest" => Map.put(opts, :method, :unnest) - } ) domains = @@ -289,11 +276,6 @@ defmodule Pleroma.LoadTesting.Fetcher do "public timeline with domain block" => fn opts -> ActivityPub.fetch_public_activities(opts) end - }, - inputs: %{ - "old filtering" => Map.delete(opts, :method), - "with psql fun" => Map.put(opts, :method, :fun), - "with unnest" => Map.put(opts, :method, :unnest) } ) end -- cgit v1.2.3 From 317e2b8d6126d86eafb493fe6c3b7a29af65ee21 Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Thu, 4 Jun 2020 21:33:16 +0400 Subject: Use atoms as keys in `ActivityPub.fetch_*` functions options --- benchmarks/load_testing/fetcher.ex | 194 ++++++++++++++++++------------------- 1 file changed, 95 insertions(+), 99 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 22a06e472..15fd06c3d 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -52,12 +52,12 @@ defmodule Pleroma.LoadTesting.Fetcher do defp opts_for_home_timeline(user) do %{ - "blocking_user" => user, - "count" => "20", - "muting_user" => user, - "type" => ["Create", "Announce"], - "user" => user, - "with_muted" => "true" + blocking_user: user, + count: "20", + muting_user: user, + type: ["Create", "Announce"], + user: user, + with_muted: true } end @@ -70,17 +70,17 @@ defmodule Pleroma.LoadTesting.Fetcher do ActivityPub.fetch_activities(recipients, opts) |> Enum.reverse() |> List.last() second_page_last = - ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", first_page_last.id)) + ActivityPub.fetch_activities(recipients, Map.put(opts, :max_id, first_page_last.id)) |> Enum.reverse() |> List.last() third_page_last = - ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", second_page_last.id)) + ActivityPub.fetch_activities(recipients, Map.put(opts, :max_id, second_page_last.id)) |> Enum.reverse() |> List.last() forth_page_last = - ActivityPub.fetch_activities(recipients, Map.put(opts, "max_id", third_page_last.id)) + ActivityPub.fetch_activities(recipients, Map.put(opts, :max_id, third_page_last.id)) |> Enum.reverse() |> List.last() @@ -90,19 +90,19 @@ defmodule Pleroma.LoadTesting.Fetcher do }, inputs: %{ "1 page" => opts, - "2 page" => Map.put(opts, "max_id", first_page_last.id), - "3 page" => Map.put(opts, "max_id", second_page_last.id), - "4 page" => Map.put(opts, "max_id", third_page_last.id), - "5 page" => Map.put(opts, "max_id", forth_page_last.id), - "1 page only media" => Map.put(opts, "only_media", "true"), + "2 page" => Map.put(opts, :max_id, first_page_last.id), + "3 page" => Map.put(opts, :max_id, second_page_last.id), + "4 page" => Map.put(opts, :max_id, third_page_last.id), + "5 page" => Map.put(opts, :max_id, forth_page_last.id), + "1 page only media" => Map.put(opts, :only_media, true), "2 page only media" => - Map.put(opts, "max_id", first_page_last.id) |> Map.put("only_media", "true"), + Map.put(opts, :max_id, first_page_last.id) |> Map.put(:only_media, true), "3 page only media" => - Map.put(opts, "max_id", second_page_last.id) |> Map.put("only_media", "true"), + Map.put(opts, :max_id, second_page_last.id) |> Map.put(:only_media, true), "4 page only media" => - Map.put(opts, "max_id", third_page_last.id) |> Map.put("only_media", "true"), + Map.put(opts, :max_id, third_page_last.id) |> Map.put(:only_media, true), "5 page only media" => - Map.put(opts, "max_id", forth_page_last.id) |> Map.put("only_media", "true") + Map.put(opts, :max_id, forth_page_last.id) |> Map.put(:only_media, true) }, formatters: formatters() ) @@ -110,12 +110,12 @@ defmodule Pleroma.LoadTesting.Fetcher do defp opts_for_direct_timeline(user) do %{ - :visibility => "direct", - "blocking_user" => user, - "count" => "20", - "type" => "Create", - "user" => user, - "with_muted" => "true" + visibility: "direct", + blocking_user: user, + count: "20", + type: "Create", + user: user, + with_muted: true } end @@ -130,7 +130,7 @@ defmodule Pleroma.LoadTesting.Fetcher do |> Pagination.fetch_paginated(opts) |> List.last() - opts2 = Map.put(opts, "max_id", first_page_last.id) + opts2 = Map.put(opts, :max_id, first_page_last.id) second_page_last = recipients @@ -138,7 +138,7 @@ defmodule Pleroma.LoadTesting.Fetcher do |> Pagination.fetch_paginated(opts2) |> List.last() - opts3 = Map.put(opts, "max_id", second_page_last.id) + opts3 = Map.put(opts, :max_id, second_page_last.id) third_page_last = recipients @@ -146,7 +146,7 @@ defmodule Pleroma.LoadTesting.Fetcher do |> Pagination.fetch_paginated(opts3) |> List.last() - opts4 = Map.put(opts, "max_id", third_page_last.id) + opts4 = Map.put(opts, :max_id, third_page_last.id) forth_page_last = recipients @@ -165,7 +165,7 @@ defmodule Pleroma.LoadTesting.Fetcher do "2 page" => opts2, "3 page" => opts3, "4 page" => opts4, - "5 page" => Map.put(opts4, "max_id", forth_page_last.id) + "5 page" => Map.put(opts4, :max_id, forth_page_last.id) }, formatters: formatters() ) @@ -173,34 +173,34 @@ defmodule Pleroma.LoadTesting.Fetcher do defp opts_for_public_timeline(user) do %{ - "type" => ["Create", "Announce"], - "local_only" => false, - "blocking_user" => user, - "muting_user" => user + type: ["Create", "Announce"], + local_only: false, + blocking_user: user, + muting_user: user } end defp opts_for_public_timeline(user, :local) do %{ - "type" => ["Create", "Announce"], - "local_only" => true, - "blocking_user" => user, - "muting_user" => user + type: ["Create", "Announce"], + local_only: true, + blocking_user: user, + muting_user: user } end defp opts_for_public_timeline(user, :tag) do %{ - "blocking_user" => user, - "count" => "20", - "local_only" => nil, - "muting_user" => user, - "tag" => ["tag"], - "tag_all" => [], - "tag_reject" => [], - "type" => "Create", - "user" => user, - "with_muted" => "true" + blocking_user: user, + count: "20", + local_only: nil, + muting_user: user, + tag: ["tag"], + tag_all: [], + tag_reject: [], + type: "Create", + user: user, + with_muted: true } end @@ -223,7 +223,7 @@ defmodule Pleroma.LoadTesting.Fetcher do end defp fetch_public_timeline(user, :only_media) do - opts = opts_for_public_timeline(user) |> Map.put("only_media", "true") + opts = opts_for_public_timeline(user) |> Map.put(:only_media, true) fetch_public_timeline(opts, "public timeline only media") end @@ -245,15 +245,13 @@ defmodule Pleroma.LoadTesting.Fetcher do user = User.get_by_id(user.id) - opts = Map.put(opts, "blocking_user", user) + opts = Map.put(opts, :blocking_user, user) - Benchee.run( - %{ - "public timeline with user block" => fn -> - ActivityPub.fetch_public_activities(opts) - end - }, - ) + Benchee.run(%{ + "public timeline with user block" => fn -> + ActivityPub.fetch_public_activities(opts) + end + }) domains = Enum.reduce(remote_non_friends, [], fn non_friend, domains -> @@ -269,30 +267,28 @@ defmodule Pleroma.LoadTesting.Fetcher do end) user = User.get_by_id(user.id) - opts = Map.put(opts, "blocking_user", user) + opts = Map.put(opts, :blocking_user, user) - Benchee.run( - %{ - "public timeline with domain block" => fn opts -> - ActivityPub.fetch_public_activities(opts) - end - } - ) + Benchee.run(%{ + "public timeline with domain block" => fn -> + ActivityPub.fetch_public_activities(opts) + end + }) end defp fetch_public_timeline(opts, title) when is_binary(title) do first_page_last = ActivityPub.fetch_public_activities(opts) |> List.last() second_page_last = - ActivityPub.fetch_public_activities(Map.put(opts, "max_id", first_page_last.id)) + ActivityPub.fetch_public_activities(Map.put(opts, :max_id, first_page_last.id)) |> List.last() third_page_last = - ActivityPub.fetch_public_activities(Map.put(opts, "max_id", second_page_last.id)) + ActivityPub.fetch_public_activities(Map.put(opts, :max_id, second_page_last.id)) |> List.last() forth_page_last = - ActivityPub.fetch_public_activities(Map.put(opts, "max_id", third_page_last.id)) + ActivityPub.fetch_public_activities(Map.put(opts, :max_id, third_page_last.id)) |> List.last() Benchee.run( @@ -303,17 +299,17 @@ defmodule Pleroma.LoadTesting.Fetcher do }, inputs: %{ "1 page" => opts, - "2 page" => Map.put(opts, "max_id", first_page_last.id), - "3 page" => Map.put(opts, "max_id", second_page_last.id), - "4 page" => Map.put(opts, "max_id", third_page_last.id), - "5 page" => Map.put(opts, "max_id", forth_page_last.id) + "2 page" => Map.put(opts, :max_id, first_page_last.id), + "3 page" => Map.put(opts, :max_id, second_page_last.id), + "4 page" => Map.put(opts, :max_id, third_page_last.id), + "5 page" => Map.put(opts, :max_id, forth_page_last.id) }, formatters: formatters() ) end defp opts_for_notifications do - %{"count" => "20", "with_muted" => "true"} + %{count: "20", with_muted: true} end defp fetch_notifications(user) do @@ -322,15 +318,15 @@ defmodule Pleroma.LoadTesting.Fetcher do first_page_last = MastodonAPI.get_notifications(user, opts) |> List.last() second_page_last = - MastodonAPI.get_notifications(user, Map.put(opts, "max_id", first_page_last.id)) + MastodonAPI.get_notifications(user, Map.put(opts, :max_id, first_page_last.id)) |> List.last() third_page_last = - MastodonAPI.get_notifications(user, Map.put(opts, "max_id", second_page_last.id)) + MastodonAPI.get_notifications(user, Map.put(opts, :max_id, second_page_last.id)) |> List.last() forth_page_last = - MastodonAPI.get_notifications(user, Map.put(opts, "max_id", third_page_last.id)) + MastodonAPI.get_notifications(user, Map.put(opts, :max_id, third_page_last.id)) |> List.last() Benchee.run( @@ -341,10 +337,10 @@ defmodule Pleroma.LoadTesting.Fetcher do }, inputs: %{ "1 page" => opts, - "2 page" => Map.put(opts, "max_id", first_page_last.id), - "3 page" => Map.put(opts, "max_id", second_page_last.id), - "4 page" => Map.put(opts, "max_id", third_page_last.id), - "5 page" => Map.put(opts, "max_id", forth_page_last.id) + "2 page" => Map.put(opts, :max_id, first_page_last.id), + "3 page" => Map.put(opts, :max_id, second_page_last.id), + "4 page" => Map.put(opts, :max_id, third_page_last.id), + "5 page" => Map.put(opts, :max_id, forth_page_last.id) }, formatters: formatters() ) @@ -354,13 +350,13 @@ defmodule Pleroma.LoadTesting.Fetcher do first_page_last = ActivityPub.fetch_favourites(user) |> List.last() second_page_last = - ActivityPub.fetch_favourites(user, %{"max_id" => first_page_last.id}) |> List.last() + ActivityPub.fetch_favourites(user, %{:max_id => first_page_last.id}) |> List.last() third_page_last = - ActivityPub.fetch_favourites(user, %{"max_id" => second_page_last.id}) |> List.last() + ActivityPub.fetch_favourites(user, %{:max_id => second_page_last.id}) |> List.last() forth_page_last = - ActivityPub.fetch_favourites(user, %{"max_id" => third_page_last.id}) |> List.last() + ActivityPub.fetch_favourites(user, %{:max_id => third_page_last.id}) |> List.last() Benchee.run( %{ @@ -370,10 +366,10 @@ defmodule Pleroma.LoadTesting.Fetcher do }, inputs: %{ "1 page" => %{}, - "2 page" => %{"max_id" => first_page_last.id}, - "3 page" => %{"max_id" => second_page_last.id}, - "4 page" => %{"max_id" => third_page_last.id}, - "5 page" => %{"max_id" => forth_page_last.id} + "2 page" => %{:max_id => first_page_last.id}, + "3 page" => %{:max_id => second_page_last.id}, + "4 page" => %{:max_id => third_page_last.id}, + "5 page" => %{:max_id => forth_page_last.id} }, formatters: formatters() ) @@ -381,8 +377,8 @@ defmodule Pleroma.LoadTesting.Fetcher do defp opts_for_long_thread(user) do %{ - "blocking_user" => user, - "user" => user + blocking_user: user, + user: user } end @@ -392,9 +388,9 @@ defmodule Pleroma.LoadTesting.Fetcher do opts = opts_for_long_thread(user) - private_input = {private.data["context"], Map.put(opts, "exclude_id", private.id)} + private_input = {private.data["context"], Map.put(opts, :exclude_id, private.id)} - public_input = {public.data["context"], Map.put(opts, "exclude_id", public.id)} + public_input = {public.data["context"], Map.put(opts, :exclude_id, public.id)} Benchee.run( %{ @@ -514,13 +510,13 @@ defmodule Pleroma.LoadTesting.Fetcher do public_context = ActivityPub.fetch_activities_for_context( public.data["context"], - Map.put(fetch_opts, "exclude_id", public.id) + Map.put(fetch_opts, :exclude_id, public.id) ) private_context = ActivityPub.fetch_activities_for_context( private.data["context"], - Map.put(fetch_opts, "exclude_id", private.id) + Map.put(fetch_opts, :exclude_id, private.id) ) Benchee.run( @@ -551,14 +547,14 @@ defmodule Pleroma.LoadTesting.Fetcher do end, "Public timeline with reply filtering - following" => fn -> public_params - |> Map.put("reply_visibility", "following") - |> Map.put("reply_filtering_user", user) + |> Map.put(:reply_visibility, "following") + |> Map.put(:reply_filtering_user, user) |> ActivityPub.fetch_public_activities() end, "Public timeline with reply filtering - self" => fn -> public_params - |> Map.put("reply_visibility", "self") - |> Map.put("reply_filtering_user", user) + |> Map.put(:reply_visibility, "self") + |> Map.put(:reply_filtering_user, user) |> ActivityPub.fetch_public_activities() end }, @@ -577,16 +573,16 @@ defmodule Pleroma.LoadTesting.Fetcher do "Home timeline with reply filtering - following" => fn -> private_params = private_params - |> Map.put("reply_filtering_user", user) - |> Map.put("reply_visibility", "following") + |> Map.put(:reply_filtering_user, user) + |> Map.put(:reply_visibility, "following") ActivityPub.fetch_activities(recipients, private_params) end, "Home timeline with reply filtering - self" => fn -> private_params = private_params - |> Map.put("reply_filtering_user", user) - |> Map.put("reply_visibility", "self") + |> Map.put(:reply_filtering_user, user) + |> Map.put(:reply_visibility, "self") ActivityPub.fetch_activities(recipients, private_params) end -- cgit v1.2.3 From da509487b21bbb627e5fdac6815ad9b3e4e4728b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 31 Mar 2020 16:53:11 +0300 Subject: adding benchmarks in new format --- benchmarks/load_testing/activities.ex | 10 ++++++++++ benchmarks/load_testing/fetcher.ex | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 42 insertions(+), 2 deletions(-) (limited to 'benchmarks/load_testing') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 074ded457..f5c7bfce8 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -24,6 +24,7 @@ defmodule Pleroma.LoadTesting.Activities do @visibility ~w(public private direct unlisted) @types [ :simple, + :simple_filtered, :emoji, :mentions, :hell_thread, @@ -242,6 +243,15 @@ defmodule Pleroma.LoadTesting.Activities do insert_local_activity(visibility, group, users, "Simple status") end + defp insert_activity(:simple_filtered, visibility, group, users, _opts) + when group in @remote_groups do + insert_remote_activity(visibility, group, users, "Remote status which must be filtered") + end + + defp insert_activity(:simple_filtered, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Simple status which must be filtered") + end + defp insert_activity(:emoji, visibility, group, users, _opts) when group in @remote_groups do insert_remote_activity(visibility, group, users, "Remote status with emoji :firefox:") diff --git a/benchmarks/load_testing/fetcher.ex b/benchmarks/load_testing/fetcher.ex index 15fd06c3d..dfbd916be 100644 --- a/benchmarks/load_testing/fetcher.ex +++ b/benchmarks/load_testing/fetcher.ex @@ -32,10 +32,22 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end + defp create_filter(user) do + Pleroma.Filter.create(%Pleroma.Filter{ + user_id: user.id, + phrase: "must be filtered", + hide: true + }) + end + + defp delete_filter(filter), do: Repo.delete(filter) + defp fetch_timelines(user) do fetch_home_timeline(user) + fetch_home_timeline_with_filter(user) fetch_direct_timeline(user) fetch_public_timeline(user) + fetch_public_timeline_with_filter(user) fetch_public_timeline(user, :with_blocks) fetch_public_timeline(user, :local) fetch_public_timeline(user, :tag) @@ -61,7 +73,7 @@ defmodule Pleroma.LoadTesting.Fetcher do } end - defp fetch_home_timeline(user) do + defp fetch_home_timeline(user, title_end \\ "") do opts = opts_for_home_timeline(user) recipients = [user.ap_id | User.following(user)] @@ -84,9 +96,11 @@ defmodule Pleroma.LoadTesting.Fetcher do |> Enum.reverse() |> List.last() + title = "home timeline " <> title_end + Benchee.run( %{ - "home timeline" => fn opts -> ActivityPub.fetch_activities(recipients, opts) end + title => fn opts -> ActivityPub.fetch_activities(recipients, opts) end }, inputs: %{ "1 page" => opts, @@ -108,6 +122,14 @@ defmodule Pleroma.LoadTesting.Fetcher do ) end + defp fetch_home_timeline_with_filter(user) do + {:ok, filter} = create_filter(user) + + fetch_home_timeline(user, "with filters") + + delete_filter(filter) + end + defp opts_for_direct_timeline(user) do %{ visibility: "direct", @@ -210,6 +232,14 @@ defmodule Pleroma.LoadTesting.Fetcher do fetch_public_timeline(opts, "public timeline") end + defp fetch_public_timeline_with_filter(user) do + {:ok, filter} = create_filter(user) + opts = opts_for_public_timeline(user) + + fetch_public_timeline(opts, "public timeline with filters") + delete_filter(filter) + end + defp fetch_public_timeline(user, :local) do opts = opts_for_public_timeline(user, :local) -- cgit v1.2.3