From eb9744cadea7191b088ddaadfbd5fa4d4fd45090 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 14 Jan 2020 14:42:30 +0300 Subject: activities generation tasks --- benchmarks/load_testing/activities.ex | 515 ++++++++++++++++++++++++++++++++++ 1 file changed, 515 insertions(+) create mode 100644 benchmarks/load_testing/activities.ex (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex new file mode 100644 index 000000000..db0e5a66f --- /dev/null +++ b/benchmarks/load_testing/activities.ex @@ -0,0 +1,515 @@ +defmodule Pleroma.LoadTesting.Activities do + @moduledoc """ + Module for generating different activities. + """ + import Ecto.Query + import Pleroma.LoadTesting.Helper, only: [to_sec: 1] + + alias Ecto.UUID + alias Pleroma.Constants + alias Pleroma.LoadTesting.Users + alias Pleroma.Repo + alias Pleroma.Web.CommonAPI + + require Constants + + @defaults [ + iterations: 170, + friends_used: 20, + non_friends_used: 20 + ] + + @max_concurrency 30 + + @visibility ~w(public private direct unlisted) + @types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote) + @groups ~w(user friends non_friends) + + @spec generate(User.t(), keyword()) :: :ok + def generate(user, opts \\ []) do + {:ok, _} = + Agent.start_link(fn -> %{} end, + name: :benchmark_state + ) + + opts = Keyword.merge(@defaults, opts) + + friends = + user + |> Users.get_users(limit: opts[:friends_used], local: :local, friends?: true) + |> Enum.shuffle() + + non_friends = + user + |> Users.get_users(limit: opts[:non_friends_used], local: :local, friends?: false) + |> Enum.shuffle() + + task_data = + for visibility <- @visibility, + type <- @types, + group <- @groups, + do: {visibility, type, group} + + IO.puts("Starting generating #{opts[:iterations]} iterations of activities...") + + friends_thread = Enum.take(friends, 5) + non_friends_thread = Enum.take(friends, 5) + + public_long_thread = fn -> + generate_long_thread("public", user, friends_thread, non_friends_thread, opts) + end + + private_long_thread = fn -> + generate_long_thread("private", user, friends_thread, non_friends_thread, opts) + end + + iterations = opts[:iterations] + + {time, _} = + :timer.tc(fn -> + Enum.each( + 1..iterations, + fn + i when i == iterations - 2 -> + spawn(public_long_thread) + spawn(private_long_thread) + generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) + + _ -> + generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) + end + ) + end) + + IO.puts("Generating iterations activities take #{to_sec(time)} sec.\n") + :ok + end + + defp generate_long_thread(visibility, user, friends, non_friends, _opts) do + group = + if visibility == "public", + do: "friends", + else: "user" + + tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "Start of #{visibility} long thread", + "visibility" => visibility + }) + + Agent.update(:benchmark_state, fn state -> + key = + if visibility == "public", + do: :public_thread, + else: :private_thread + + Map.put(state, key, activity) + end) + + acc = {activity.id, ["@" <> user.nickname, "reply to long thread"]} + insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) + IO.puts("Generating #{visibility} long thread ended\n") + end + + defp insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) do + Enum.reduce(tasks, acc, fn + "friend", {id, data} -> + friend = Enum.random(friends) + insert_reply(friend, List.delete(data, "@" <> friend.nickname), id, visibility) + + "non_friend", {id, data} -> + non_friend = Enum.random(non_friends) + insert_reply(non_friend, List.delete(data, "@" <> non_friend.nickname), id, visibility) + + "user", {id, data} -> + insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility) + end) + end + + defp generate_activities(user, friends, non_friends, task_data, opts) do + Task.async_stream( + task_data, + fn {visibility, type, group} -> + insert_activity(type, visibility, group, user, friends, non_friends, opts) + end, + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end + + defp insert_activity("simple", visibility, group, user, friends, non_friends, _opts) do + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{"status" => "Simple status", "visibility" => visibility}) + end + + defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{ + "status" => "Simple status with emoji :firefox:", + "visibility" => visibility + }) + end + + defp insert_activity("mentions", visibility, group, user, friends, non_friends, _opts) do + user_mentions = + get_random_mentions(friends, Enum.random(0..3)) ++ + get_random_mentions(non_friends, Enum.random(0..3)) + + user_mentions = + if Enum.random([true, false]), + do: ["@" <> user.nickname | user_mentions], + else: user_mentions + + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{ + "status" => Enum.join(user_mentions, ", ") <> " simple status with mentions", + "visibility" => visibility + }) + end + + defp insert_activity("hell_thread", visibility, group, user, friends, non_friends, _opts) do + mentions = + with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do + cached = + ([user | Enum.take(friends, 10)] ++ Enum.take(non_friends, 10)) + |> Enum.map(&"@#{&1.nickname}") + |> Enum.join(", ") + + Cachex.put(:user_cache, "hell_thread_mentions", cached) + cached + else + {:ok, cached} -> cached + end + + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{ + "status" => mentions <> " hell thread status", + "visibility" => visibility + }) + end + + defp insert_activity("attachment", visibility, group, user, friends, non_friends, _opts) do + actor = get_actor(group, user, friends, non_friends) + + obj_data = %{ + "actor" => actor.ap_id, + "name" => "4467-11.jpg", + "type" => "Document", + "url" => [ + %{ + "href" => + "#{Pleroma.Web.base_url()}/media/b1b873552422a07bf53af01f3c231c841db4dfc42c35efde681abaf0f2a4eab7.jpg", + "mediaType" => "image/jpeg", + "type" => "Link" + } + ] + } + + object = Repo.insert!(%Pleroma.Object{data: obj_data}) + + {:ok, _activity} = + CommonAPI.post(actor, %{ + "status" => "Post with attachment", + "visibility" => visibility, + "media_ids" => [object.id] + }) + end + + defp insert_activity("tag", visibility, group, user, friends, non_friends, _opts) do + {:ok, _activity} = + group + |> get_actor(user, friends, non_friends) + |> CommonAPI.post(%{"status" => "Status with #tag", "visibility" => visibility}) + end + + defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do + actor = get_actor(group, user, friends, non_friends) + + with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), + {:ok, _activity, _object} <- CommonAPI.favorite(activity_id, actor) do + :ok + else + {:error, _} -> + insert_activity("like", visibility, group, user, friends, non_friends, opts) + + nil -> + Process.sleep(15) + insert_activity("like", visibility, group, user, friends, non_friends, opts) + end + end + + defp insert_activity("reblog", visibility, group, user, friends, non_friends, opts) do + actor = get_actor(group, user, friends, non_friends) + + with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), + {:ok, _activity, _object} <- CommonAPI.repeat(activity_id, actor) do + :ok + else + {:error, _} -> + insert_activity("reblog", visibility, group, user, friends, non_friends, opts) + + nil -> + Process.sleep(15) + insert_activity("reblog", visibility, group, user, friends, non_friends, opts) + end + end + + defp insert_activity("simple_thread", visibility, group, user, friends, non_friends, _opts) + when visibility in ["public", "unlisted", "private"] do + actor = get_actor(group, user, friends, non_friends) + tasks = get_reply_tasks(visibility, group) + + {:ok, activity} = + CommonAPI.post(user, %{"status" => "Simple status", "visibility" => "unlisted"}) + + acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} + insert_replies(tasks, visibility, user, friends, non_friends, acc) + end + + defp insert_activity("simple_thread", "direct", group, user, friends, non_friends, _opts) do + actor = get_actor(group, user, friends, non_friends) + tasks = get_reply_tasks("direct", group) + + list = + case group do + "non_friends" -> + Enum.take(non_friends, 3) + + _ -> + Enum.take(friends, 3) + end + + data = Enum.map(list, &("@" <> &1.nickname)) + + {:ok, activity} = + CommonAPI.post(actor, %{ + "status" => Enum.join(data, ", ") <> "simple status", + "visibility" => "direct" + }) + + acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]} + insert_direct_replies(tasks, user, list, acc) + end + + defp insert_activity("remote", _, "user", _, _, _, _), do: :ok + + defp insert_activity("remote", visibility, group, user, _friends, _non_friends, opts) do + remote_friends = + Users.get_users(user, limit: opts[:friends_used], local: :external, friends?: true) + + remote_non_friends = + Users.get_users(user, limit: opts[:non_friends_used], local: :external, friends?: false) + + actor = get_actor(group, user, remote_friends, remote_non_friends) + + {act_data, obj_data} = prepare_activity_data(actor, visibility, user) + {activity_data, object_data} = other_data(actor) + + activity_data + |> Map.merge(act_data) + |> Map.put("object", Map.merge(object_data, obj_data)) + |> Pleroma.Web.ActivityPub.ActivityPub.insert(false) + end + + defp get_actor("user", user, _friends, _non_friends), do: user + defp get_actor("friends", _user, friends, _non_friends), do: Enum.random(friends) + defp get_actor("non_friends", _user, _friends, non_friends), do: Enum.random(non_friends) + + defp other_data(actor) do + %{host: host} = URI.parse(actor.ap_id) + datetime = DateTime.utc_now() + context_id = "http://#{host}:4000/contexts/#{UUID.generate()}" + activity_id = "http://#{host}:4000/activities/#{UUID.generate()}" + object_id = "http://#{host}:4000/objects/#{UUID.generate()}" + + activity_data = %{ + "actor" => actor.ap_id, + "context" => context_id, + "id" => activity_id, + "published" => datetime, + "type" => "Create", + "directMessage" => false + } + + object_data = %{ + "actor" => actor.ap_id, + "attachment" => [], + "attributedTo" => actor.ap_id, + "bcc" => [], + "bto" => [], + "content" => "Remote post", + "context" => context_id, + "conversation" => context_id, + "emoji" => %{}, + "id" => object_id, + "published" => datetime, + "sensitive" => false, + "summary" => "", + "tag" => [], + "to" => ["https://www.w3.org/ns/activitystreams#Public"], + "type" => "Note" + } + + {activity_data, object_data} + end + + defp prepare_activity_data(actor, "public", _mention) do + obj_data = %{ + "cc" => [actor.follower_address], + "to" => [Constants.as_public()] + } + + act_data = %{ + "cc" => [actor.follower_address], + "to" => [Constants.as_public()] + } + + {act_data, obj_data} + end + + defp prepare_activity_data(actor, "private", _mention) do + obj_data = %{ + "cc" => [], + "to" => [actor.follower_address] + } + + act_data = %{ + "cc" => [], + "to" => [actor.follower_address] + } + + {act_data, obj_data} + end + + defp prepare_activity_data(actor, "unlisted", _mention) do + obj_data = %{ + "cc" => [Constants.as_public()], + "to" => [actor.follower_address] + } + + act_data = %{ + "cc" => [Constants.as_public()], + "to" => [actor.follower_address] + } + + {act_data, obj_data} + end + + defp prepare_activity_data(_actor, "direct", mention) do + %{host: mentioned_host} = URI.parse(mention.ap_id) + + obj_data = %{ + "cc" => [], + "content" => + "@#{ + mention.nickname + } direct message", + "tag" => [ + %{ + "href" => mention.ap_id, + "name" => "@#{mention.nickname}@#{mentioned_host}", + "type" => "Mention" + } + ], + "to" => [mention.ap_id] + } + + act_data = %{ + "cc" => [], + "directMessage" => true, + "to" => [mention.ap_id] + } + + {act_data, obj_data} + end + + defp get_reply_tasks("public", "user"), do: ~w(friend non_friend user) + defp get_reply_tasks("public", "friends"), do: ~w(non_friend user friend) + defp get_reply_tasks("public", "non_friends"), do: ~w(user friend non_friend) + + defp get_reply_tasks(visibility, "user") when visibility in ["unlisted", "private"], + do: ~w(friend user friend) + + defp get_reply_tasks(visibility, "friends") when visibility in ["unlisted", "private"], + do: ~w(user friend user) + + defp get_reply_tasks(visibility, "non_friends") when visibility in ["unlisted", "private"], + do: [] + + defp get_reply_tasks("direct", "user"), do: ~w(friend user friend) + defp get_reply_tasks("direct", "friends"), do: ~w(user friend user) + defp get_reply_tasks("direct", "non_friends"), do: ~w(user non_friend user) + + defp insert_replies(tasks, visibility, user, friends, non_friends, acc) do + Enum.reduce(tasks, acc, fn + "friend", {id, data} -> + friend = Enum.random(friends) + insert_reply(friend, data, id, visibility) + + "non_friend", {id, data} -> + non_friend = Enum.random(non_friends) + insert_reply(non_friend, data, id, visibility) + + "user", {id, data} -> + insert_reply(user, data, id, visibility) + end) + end + + defp insert_direct_replies(tasks, user, list, acc) do + Enum.reduce(tasks, acc, fn + group, {id, data} when group in ["friend", "non_friend"] -> + actor = Enum.random(list) + + {reply_id, _} = + insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct") + + {reply_id, data} + + "user", {id, data} -> + {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct") + {reply_id, data} + end) + end + + defp insert_reply(actor, data, activity_id, visibility) do + {:ok, reply} = + CommonAPI.post(actor, %{ + "status" => Enum.join(data, ", "), + "visibility" => visibility, + "in_reply_to_status_id" => activity_id + }) + + {reply.id, ["@" <> actor.nickname | data]} + end + + defp get_random_mentions(_users, count) when count == 0, do: [] + + defp get_random_mentions(users, count) do + users + |> Enum.shuffle() + |> Enum.take(count) + |> Enum.map(&"@#{&1.nickname}") + end + + defp get_random_create_activity_id do + Repo.one( + from(a in Pleroma.Activity, + where: fragment("(?)->>'type' = ?", a.data, ^"Create"), + order_by: fragment("RANDOM()"), + limit: 1, + select: a.id + ) + ) + end +end -- cgit v1.2.3 From 1f29ecdcd7ecdc4ad8d6bc8fc4c34efbc9b7fe1d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 18 Feb 2020 12:19:10 +0300 Subject: sync with develop --- benchmarks/load_testing/activities.ex | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index db0e5a66f..121d5c500 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -85,6 +85,48 @@ defmodule Pleroma.LoadTesting.Activities do :ok end + def generate_power_intervals(opts \\ []) do + count = Keyword.get(opts, :count, 20) + power = Keyword.get(opts, :power, 2) + IO.puts("Generating #{count} intervals for a power #{power} series...") + counts = Enum.map(1..count, fn n -> :math.pow(n, power) end) + sum = Enum.sum(counts) + + densities = + Enum.map(counts, fn c -> + c / sum + end) + + densities + |> Enum.reduce(0, fn density, acc -> + if acc == 0 do + [{0, density}] + else + [{_, lower} | _] = acc + [{lower, lower + density} | acc] + end + end) + |> Enum.reverse() + end + + def generate_tagged_activities(opts \\ []) do + tag_count = Keyword.get(opts, :tag_count, 20) + users = Keyword.get(opts, :users, Repo.all(Pleroma.User)) + activity_count = Keyword.get(opts, :count, 200_000) + + intervals = generate_power_intervals(count: tag_count) + + IO.puts( + "Generating #{activity_count} activities using #{tag_count} different tags of format `tag_n`, starting at tag_0" + ) + + Enum.each(1..activity_count, fn _ -> + random = :rand.uniform() + i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end) + CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"}) + end) + end + defp generate_long_thread(visibility, user, friends, non_friends, _opts) do group = if visibility == "public", -- cgit v1.2.3 From 56503c385e8412a1189748bcf3fdfd4090be9f56 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 17 Mar 2020 13:47:13 +0300 Subject: fix --- benchmarks/load_testing/activities.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 121d5c500..24c6b5531 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -19,7 +19,7 @@ defmodule Pleroma.LoadTesting.Activities do non_friends_used: 20 ] - @max_concurrency 30 + @max_concurrency 10 @visibility ~w(public private direct unlisted) @types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote) @@ -81,7 +81,7 @@ defmodule Pleroma.LoadTesting.Activities do ) end) - IO.puts("Generating iterations activities take #{to_sec(time)} sec.\n") + IO.puts("Generating iterations of activities take #{to_sec(time)} sec.\n") :ok end -- cgit v1.2.3 From 96e279655763fedcb701e59c500023a70568c4c6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 11:59:14 +0300 Subject: use in timelines benchmark new user generator --- benchmarks/load_testing/activities.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 24c6b5531..23ee2b987 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -81,7 +81,7 @@ defmodule Pleroma.LoadTesting.Activities do ) end) - IO.puts("Generating iterations of activities take #{to_sec(time)} sec.\n") + IO.puts("Generating iterations of activities took #{to_sec(time)} sec.\n") :ok end -- cgit v1.2.3 From 1a75ef63b2f6fef96b9bf9d07b4963fb217d4017 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 24 Apr 2020 09:24:08 +0300 Subject: updating benchmarks --- benchmarks/load_testing/activities.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 23ee2b987..2b032943b 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -313,7 +313,7 @@ defmodule Pleroma.LoadTesting.Activities do tasks = get_reply_tasks(visibility, group) {:ok, activity} = - CommonAPI.post(user, %{"status" => "Simple status", "visibility" => "unlisted"}) + CommonAPI.post(user, %{"status" => "Simple status", "visibility" => visibility}) acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} insert_replies(tasks, visibility, user, friends, non_friends, acc) -- cgit v1.2.3 From 375ab05234a3590c161a2a5a4f715fbc61dafb34 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Fri, 24 Apr 2020 09:57:30 +0300 Subject: bench sync --- benchmarks/load_testing/activities.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 2b032943b..482e42fc1 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -279,7 +279,7 @@ defmodule Pleroma.LoadTesting.Activities do actor = get_actor(group, user, friends, non_friends) with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), - {:ok, _activity, _object} <- CommonAPI.favorite(activity_id, actor) do + {:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do :ok else {:error, _} -> -- cgit v1.2.3 From 7b500d6b4fdad59c7242dc41646bd0bb5cc04e67 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 18 May 2020 13:56:58 +0200 Subject: Load Testing: adjust to new CommonAPI.post format. --- benchmarks/load_testing/activities.ex | 41 +++++++++++++++++------------------ 1 file changed, 20 insertions(+), 21 deletions(-) (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 482e42fc1..ff0d481a8 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -123,7 +123,7 @@ defmodule Pleroma.LoadTesting.Activities do Enum.each(1..activity_count, fn _ -> random = :rand.uniform() i = Enum.find_index(intervals, fn {lower, upper} -> lower <= random && upper > random end) - CommonAPI.post(Enum.random(users), %{"status" => "a post with the tag #tag_#{i}"}) + CommonAPI.post(Enum.random(users), %{status: "a post with the tag #tag_#{i}"}) end) end @@ -137,8 +137,8 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, activity} = CommonAPI.post(user, %{ - "status" => "Start of #{visibility} long thread", - "visibility" => visibility + status: "Start of #{visibility} long thread", + visibility: visibility }) Agent.update(:benchmark_state, fn state -> @@ -186,7 +186,7 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, _activity} = group |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{"status" => "Simple status", "visibility" => visibility}) + |> CommonAPI.post(%{status: "Simple status", visibility: visibility}) end defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do @@ -194,8 +194,8 @@ defmodule Pleroma.LoadTesting.Activities do group |> get_actor(user, friends, non_friends) |> CommonAPI.post(%{ - "status" => "Simple status with emoji :firefox:", - "visibility" => visibility + status: "Simple status with emoji :firefox:", + visibility: visibility }) end @@ -213,8 +213,8 @@ defmodule Pleroma.LoadTesting.Activities do group |> get_actor(user, friends, non_friends) |> CommonAPI.post(%{ - "status" => Enum.join(user_mentions, ", ") <> " simple status with mentions", - "visibility" => visibility + status: Enum.join(user_mentions, ", ") <> " simple status with mentions", + visibility: visibility }) end @@ -236,8 +236,8 @@ defmodule Pleroma.LoadTesting.Activities do group |> get_actor(user, friends, non_friends) |> CommonAPI.post(%{ - "status" => mentions <> " hell thread status", - "visibility" => visibility + status: mentions <> " hell thread status", + visibility: visibility }) end @@ -262,9 +262,9 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, _activity} = CommonAPI.post(actor, %{ - "status" => "Post with attachment", - "visibility" => visibility, - "media_ids" => [object.id] + status: "Post with attachment", + visibility: visibility, + media_ids: [object.id] }) end @@ -272,7 +272,7 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, _activity} = group |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{"status" => "Status with #tag", "visibility" => visibility}) + |> CommonAPI.post(%{status: "Status with #tag", visibility: visibility}) end defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do @@ -312,8 +312,7 @@ defmodule Pleroma.LoadTesting.Activities do actor = get_actor(group, user, friends, non_friends) tasks = get_reply_tasks(visibility, group) - {:ok, activity} = - CommonAPI.post(user, %{"status" => "Simple status", "visibility" => visibility}) + {:ok, activity} = CommonAPI.post(user, %{status: "Simple status", visibility: visibility}) acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} insert_replies(tasks, visibility, user, friends, non_friends, acc) @@ -336,8 +335,8 @@ defmodule Pleroma.LoadTesting.Activities do {:ok, activity} = CommonAPI.post(actor, %{ - "status" => Enum.join(data, ", ") <> "simple status", - "visibility" => "direct" + status: Enum.join(data, ", ") <> "simple status", + visibility: "direct" }) acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]} @@ -527,9 +526,9 @@ defmodule Pleroma.LoadTesting.Activities do defp insert_reply(actor, data, activity_id, visibility) do {:ok, reply} = CommonAPI.post(actor, %{ - "status" => Enum.join(data, ", "), - "visibility" => visibility, - "in_reply_to_status_id" => activity_id + status: Enum.join(data, ", "), + visibility: visibility, + in_reply_to_status_id: activity_id }) {reply.id, ["@" <> actor.nickname | data]} -- cgit v1.2.3 From 7e6ec778d965419ed4083428d4d39b2a689f7619 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 20 May 2020 17:45:06 +0300 Subject: exclude replies on blocked domains --- benchmarks/load_testing/activities.ex | 361 ++++++++++++++++++---------------- 1 file changed, 195 insertions(+), 166 deletions(-) (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index ff0d481a8..074ded457 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -22,8 +22,21 @@ defmodule Pleroma.LoadTesting.Activities do @max_concurrency 10 @visibility ~w(public private direct unlisted) - @types ~w(simple emoji mentions hell_thread attachment tag like reblog simple_thread remote) - @groups ~w(user friends non_friends) + @types [ + :simple, + :emoji, + :mentions, + :hell_thread, + :attachment, + :tag, + :like, + :reblog, + :simple_thread + ] + @groups [:friends_local, :friends_remote, :non_friends_local, :non_friends_local] + @remote_groups [:friends_remote, :non_friends_remote] + @friends_groups [:friends_local, :friends_remote] + @non_friends_groups [:non_friends_local, :non_friends_remote] @spec generate(User.t(), keyword()) :: :ok def generate(user, opts \\ []) do @@ -34,33 +47,24 @@ defmodule Pleroma.LoadTesting.Activities do opts = Keyword.merge(@defaults, opts) - friends = - user - |> Users.get_users(limit: opts[:friends_used], local: :local, friends?: true) - |> Enum.shuffle() + users = Users.prepare_users(user, opts) - non_friends = - user - |> Users.get_users(limit: opts[:non_friends_used], local: :local, friends?: false) - |> Enum.shuffle() + {:ok, _} = Agent.start_link(fn -> users[:non_friends_remote] end, name: :non_friends_remote) task_data = for visibility <- @visibility, type <- @types, - group <- @groups, + group <- [:user | @groups], do: {visibility, type, group} IO.puts("Starting generating #{opts[:iterations]} iterations of activities...") - friends_thread = Enum.take(friends, 5) - non_friends_thread = Enum.take(friends, 5) - public_long_thread = fn -> - generate_long_thread("public", user, friends_thread, non_friends_thread, opts) + generate_long_thread("public", users, opts) end private_long_thread = fn -> - generate_long_thread("private", user, friends_thread, non_friends_thread, opts) + generate_long_thread("private", users, opts) end iterations = opts[:iterations] @@ -73,10 +77,10 @@ defmodule Pleroma.LoadTesting.Activities do i when i == iterations - 2 -> spawn(public_long_thread) spawn(private_long_thread) - generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) + generate_activities(users, Enum.shuffle(task_data), opts) _ -> - generate_activities(user, friends, non_friends, Enum.shuffle(task_data), opts) + generate_activities(users, Enum.shuffle(task_data), opts) end ) end) @@ -127,16 +131,16 @@ defmodule Pleroma.LoadTesting.Activities do end) end - defp generate_long_thread(visibility, user, friends, non_friends, _opts) do + defp generate_long_thread(visibility, users, _opts) do group = if visibility == "public", - do: "friends", - else: "user" + do: :friends_local, + else: :user tasks = get_reply_tasks(visibility, group) |> Stream.cycle() |> Enum.take(50) {:ok, activity} = - CommonAPI.post(user, %{ + CommonAPI.post(users[:user], %{ status: "Start of #{visibility} long thread", visibility: visibility }) @@ -150,31 +154,28 @@ defmodule Pleroma.LoadTesting.Activities do Map.put(state, key, activity) end) - acc = {activity.id, ["@" <> user.nickname, "reply to long thread"]} - insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) + acc = {activity.id, ["@" <> users[:user].nickname, "reply to long thread"]} + insert_replies_for_long_thread(tasks, visibility, users, acc) IO.puts("Generating #{visibility} long thread ended\n") end - defp insert_replies_for_long_thread(tasks, visibility, user, friends, non_friends, acc) do + defp insert_replies_for_long_thread(tasks, visibility, users, acc) do Enum.reduce(tasks, acc, fn - "friend", {id, data} -> - friend = Enum.random(friends) - insert_reply(friend, List.delete(data, "@" <> friend.nickname), id, visibility) - - "non_friend", {id, data} -> - non_friend = Enum.random(non_friends) - insert_reply(non_friend, List.delete(data, "@" <> non_friend.nickname), id, visibility) - - "user", {id, data} -> + :user, {id, data} -> + user = users[:user] insert_reply(user, List.delete(data, "@" <> user.nickname), id, visibility) + + group, {id, data} -> + replier = Enum.random(users[group]) + insert_reply(replier, List.delete(data, "@" <> replier.nickname), id, visibility) end) end - defp generate_activities(user, friends, non_friends, task_data, opts) do + defp generate_activities(users, task_data, opts) do Task.async_stream( task_data, fn {visibility, type, group} -> - insert_activity(type, visibility, group, user, friends, non_friends, opts) + insert_activity(type, visibility, group, users, opts) end, max_concurrency: @max_concurrency, timeout: 30_000 @@ -182,67 +183,104 @@ defmodule Pleroma.LoadTesting.Activities do |> Stream.run() end - defp insert_activity("simple", visibility, group, user, friends, non_friends, _opts) do - {:ok, _activity} = + defp insert_local_activity(visibility, group, users, status) do + {:ok, _} = group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{status: "Simple status", visibility: visibility}) + |> get_actor(users) + |> CommonAPI.post(%{status: status, visibility: visibility}) end - defp insert_activity("emoji", visibility, group, user, friends, non_friends, _opts) do - {:ok, _activity} = - group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{ - status: "Simple status with emoji :firefox:", - visibility: visibility - }) + defp insert_remote_activity(visibility, group, users, status) do + actor = get_actor(group, users) + {act_data, obj_data} = prepare_activity_data(actor, visibility, users[:user]) + {activity_data, object_data} = other_data(actor, status) + + activity_data + |> Map.merge(act_data) + |> Map.put("object", Map.merge(object_data, obj_data)) + |> Pleroma.Web.ActivityPub.ActivityPub.insert(false) end - defp insert_activity("mentions", visibility, group, user, friends, non_friends, _opts) do + defp user_mentions(users) do user_mentions = - get_random_mentions(friends, Enum.random(0..3)) ++ - get_random_mentions(non_friends, Enum.random(0..3)) + Enum.reduce( + @groups, + [], + fn group, acc -> + acc ++ get_random_mentions(users[group], Enum.random(0..2)) + end + ) - user_mentions = - if Enum.random([true, false]), - do: ["@" <> user.nickname | user_mentions], - else: user_mentions + if Enum.random([true, false]), + do: ["@" <> users[:user].nickname | user_mentions], + else: user_mentions + end - {:ok, _activity} = - group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{ - status: Enum.join(user_mentions, ", ") <> " simple status with mentions", - visibility: visibility - }) + defp hell_thread_mentions(users) do + with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do + cached = + @groups + |> Enum.reduce([users[:user]], fn group, acc -> + acc ++ Enum.take(users[group], 5) + end) + |> Enum.map(&"@#{&1.nickname}") + |> Enum.join(", ") + + Cachex.put(:user_cache, "hell_thread_mentions", cached) + cached + else + {:ok, cached} -> cached + end end - defp insert_activity("hell_thread", visibility, group, user, friends, non_friends, _opts) do - mentions = - with {:ok, nil} <- Cachex.get(:user_cache, "hell_thread_mentions") do - cached = - ([user | Enum.take(friends, 10)] ++ Enum.take(non_friends, 10)) - |> Enum.map(&"@#{&1.nickname}") - |> Enum.join(", ") + defp insert_activity(:simple, visibility, group, users, _opts) + when group in @remote_groups do + insert_remote_activity(visibility, group, users, "Remote status") + end - Cachex.put(:user_cache, "hell_thread_mentions", cached) - cached - else - {:ok, cached} -> cached - end + defp insert_activity(:simple, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Simple status") + end - {:ok, _activity} = - group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{ - status: mentions <> " hell thread status", - visibility: visibility - }) + defp insert_activity(:emoji, visibility, group, users, _opts) + when group in @remote_groups do + insert_remote_activity(visibility, group, users, "Remote status with emoji :firefox:") + end + + defp insert_activity(:emoji, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Simple status with emoji :firefox:") + end + + defp insert_activity(:mentions, visibility, group, users, _opts) + when group in @remote_groups do + mentions = user_mentions(users) + + status = Enum.join(mentions, ", ") <> " remote status with mentions" + + insert_remote_activity(visibility, group, users, status) + end + + defp insert_activity(:mentions, visibility, group, users, _opts) do + mentions = user_mentions(users) + + status = Enum.join(mentions, ", ") <> " simple status with mentions" + insert_remote_activity(visibility, group, users, status) + end + + defp insert_activity(:hell_thread, visibility, group, users, _) + when group in @remote_groups do + mentions = hell_thread_mentions(users) + insert_remote_activity(visibility, group, users, mentions <> " remote hell thread status") + end + + defp insert_activity(:hell_thread, visibility, group, users, _opts) do + mentions = hell_thread_mentions(users) + + insert_local_activity(visibility, group, users, mentions <> " hell thread status") end - defp insert_activity("attachment", visibility, group, user, friends, non_friends, _opts) do - actor = get_actor(group, user, friends, non_friends) + defp insert_activity(:attachment, visibility, group, users, _opts) do + actor = get_actor(group, users) obj_data = %{ "actor" => actor.ap_id, @@ -268,67 +306,54 @@ defmodule Pleroma.LoadTesting.Activities do }) end - defp insert_activity("tag", visibility, group, user, friends, non_friends, _opts) do - {:ok, _activity} = - group - |> get_actor(user, friends, non_friends) - |> CommonAPI.post(%{status: "Status with #tag", visibility: visibility}) + defp insert_activity(:tag, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Status with #tag") end - defp insert_activity("like", visibility, group, user, friends, non_friends, opts) do - actor = get_actor(group, user, friends, non_friends) + defp insert_activity(:like, visibility, group, users, opts) do + actor = get_actor(group, users) with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), {:ok, _activity} <- CommonAPI.favorite(actor, activity_id) do :ok else {:error, _} -> - insert_activity("like", visibility, group, user, friends, non_friends, opts) + insert_activity(:like, visibility, group, users, opts) nil -> Process.sleep(15) - insert_activity("like", visibility, group, user, friends, non_friends, opts) + insert_activity(:like, visibility, group, users, opts) end end - defp insert_activity("reblog", visibility, group, user, friends, non_friends, opts) do - actor = get_actor(group, user, friends, non_friends) + defp insert_activity(:reblog, visibility, group, users, opts) do + actor = get_actor(group, users) with activity_id when not is_nil(activity_id) <- get_random_create_activity_id(), - {:ok, _activity, _object} <- CommonAPI.repeat(activity_id, actor) do + {:ok, _activity} <- CommonAPI.repeat(activity_id, actor) do :ok else {:error, _} -> - insert_activity("reblog", visibility, group, user, friends, non_friends, opts) + insert_activity(:reblog, visibility, group, users, opts) nil -> Process.sleep(15) - insert_activity("reblog", visibility, group, user, friends, non_friends, opts) + insert_activity(:reblog, visibility, group, users, opts) end end - defp insert_activity("simple_thread", visibility, group, user, friends, non_friends, _opts) - when visibility in ["public", "unlisted", "private"] do - actor = get_actor(group, user, friends, non_friends) - tasks = get_reply_tasks(visibility, group) - - {:ok, activity} = CommonAPI.post(user, %{status: "Simple status", visibility: visibility}) - - acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} - insert_replies(tasks, visibility, user, friends, non_friends, acc) - end - - defp insert_activity("simple_thread", "direct", group, user, friends, non_friends, _opts) do - actor = get_actor(group, user, friends, non_friends) + defp insert_activity(:simple_thread, "direct", group, users, _opts) do + actor = get_actor(group, users) tasks = get_reply_tasks("direct", group) list = case group do - "non_friends" -> - Enum.take(non_friends, 3) + :user -> + group = Enum.random(@friends_groups) + Enum.take(users[group], 3) _ -> - Enum.take(friends, 3) + Enum.take(users[group], 3) end data = Enum.map(list, &("@" <> &1.nickname)) @@ -339,40 +364,30 @@ defmodule Pleroma.LoadTesting.Activities do visibility: "direct" }) - acc = {activity.id, ["@" <> user.nickname | data] ++ ["reply to status"]} - insert_direct_replies(tasks, user, list, acc) + acc = {activity.id, ["@" <> users[:user].nickname | data] ++ ["reply to status"]} + insert_direct_replies(tasks, users[:user], list, acc) end - defp insert_activity("remote", _, "user", _, _, _, _), do: :ok - - defp insert_activity("remote", visibility, group, user, _friends, _non_friends, opts) do - remote_friends = - Users.get_users(user, limit: opts[:friends_used], local: :external, friends?: true) - - remote_non_friends = - Users.get_users(user, limit: opts[:non_friends_used], local: :external, friends?: false) - - actor = get_actor(group, user, remote_friends, remote_non_friends) + defp insert_activity(:simple_thread, visibility, group, users, _opts) do + actor = get_actor(group, users) + tasks = get_reply_tasks(visibility, group) - {act_data, obj_data} = prepare_activity_data(actor, visibility, user) - {activity_data, object_data} = other_data(actor) + {:ok, activity} = + CommonAPI.post(users[:user], %{status: "Simple status", visibility: visibility}) - activity_data - |> Map.merge(act_data) - |> Map.put("object", Map.merge(object_data, obj_data)) - |> Pleroma.Web.ActivityPub.ActivityPub.insert(false) + acc = {activity.id, ["@" <> actor.nickname, "reply to status"]} + insert_replies(tasks, visibility, users, acc) end - defp get_actor("user", user, _friends, _non_friends), do: user - defp get_actor("friends", _user, friends, _non_friends), do: Enum.random(friends) - defp get_actor("non_friends", _user, _friends, non_friends), do: Enum.random(non_friends) + defp get_actor(:user, %{user: user}), do: user + defp get_actor(group, users), do: Enum.random(users[group]) - defp other_data(actor) do + defp other_data(actor, content) do %{host: host} = URI.parse(actor.ap_id) datetime = DateTime.utc_now() - context_id = "http://#{host}:4000/contexts/#{UUID.generate()}" - activity_id = "http://#{host}:4000/activities/#{UUID.generate()}" - object_id = "http://#{host}:4000/objects/#{UUID.generate()}" + context_id = "https://#{host}/contexts/#{UUID.generate()}" + activity_id = "https://#{host}/activities/#{UUID.generate()}" + object_id = "https://#{host}/objects/#{UUID.generate()}" activity_data = %{ "actor" => actor.ap_id, @@ -389,7 +404,7 @@ defmodule Pleroma.LoadTesting.Activities do "attributedTo" => actor.ap_id, "bcc" => [], "bto" => [], - "content" => "Remote post", + "content" => content, "context" => context_id, "conversation" => context_id, "emoji" => %{}, @@ -475,51 +490,65 @@ defmodule Pleroma.LoadTesting.Activities do {act_data, obj_data} end - defp get_reply_tasks("public", "user"), do: ~w(friend non_friend user) - defp get_reply_tasks("public", "friends"), do: ~w(non_friend user friend) - defp get_reply_tasks("public", "non_friends"), do: ~w(user friend non_friend) + defp get_reply_tasks("public", :user) do + [:friends_local, :friends_remote, :non_friends_local, :non_friends_remote, :user] + end + + defp get_reply_tasks("public", group) when group in @friends_groups do + [:non_friends_local, :non_friends_remote, :user, :friends_local, :friends_remote] + end - defp get_reply_tasks(visibility, "user") when visibility in ["unlisted", "private"], - do: ~w(friend user friend) + defp get_reply_tasks("public", group) when group in @non_friends_groups do + [:user, :friends_local, :friends_remote, :non_friends_local, :non_friends_remote] + end - defp get_reply_tasks(visibility, "friends") when visibility in ["unlisted", "private"], - do: ~w(user friend user) + defp get_reply_tasks(visibility, :user) when visibility in ["unlisted", "private"] do + [:friends_local, :friends_remote, :user, :friends_local, :friends_remote] + end - defp get_reply_tasks(visibility, "non_friends") when visibility in ["unlisted", "private"], - do: [] + defp get_reply_tasks(visibility, group) + when visibility in ["unlisted", "private"] and group in @friends_groups do + [:user, :friends_remote, :friends_local, :user] + end - defp get_reply_tasks("direct", "user"), do: ~w(friend user friend) - defp get_reply_tasks("direct", "friends"), do: ~w(user friend user) - defp get_reply_tasks("direct", "non_friends"), do: ~w(user non_friend user) + defp get_reply_tasks(visibility, group) + when visibility in ["unlisted", "private"] and + group in @non_friends_groups, + do: [] - defp insert_replies(tasks, visibility, user, friends, non_friends, acc) do - Enum.reduce(tasks, acc, fn - "friend", {id, data} -> - friend = Enum.random(friends) - insert_reply(friend, data, id, visibility) + defp get_reply_tasks("direct", :user), do: [:friends_local, :user, :friends_remote] - "non_friend", {id, data} -> - non_friend = Enum.random(non_friends) - insert_reply(non_friend, data, id, visibility) + defp get_reply_tasks("direct", group) when group in @friends_groups, + do: [:user, group, :user] - "user", {id, data} -> - insert_reply(user, data, id, visibility) + defp get_reply_tasks("direct", group) when group in @non_friends_groups do + [:user, :non_friends_remote, :user, :non_friends_local] + end + + defp insert_replies(tasks, visibility, users, acc) do + Enum.reduce(tasks, acc, fn + :user, {id, data} -> + insert_reply(users[:user], data, id, visibility) + + group, {id, data} -> + replier = Enum.random(users[group]) + insert_reply(replier, data, id, visibility) end) end defp insert_direct_replies(tasks, user, list, acc) do Enum.reduce(tasks, acc, fn - group, {id, data} when group in ["friend", "non_friend"] -> + :user, {id, data} -> + {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct") + {reply_id, data} + + _, {id, data} -> actor = Enum.random(list) {reply_id, _} = insert_reply(actor, List.delete(data, "@" <> actor.nickname), id, "direct") {reply_id, data} - - "user", {id, data} -> - {reply_id, _} = insert_reply(user, List.delete(data, "@" <> user.nickname), id, "direct") - {reply_id, data} end) end -- cgit v1.2.3 From da509487b21bbb627e5fdac6815ad9b3e4e4728b Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 31 Mar 2020 16:53:11 +0300 Subject: adding benchmarks in new format --- benchmarks/load_testing/activities.ex | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'benchmarks/load_testing/activities.ex') diff --git a/benchmarks/load_testing/activities.ex b/benchmarks/load_testing/activities.ex index 074ded457..f5c7bfce8 100644 --- a/benchmarks/load_testing/activities.ex +++ b/benchmarks/load_testing/activities.ex @@ -24,6 +24,7 @@ defmodule Pleroma.LoadTesting.Activities do @visibility ~w(public private direct unlisted) @types [ :simple, + :simple_filtered, :emoji, :mentions, :hell_thread, @@ -242,6 +243,15 @@ defmodule Pleroma.LoadTesting.Activities do insert_local_activity(visibility, group, users, "Simple status") end + defp insert_activity(:simple_filtered, visibility, group, users, _opts) + when group in @remote_groups do + insert_remote_activity(visibility, group, users, "Remote status which must be filtered") + end + + defp insert_activity(:simple_filtered, visibility, group, users, _opts) do + insert_local_activity(visibility, group, users, "Simple status which must be filtered") + end + defp insert_activity(:emoji, visibility, group, users, _opts) when group in @remote_groups do insert_remote_activity(visibility, group, users, "Remote status with emoji :firefox:") -- cgit v1.2.3