From eb9744cadea7191b088ddaadfbd5fa4d4fd45090 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 14 Jan 2020 14:42:30 +0300 Subject: activities generation tasks --- benchmarks/load_testing/users.ex | 161 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 benchmarks/load_testing/users.ex (limited to 'benchmarks/load_testing/users.ex') diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex new file mode 100644 index 000000000..951b30d91 --- /dev/null +++ b/benchmarks/load_testing/users.ex @@ -0,0 +1,161 @@ +defmodule Pleroma.LoadTesting.Users do + @moduledoc """ + Module for generating users with friends. + """ + import Ecto.Query + import Pleroma.LoadTesting.Helper, only: [to_sec: 1] + + alias Pleroma.Repo + alias Pleroma.User + alias Pleroma.User.Query + + @defaults [ + users: 20_000, + friends: 100 + ] + + @max_concurrency 30 + + @spec generate(keyword()) :: User.t() + def generate(opts \\ []) do + opts = Keyword.merge(@defaults, opts) + + IO.puts("Starting generating #{opts[:users]} users...") + + {time, _} = :timer.tc(fn -> generate_users(opts[:users]) end) + + IO.puts("Generating users take #{to_sec(time)} sec.\n") + + main_user = + Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)) + + IO.puts("Starting making friends for #{opts[:friends]} users...") + {time, _} = :timer.tc(fn -> make_friends(main_user, opts[:friends]) end) + + IO.puts("Making friends take #{to_sec(time)} sec.\n") + + Repo.get(User, main_user.id) + end + + defp generate_users(max) do + Task.async_stream( + 1..max, + &generate_user(&1), + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end + + defp generate_user(i) do + remote = Enum.random([true, false]) + + %User{ + name: "Test ใƒ†ใ‚นใƒˆ User #{i}", + email: "user#{i}@example.com", + nickname: "nick#{i}", + password_hash: Comeonin.Pbkdf2.hashpwsalt("test"), + bio: "Tester Number #{i}", + local: !remote + } + |> user_urls() + |> Repo.insert!() + end + + defp user_urls(%{local: true} = user) do + urls = %{ + ap_id: User.ap_id(user), + follower_address: User.ap_followers(user), + following_address: User.ap_following(user) + } + + Map.merge(user, urls) + end + + defp user_urls(%{local: false} = user) do + base_domain = Enum.random(["domain1.com", "domain2.com", "domain3.com"]) + + ap_id = "https://#{base_domain}/users/#{user.nickname}" + + urls = %{ + ap_id: ap_id, + follower_address: ap_id <> "/followers", + following_address: ap_id <> "/following" + } + + Map.merge(user, urls) + end + + defp make_friends(main_user, max) when is_integer(max) do + number_of_users = + (max / 2) + |> Kernel.trunc() + + main_user + |> get_users(%{limit: number_of_users, local: :local}) + |> run_stream(main_user) + + main_user + |> get_users(%{limit: number_of_users, local: :external}) + |> run_stream(main_user) + end + + defp make_friends(%User{} = main_user, %User{} = user) do + {:ok, _} = User.follow(main_user, user) + {:ok, _} = User.follow(user, main_user) + end + + @spec get_users(User.t(), keyword()) :: [User.t()] + def get_users(user, opts) do + criteria = %{limit: opts[:limit]} + + criteria = + if opts[:local] do + Map.put(criteria, opts[:local], true) + else + criteria + end + + criteria = + if opts[:friends?] do + Map.put(criteria, :friends, user) + else + criteria + end + + query = + criteria + |> Query.build() + |> random_without_user(user) + + query = + if opts[:friends?] == false do + friends_ids = + %{friends: user} + |> Query.build() + |> Repo.all() + |> Enum.map(& &1.id) + + from(u in query, where: u.id not in ^friends_ids) + else + query + end + + Repo.all(query) + end + + defp random_without_user(query, user) do + from(u in query, + where: u.id != ^user.id, + order_by: fragment("RANDOM()") + ) + end + + defp run_stream(users, main_user) do + Task.async_stream(users, &make_friends(main_user, &1), + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end +end -- cgit v1.2.3 From 1f29ecdcd7ecdc4ad8d6bc8fc4c34efbc9b7fe1d Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 18 Feb 2020 12:19:10 +0300 Subject: sync with develop --- benchmarks/load_testing/users.ex | 61 ++++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 27 deletions(-) (limited to 'benchmarks/load_testing/users.ex') diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index 951b30d91..bc31dc08b 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -20,31 +20,31 @@ defmodule Pleroma.LoadTesting.Users do def generate(opts \\ []) do opts = Keyword.merge(@defaults, opts) - IO.puts("Starting generating #{opts[:users]} users...") - - {time, _} = :timer.tc(fn -> generate_users(opts[:users]) end) - - IO.puts("Generating users take #{to_sec(time)} sec.\n") + generate_users(opts[:users]) main_user = Repo.one(from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)) - IO.puts("Starting making friends for #{opts[:friends]} users...") - {time, _} = :timer.tc(fn -> make_friends(main_user, opts[:friends]) end) - - IO.puts("Making friends take #{to_sec(time)} sec.\n") + make_friends(main_user, opts[:friends]) Repo.get(User, main_user.id) end - defp generate_users(max) do - Task.async_stream( - 1..max, - &generate_user(&1), - max_concurrency: @max_concurrency, - timeout: 30_000 - ) - |> Stream.run() + def generate_users(max) do + IO.puts("Starting generating #{opts[:users]} users...") + + {time, _} = + :timer.tc(fn -> + Task.async_stream( + 1..max, + &generate_user(&1), + max_concurrency: @max_concurrency, + timeout: 30_000 + ) + |> Stream.run() + end) + + IO.puts("Generating users take #{to_sec(time)} sec.\n") end defp generate_user(i) do @@ -86,18 +86,25 @@ defmodule Pleroma.LoadTesting.Users do Map.merge(user, urls) end - defp make_friends(main_user, max) when is_integer(max) do - number_of_users = - (max / 2) - |> Kernel.trunc() + def make_friends(main_user, max) when is_integer(max) do + IO.puts("Starting making friends for #{opts[:friends]} users...") + + {time, _} = + :timer.tc(fn -> + number_of_users = + (max / 2) + |> Kernel.trunc() - main_user - |> get_users(%{limit: number_of_users, local: :local}) - |> run_stream(main_user) + main_user + |> get_users(%{limit: number_of_users, local: :local}) + |> run_stream(main_user) - main_user - |> get_users(%{limit: number_of_users, local: :external}) - |> run_stream(main_user) + main_user + |> get_users(%{limit: number_of_users, local: :external}) + |> run_stream(main_user) + end) + + IO.puts("Making friends take #{to_sec(time)} sec.\n") end defp make_friends(%User{} = main_user, %User{} = user) do -- cgit v1.2.3 From 56503c385e8412a1189748bcf3fdfd4090be9f56 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 17 Mar 2020 13:47:13 +0300 Subject: fix --- benchmarks/load_testing/users.ex | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'benchmarks/load_testing/users.ex') diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index bc31dc08b..b73ac8651 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -14,7 +14,7 @@ defmodule Pleroma.LoadTesting.Users do friends: 100 ] - @max_concurrency 30 + @max_concurrency 10 @spec generate(keyword()) :: User.t() def generate(opts \\ []) do @@ -31,7 +31,7 @@ defmodule Pleroma.LoadTesting.Users do end def generate_users(max) do - IO.puts("Starting generating #{opts[:users]} users...") + IO.puts("Starting generating #{max} users...") {time, _} = :timer.tc(fn -> @@ -87,7 +87,7 @@ defmodule Pleroma.LoadTesting.Users do end def make_friends(main_user, max) when is_integer(max) do - IO.puts("Starting making friends for #{opts[:friends]} users...") + IO.puts("Starting making friends for #{max} users...") {time, _} = :timer.tc(fn -> @@ -107,7 +107,7 @@ defmodule Pleroma.LoadTesting.Users do IO.puts("Making friends take #{to_sec(time)} sec.\n") end - defp make_friends(%User{} = main_user, %User{} = user) do + def make_friends(%User{} = main_user, %User{} = user) do {:ok, _} = User.follow(main_user, user) {:ok, _} = User.follow(user, main_user) end -- cgit v1.2.3 From 96e279655763fedcb701e59c500023a70568c4c6 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 30 Mar 2020 11:59:14 +0300 Subject: use in timelines benchmark new user generator --- benchmarks/load_testing/users.ex | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'benchmarks/load_testing/users.ex') diff --git a/benchmarks/load_testing/users.ex b/benchmarks/load_testing/users.ex index b73ac8651..1a8c6e22f 100644 --- a/benchmarks/load_testing/users.ex +++ b/benchmarks/load_testing/users.ex @@ -33,7 +33,7 @@ defmodule Pleroma.LoadTesting.Users do def generate_users(max) do IO.puts("Starting generating #{max} users...") - {time, _} = + {time, users} = :timer.tc(fn -> Task.async_stream( 1..max, @@ -41,10 +41,11 @@ defmodule Pleroma.LoadTesting.Users do max_concurrency: @max_concurrency, timeout: 30_000 ) - |> Stream.run() + |> Enum.to_list() end) - IO.puts("Generating users take #{to_sec(time)} sec.\n") + IO.puts("Generating users took #{to_sec(time)} sec.\n") + users end defp generate_user(i) do @@ -104,7 +105,7 @@ defmodule Pleroma.LoadTesting.Users do |> run_stream(main_user) end) - IO.puts("Making friends take #{to_sec(time)} sec.\n") + IO.puts("Making friends took #{to_sec(time)} sec.\n") end def make_friends(%User{} = main_user, %User{} = user) do -- cgit v1.2.3