Merge branch 'develop' into 'remove-twitter-api'

# Conflicts:
#   lib/pleroma/web/nodeinfo/nodeinfo_controller.ex

3 files changed, 113 insertions, 118 deletions

diff --git a/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex
index fd1506907..657403202 100644
--- a/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex
+++ b/benchmarks/mix/tasks/pleroma/benchmarks/tags.ex
@@ -1,9 +1,12 @@
 defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
   use Mix.Task
-  alias Pleroma.Repo
-  alias Pleroma.LoadTesting.Generator
+
+  import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
   import Ecto.Query
 
+  alias Pleroma.Repo
+  alias Pleroma.Web.MastodonAPI.TimelineController
+
   def run(_args) do
     Mix.Pleroma.start_pleroma()
     activities_count = Repo.aggregate(from(a in Pleroma.Activity), :count, :id)
@@ -11,8 +14,8 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
     if activities_count == 0 do
       IO.puts("Did not find any activities, cleaning and generating")
       clean_tables()
-      Generator.generate_users(users_max: 10)
-      Generator.generate_tagged_activities()
+      Pleroma.LoadTesting.Users.generate_users(10)
+      Pleroma.LoadTesting.Activities.generate_tagged_activities()
     else
       IO.puts("Found #{activities_count} activities, won't generate new ones")
     end
@@ -34,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
     Benchee.run(
       %{
         "Hashtag fetching, any" => fn tags ->
-          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+          TimelineController.hashtag_fetching(
             %{
               "any" => tags
             },
@@ -44,7 +47,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
         end,
         # Will always return zero results because no overlapping hashtags are generated.
         "Hashtag fetching, all" => fn tags ->
-          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+          TimelineController.hashtag_fetching(
             %{
               "all" => tags
             },
@@ -64,7 +67,7 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
     Benchee.run(
       %{
         "Hashtag fetching" => fn tag ->
-          Pleroma.Web.MastodonAPI.TimelineController.hashtag_fetching(
+          TimelineController.hashtag_fetching(
             %{
               "tag" => tag
             },
@@ -77,11 +80,4 @@ defmodule Mix.Tasks.Pleroma.Benchmarks.Tags do
       time: 5
     )
   end
-
-  defp clean_tables do
-    IO.puts("Deleting old data...\n")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
-  end
 end
diff --git a/benchmarks/mix/tasks/pleroma/benchmarks/timelines.ex b/benchmarks/mix/tasks/pleroma/benchmarks/timelines.ex
new file mode 100644
index 000000000..9b7ac6111
--- /dev/null
+++ b/benchmarks/mix/tasks/pleroma/benchmarks/timelines.ex
@@ -0,0 +1,70 @@
+defmodule Mix.Tasks.Pleroma.Benchmarks.Timelines do
+  use Mix.Task
+
+  import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
+
+  alias Pleroma.Web.CommonAPI
+  alias Plug.Conn
+
+  def run(_args) do
+    Mix.Pleroma.start_pleroma()
+
+    # Cleaning tables
+    clean_tables()
+
+    [{:ok, user} | users] = Pleroma.LoadTesting.Users.generate_users(1000)
+
+    # Let the user make 100 posts
+
+    1..100
+    |> Enum.each(fn i -> CommonAPI.post(user, %{"status" => to_string(i)}) end)
+
+    # Let 10 random users post
+    posts =
+      users
+      |> Enum.take_random(10)
+      |> Enum.map(fn {:ok, random_user} ->
+        {:ok, activity} = CommonAPI.post(random_user, %{"status" => "."})
+        activity
+      end)
+
+    # let our user repeat them
+    posts
+    |> Enum.each(fn activity ->
+      CommonAPI.repeat(activity.id, user)
+    end)
+
+    Benchee.run(
+      %{
+        "user timeline, no followers" => fn reading_user ->
+          conn =
+            Phoenix.ConnTest.build_conn()
+            |> Conn.assign(:user, reading_user)
+            |> Conn.assign(:skip_link_headers, true)
+
+          Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
+        end
+      },
+      inputs: %{"user" => user, "no user" => nil},
+      time: 60
+    )
+
+    users
+    |> Enum.each(fn {:ok, follower} -> Pleroma.User.follow(follower, user) end)
+
+    Benchee.run(
+      %{
+        "user timeline, all following" => fn reading_user ->
+          conn =
+            Phoenix.ConnTest.build_conn()
+            |> Conn.assign(:user, reading_user)
+            |> Conn.assign(:skip_link_headers, true)
+
+          Pleroma.Web.MastodonAPI.AccountController.statuses(conn, %{"id" => user.id})
+        end
+      },
+      inputs: %{"user" => user, "no user" => nil},
+      time: 60
+    )
+  end
+end
diff --git a/benchmarks/mix/tasks/pleroma/load_testing.ex b/benchmarks/mix/tasks/pleroma/load_testing.ex
index 0a751adac..388883240 100644
--- a/benchmarks/mix/tasks/pleroma/load_testing.ex
+++ b/benchmarks/mix/tasks/pleroma/load_testing.ex
@@ -1,114 +1,56 @@
 defmodule Mix.Tasks.Pleroma.LoadTesting do
   use Mix.Task
-  use Pleroma.LoadTesting.Helper
-  import Mix.Pleroma
-  import Pleroma.LoadTesting.Generator
-  import Pleroma.LoadTesting.Fetcher
+  import Ecto.Query
+  import Pleroma.LoadTesting.Helper, only: [clean_tables: 0]
+
+  alias Pleroma.Repo
+  alias Pleroma.User
 
   @shortdoc "Factory for generation data"
   @moduledoc """
   Generates data like:
   - local/remote users
-  - local/remote activities with notifications
-  - direct messages
-  - long thread
-  - non visible posts
+  - local/remote activities with differrent visibility:
+    - simple activiities
+    - with emoji
+    - with mentions
+    - hellthreads
+    - with attachments
+    - with tags
+    - likes
+    - reblogs
+    - simple threads
+    - long threads
 
   ## Generate data
-      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
-      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
+      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --friends 1000 --iterations 170 --friends_used 20 --non_friends_used 20
+      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -f 1000 -i 170 -fu 20 -nfu 20
 
   Options:
   - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
-  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
-  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
+  - `--friends NUMBER` - number of friends for main user. Defaults to: 1000. Alias: `-f`
+  - `--iterations NUMBER` - number of iterations to generate activities. For each iteration in database is inserted about 120+ activities with different visibility, actors and types.Defaults to: 170. Alias: `-i`
+  - `--friends_used NUMBER` - number of main user friends used in activity generation. Defaults to: 20. Alias: `-fu`
+  - `--non_friends_used NUMBER` - number of non friends used in activity generation. Defaults to: 20. Alias: `-nfu`
   """
 
-  @aliases [u: :users, d: :dms, t: :thread_length]
+  @aliases [u: :users, f: :friends, i: :iterations, fu: :friends_used, nfu: :non_friends_used]
   @switches [
     users: :integer,
-    dms: :integer,
-    thread_length: :integer
+    friends: :integer,
+    iterations: :integer,
+    friends_used: :integer,
+    non_friends_used: :integer
   ]
-  @users_default 20_000
-  @dms_default 1_000
-  @thread_length_default 2_000
 
   def run(args) do
-    start_pleroma()
-    Pleroma.Config.put([:instance, :skip_thread_containment], true)
-    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
-
-    users_max = Keyword.get(opts, :users, @users_default)
-    dms_max = Keyword.get(opts, :dms, @dms_default)
-    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
-
+    Logger.configure(level: :error)
+    Mix.Pleroma.start_pleroma()
     clean_tables()
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
-    opts =
-      Keyword.put(opts, :users_max, users_max)
-      |> Keyword.put(:dms_max, dms_max)
-      |> Keyword.put(:thread_length, thread_length)
-
-    generate_users(opts)
-
-    # main user for queries
-    IO.puts("Fetching local main user...")
-
-    {time, user} =
-      :timer.tc(fn ->
-        Repo.one(
-          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
-        )
-      end)
-
-    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
-
-    IO.puts("Fetching local users...")
-
-    {time, users} =
-      :timer.tc(fn ->
-        Repo.all(
-          from(u in User,
-            where: u.id != ^user.id,
-            where: u.local == true,
-            order_by: fragment("RANDOM()"),
-            limit: 10
-          )
-        )
-      end)
-
-    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
-
-    IO.puts("Fetching remote users...")
-
-    {time, remote_users} =
-      :timer.tc(fn ->
-        Repo.all(
-          from(u in User,
-            where: u.id != ^user.id,
-            where: u.local == false,
-            order_by: fragment("RANDOM()"),
-            limit: 10
-          )
-        )
-      end)
-
-    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
-
-    generate_activities(user, users)
-
-    generate_remote_activities(user, remote_users)
-
-    generate_like_activities(
-      user, Pleroma.Repo.all(Pleroma.Activity.Queries.by_type("Create"))
-    )
-
-    generate_dms(user, users, opts)
-
-    {:ok, activity} = generate_long_thread(user, users, opts)
-
-    generate_non_visible_message(user, users)
+    user = Pleroma.LoadTesting.Users.generate(opts)
+    Pleroma.LoadTesting.Activities.generate(user, opts)
 
     IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
 
@@ -120,19 +62,6 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
       "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
     )
 
-    fetch_user(user)
-    query_timelines(user)
-    query_notifications(user)
-    query_dms(user)
-    query_long_thread(user, activity)
-    Pleroma.Config.put([:instance, :skip_thread_containment], false)
-    query_timelines(user)
-  end
-
-  defp clean_tables do
-    IO.puts("Deleting old data...\n")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+    Pleroma.LoadTesting.Fetcher.run_benchmarks(user)
   end
 end