From db690bede92e9ba65b1afaa8605aa5ddb3c992a4 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Fri, 2 Aug 2019 21:33:12 +0300
Subject: temp commit

---
 lib/load_testing/fetcher.ex           | 116 ++++++++++++++++++++++++++++++++++
 lib/load_testing/generator.ex         |  83 ++++++++++++++++++++++++
 lib/load_testing/helper.ex            |  16 +++++
 lib/mix/tasks/pleroma/load_testing.ex | 100 +++++++++++++++++++++++++++++
 4 files changed, 315 insertions(+)
 create mode 100644 lib/load_testing/fetcher.ex
 create mode 100644 lib/load_testing/generator.ex
 create mode 100644 lib/load_testing/helper.ex
 create mode 100644 lib/mix/tasks/pleroma/load_testing.ex

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
new file mode 100644
index 000000000..70c0fcd0c
--- /dev/null
+++ b/lib/load_testing/fetcher.ex
@@ -0,0 +1,116 @@
+defmodule Pleroma.LoadTesting.Fetcher do
+  use Pleroma.LoadTesting.Helper
+
+  def fetch_user(user) do
+    IO.puts("=================================")
+
+    {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end)
+
+    IO.puts("Query user by id: #{to_sec(time)} sec.")
+
+    {time, _value} =
+      :timer.tc(fn ->
+        Repo.get_by(User, ap_id: user.ap_id)
+      end)
+
+    IO.puts("Query user by ap_id: #{to_sec(time)} sec.")
+
+    {time, _value} =
+      :timer.tc(fn ->
+        Repo.get_by(User, email: user.email)
+      end)
+
+    IO.puts("Query user by email: #{to_sec(time)} sec.")
+
+    {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end)
+
+    IO.puts("Query user by nickname: #{to_sec(time)} sec.")
+  end
+
+  def query_timelines(user) do
+    IO.puts("\n=================================")
+
+    params = %{
+      "count" => 20,
+      "with_muted" => true,
+      "type" => ["Create", "Announce"],
+      "blocking_user" => user,
+      "muting_user" => user,
+      "user" => user
+    }
+
+    {time, _} =
+      :timer.tc(fn ->
+        ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params)
+      end)
+
+    IO.puts("Query user home timeline: #{to_sec(time)} sec.")
+
+    params = %{
+      "count" => 20,
+      "local_only" => true,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    {time, _} =
+      :timer.tc(fn ->
+        ActivityPub.ActivityPub.fetch_public_activities(params)
+      end)
+
+    IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.")
+
+    params = %{
+      "count" => 20,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    {time, _} =
+      :timer.tc(fn ->
+        ActivityPub.ActivityPub.fetch_public_activities(params)
+      end)
+
+    IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.")
+  end
+
+  def query_notifications(user) do
+    IO.puts("\n=================================")
+    params = %{"count" => "20", "with_muted" => "false"}
+
+    {time, _} =
+      :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end)
+
+    IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.")
+
+    params = %{"count" => "20", "with_muted" => "true"}
+
+    {time, _} =
+      :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end)
+
+    IO.puts("Query user notifications with muted: #{to_sec(time)} sec.")
+  end
+
+  def query_long_thread(user, activity) do
+    IO.puts("\n=================================")
+
+    {time, replies} =
+      :timer.tc(fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+          activity.data["context"],
+          %{
+            "blocking_user" => user,
+            "user" => user
+          }
+        )
+      end)
+
+    IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.")
+  end
+end
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
new file mode 100644
index 000000000..a9016b9e8
--- /dev/null
+++ b/lib/load_testing/generator.ex
@@ -0,0 +1,83 @@
+defmodule Pleroma.LoadTesting.Generator do
+  use Pleroma.LoadTesting.Helper
+
+  def generate_users(opts) do
+    IO.puts("Starting generating #{opts[:users_max]} users...")
+    {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
+    IO.puts("Inserting users take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_users(opts) do
+    min = Keyword.get(opts, :users_min, 1)
+    max = Keyword.get(opts, :users_max)
+
+    query =
+      "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\",
+      \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n"
+
+    users =
+      Task.async_stream(
+        min..max,
+        &generate_user_data(&1),
+        max_concurrency: 10,
+        timeout: 30_000
+      )
+      |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end)
+
+    query = query <> String.replace_trailing(users, ", \n", ";")
+
+    Ecto.Adapters.SQL.query!(Repo, query)
+  end
+
+  defp generate_user_data(i) do
+    user = %User{
+      name: "Test テスト User #{i}",
+      email: "user#{i}@example.com",
+      nickname: "nick#{i}",
+      password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
+      bio: "Tester Number #{i}",
+      info: %{}
+    }
+
+    user = %{
+      user
+      | ap_id: User.ap_id(user),
+        follower_address: User.ap_followers(user),
+        following_address: User.ap_following(user),
+        following: [User.ap_id(user)]
+    }
+
+    "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{
+      user.following
+    }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{
+      user.name
+    }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())"
+  end
+
+  def generate_activities(users, opts) do
+    IO.puts("Starting generating #{opts[:activities_max]} activities...")
+    {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end)
+    IO.puts("Inserting activities take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_activities(users, opts) do
+    Task.async_stream(
+      1..opts[:activities_max],
+      fn _ ->
+        do_generate_activity(users, opts)
+      end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp do_generate_activity(users, opts) do
+    status =
+      if opts[:mention],
+        do: "some status with @#{opts[:mention].nickname}",
+        else: "some status"
+
+    Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status})
+  end
+end
diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex
new file mode 100644
index 000000000..338dba323
--- /dev/null
+++ b/lib/load_testing/helper.ex
@@ -0,0 +1,16 @@
+defmodule Pleroma.LoadTesting.Helper do
+  defmacro __using__(_) do
+    quote do
+      import Ecto.Query
+      alias Pleroma.Activity
+      alias Pleroma.Notification
+      alias Pleroma.Object
+      alias Pleroma.Repo
+      alias Pleroma.User
+      alias Pleroma.Web.ActivityPub
+      alias Pleroma.Web.CommonAPI
+
+      defp to_sec(microseconds), do: microseconds / 1_000_000
+    end
+  end
+end
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
new file mode 100644
index 000000000..9ed30db7e
--- /dev/null
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -0,0 +1,100 @@
+defmodule Mix.Tasks.Pleroma.LoadTesting do
+  use Mix.Task
+  use Pleroma.LoadTesting.Helper
+  import Mix.Pleroma
+  import Pleroma.LoadTesting.Generator
+  import Pleroma.LoadTesting.Fetcher
+
+  # tODO: remove autovacuum worker until generation is not ended
+  @shortdoc "Factory for generation data"
+  @moduledoc """
+  Generates data like:
+  - users
+  - activities with notifications
+
+  ## Generate data
+      MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000
+      MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000
+
+  Options:
+  - `--users NUMBER` - number of users to generate (default: 10000)
+  - `--activities NUMBER` - number of activities to generate (default: 20000)
+  """
+
+  @aliases [u: :users, a: :activities, d: :delete]
+  @switches [users: :integer, activities: :integer, delete: :boolean]
+  @users_default 20_000
+  @activities_default 50_000
+
+  def run(args) do
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
+    start_pleroma()
+
+    current_max = Keyword.get(opts, :users, @users_default)
+    activities_max = Keyword.get(opts, :activities, @activities_default)
+
+    {users_min, users_max} =
+      if opts[:delete] do
+        clean_tables()
+        {1, current_max}
+      else
+        current_count = Repo.aggregate(from(u in User), :count, :id) + 1
+        {current_count, current_max + current_count}
+      end
+
+    opts =
+      Keyword.put(opts, :users_min, users_min)
+      |> Keyword.put(:users_max, users_max)
+      |> Keyword.put(:activities_max, activities_max)
+
+    generate_users(opts)
+
+    # main user for queries
+    IO.puts("Fetching main user...")
+
+    {time, user} =
+      :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end)
+
+    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching users...")
+
+    {time, users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching users take #{to_sec(time)} sec.\n")
+
+    generate_activities(users, opts)
+
+    generate_activities(users, Keyword.put(opts, :mention, user))
+
+    # generate_replies(user, users, activities)
+
+    # activity = Enum.random(activities)
+    # generate_long_thread(user, users, activity)
+
+    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
+    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}")
+    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}")
+    IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}")
+
+    query_timelines(user)
+    query_notifications(user)
+    # query_long_thread(user, activity)
+  end
+
+  defp clean_tables do
+    IO.puts("\n\nDeleting old data...\n")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+  end
+end
-- 
cgit v1.2.3


From 79dde5804427ab24c5bc9aa1b5a44d2259cecc6d Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Wed, 4 Sep 2019 20:18:11 +0300
Subject: one more temp commit

---
 lib/load_testing/fetcher.ex                  | 122 ++++++++++++-------------
 lib/load_testing/generator.ex                | 127 ++++++++++++++++++++++-----
 lib/mix/tasks/pleroma/load_testing.ex        |  46 ++++++----
 lib/pleroma/web/activity_pub/activity_pub.ex |   5 ++
 4 files changed, 203 insertions(+), 97 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
index 70c0fcd0c..ed744db9b 100644
--- a/lib/load_testing/fetcher.ex
+++ b/lib/load_testing/fetcher.ex
@@ -4,33 +4,18 @@ defmodule Pleroma.LoadTesting.Fetcher do
   def fetch_user(user) do
     IO.puts("=================================")
 
-    {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end)
-
-    IO.puts("Query user by id: #{to_sec(time)} sec.")
-
-    {time, _value} =
-      :timer.tc(fn ->
-        Repo.get_by(User, ap_id: user.ap_id)
-      end)
-
-    IO.puts("Query user by ap_id: #{to_sec(time)} sec.")
-
-    {time, _value} =
-      :timer.tc(fn ->
-        Repo.get_by(User, email: user.email)
-      end)
-
-    IO.puts("Query user by email: #{to_sec(time)} sec.")
-
-    {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end)
-
-    IO.puts("Query user by nickname: #{to_sec(time)} sec.")
+    Benchee.run(%{
+      "By id" => fn -> Repo.get_by(User, id: user.id) end,
+      "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
+      "By email" => fn -> Repo.get_by(User, email: user.email) end,
+      "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
+    })
   end
 
   def query_timelines(user) do
     IO.puts("\n=================================")
 
-    params = %{
+    home_timeline_params = %{
       "count" => 20,
       "with_muted" => true,
       "type" => ["Create", "Announce"],
@@ -39,14 +24,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
       "user" => user
     }
 
-    {time, _} =
-      :timer.tc(fn ->
-        ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params)
-      end)
-
-    IO.puts("Query user home timeline: #{to_sec(time)} sec.")
-
-    params = %{
+    mastodon_public_timeline_params = %{
       "count" => 20,
       "local_only" => true,
       "only_media" => "false",
@@ -56,14 +34,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
       "muting_user" => user
     }
 
-    {time, _} =
-      :timer.tc(fn ->
-        ActivityPub.ActivityPub.fetch_public_activities(params)
-      end)
-
-    IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.")
-
-    params = %{
+    mastodon_federated_timeline_params = %{
       "count" => 20,
       "only_media" => "false",
       "type" => ["Create", "Announce"],
@@ -72,45 +43,76 @@ defmodule Pleroma.LoadTesting.Fetcher do
       "muting_user" => user
     }
 
-    {time, _} =
-      :timer.tc(fn ->
-        ActivityPub.ActivityPub.fetch_public_activities(params)
-      end)
-
-    IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.")
+    Benchee.run(%{
+      "User home timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+          [user.ap_id | user.following],
+          home_timeline_params
+        )
+      end,
+      "User mastodon public timeline" => fn ->
+        ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+      end,
+      "User mastodon federated public timeline" => fn ->
+        ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params)
+      end
+    })
   end
 
   def query_notifications(user) do
     IO.puts("\n=================================")
-    params = %{"count" => "20", "with_muted" => "false"}
-
-    {time, _} =
-      :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end)
-
-    IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.")
+    without_muted_params = %{"count" => "20", "with_muted" => "false"}
+    with_muted_params = %{"count" => "20", "with_muted" => "true"}
+
+    Benchee.run(%{
+      "Notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+      end,
+      "Notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+      end
+    })
+  end
 
-    params = %{"count" => "20", "with_muted" => "true"}
+  def query_dms(user) do
+    IO.puts("\n=================================")
 
-    {time, _} =
-      :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end)
+    params = %{
+      "count" => "20",
+      "with_muted" => "true",
+      "type" => "Create",
+      "blocking_user" => user,
+      "user" => user,
+      visibility: "direct"
+    }
 
-    IO.puts("Query user notifications with muted: #{to_sec(time)} sec.")
+    Benchee.run(%{
+      "Direct messages with muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(params)
+      end,
+      "Direct messages without muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+      end
+    })
   end
 
   def query_long_thread(user, activity) do
     IO.puts("\n=================================")
 
-    {time, replies} =
-      :timer.tc(fn ->
+    Benchee.run(%{
+      "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end,
+      "Fetch context of main post" => fn ->
         Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
           activity.data["context"],
           %{
             "blocking_user" => user,
-            "user" => user
+            "user" => user,
+            "exclude_id" => activity.id
           }
         )
-      end)
-
-    IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.")
+      end
+    })
   end
 end
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index a9016b9e8..7f50ee68e 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -8,25 +8,15 @@ defmodule Pleroma.LoadTesting.Generator do
   end
 
   defp do_generate_users(opts) do
-    min = Keyword.get(opts, :users_min, 1)
     max = Keyword.get(opts, :users_max)
 
-    query =
-      "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\",
-      \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n"
-
-    users =
-      Task.async_stream(
-        min..max,
-        &generate_user_data(&1),
-        max_concurrency: 10,
-        timeout: 30_000
-      )
-      |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end)
-
-    query = query <> String.replace_trailing(users, ", \n", ";")
-
-    Ecto.Adapters.SQL.query!(Repo, query)
+    Task.async_stream(
+      1..max,
+      &generate_user_data(&1),
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Enum.to_list()
   end
 
   defp generate_user_data(i) do
@@ -47,11 +37,7 @@ defmodule Pleroma.LoadTesting.Generator do
         following: [User.ap_id(user)]
     }
 
-    "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{
-      user.following
-    }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{
-      user.name
-    }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())"
+    Pleroma.Repo.insert!(user)
   end
 
   def generate_activities(users, opts) do
@@ -80,4 +66,101 @@ defmodule Pleroma.LoadTesting.Generator do
 
     Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status})
   end
+
+  def generate_dms(user, users, opts) do
+    IO.puts("Starting generating #{opts[:dms_max]} DMs")
+    {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
+    IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_dms(user, users, opts) do
+    Task.async_stream(
+      1..opts[:dms_max],
+      fn _ ->
+        do_generate_dm(user, users)
+      end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp do_generate_dm(user, users) do
+    post = %{
+      "status" => "@#{user.nickname} some direct message",
+      "visibility" => "direct"
+    }
+
+    Pleroma.Web.CommonAPI.post(Enum.random(users), post)
+  end
+
+  def generate_long_thread(user, users, opts) do
+    IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies")
+    {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
+    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
+    {:ok, activity}
+  end
+
+  defp do_generate_long_thread(user, users, opts) do
+    {:ok, %{id: id} = activity} =
+      Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"})
+
+    Task.async_stream(
+      1..opts[:long_thread_length],
+      fn _ -> do_generate_thread(users, id) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+
+    activity
+  end
+
+  defp do_generate_thread(users, activity_id) do
+    Pleroma.Web.CommonAPI.post(Enum.random(users), %{
+      "status" => "reply to main post",
+      "in_reply_to_status_id" => activity_id
+    })
+  end
+
+  def generate_private_thread(users, opts) do
+    IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies")
+    {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end)
+    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_non_visible_posts(users, opts) do
+    [user1, user2] = Enum.take(users, 2)
+    {:ok, user1} = Pleroma.User.follow(user1, user2)
+    {:ok, user2} = Pleroma.User.follow(user2, user1)
+
+    {:ok, activity} =
+      Pleroma.Web.CommonAPI.post(user1, %{
+        "status" => "Some private post",
+        "visibility" => "private"
+      })
+
+    {:ok, activity_public} =
+      Pleroma.Web.CommonAPI.post(user2, %{
+        "status" => "Some public reply",
+        "in_reply_to_status_id" => activity.id
+      })
+
+    Task.async_stream(
+      1..opts[:non_visible_posts_max],
+      fn _ -> do_generate_non_visible_post(users, activity_public) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+  end
+
+  defp do_generate_non_visible_post(users, activity) do
+    visibility = Enum.random(["private", "public"])
+
+    Pleroma.Web.CommonAPI.post(Enum.random(users), %{
+      "visibility" => visibility,
+      "status" => "Some #{visibility} reply",
+      "in_reply_to_status_id" => activity.id
+    })
+  end
 end
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
index 9ed30db7e..d17cf0b3e 100644
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -21,31 +21,38 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
   - `--activities NUMBER` - number of activities to generate (default: 20000)
   """
 
-  @aliases [u: :users, a: :activities, d: :delete]
-  @switches [users: :integer, activities: :integer, delete: :boolean]
+  @aliases [u: :users, a: :activities]
+  @switches [
+    users: :integer,
+    activities: :integer,
+    dms: :integer,
+    thread_length: :integer,
+    non_visible_posts: :integer
+  ]
   @users_default 20_000
   @activities_default 50_000
+  @dms_default 50_000
+  @thread_length_default 2_000
+  @non_visible_posts_default 2_000
 
   def run(args) do
-    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
     start_pleroma()
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
-    current_max = Keyword.get(opts, :users, @users_default)
+    users_max = Keyword.get(opts, :users, @users_default)
     activities_max = Keyword.get(opts, :activities, @activities_default)
+    dms_max = Keyword.get(opts, :dms, @dms_default)
+    long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
+    non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default)
 
-    {users_min, users_max} =
-      if opts[:delete] do
-        clean_tables()
-        {1, current_max}
-      else
-        current_count = Repo.aggregate(from(u in User), :count, :id) + 1
-        {current_count, current_max + current_count}
-      end
+    clean_tables()
 
     opts =
-      Keyword.put(opts, :users_min, users_min)
-      |> Keyword.put(:users_max, users_max)
+      Keyword.put(opts, :users_max, users_max)
       |> Keyword.put(:activities_max, activities_max)
+      |> Keyword.put(:dms_max, dms_max)
+      |> Keyword.put(:long_thread_length, long_thread_length)
+      |> Keyword.put(:non_visible_posts_max, non_visible_posts)
 
     generate_users(opts)
 
@@ -76,6 +83,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
 
     generate_activities(users, Keyword.put(opts, :mention, user))
 
+    generate_dms(user, users, opts)
+
+    {:ok, activity} = generate_long_thread(user, users, opts)
+
+    generate_private_thread(users, opts)
+
     # generate_replies(user, users, activities)
 
     # activity = Enum.random(activities)
@@ -86,9 +99,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
     IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}")
     IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}")
 
+    fetch_user(user)
     query_timelines(user)
     query_notifications(user)
-    # query_long_thread(user, activity)
+    query_dms(user)
+    query_long_thread(user, activity)
+    query_timelines(user)
   end
 
   defp clean_tables do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index eeb826814..f2b322314 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -249,6 +249,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     # only accept false as false value
     local = !(params[:local] == false)
     published = params[:published]
+    quick_insert? = Pleroma.Config.get([:env]) == :benchmark
 
     with create_data <-
            make_create_data(
@@ -259,12 +260,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
          _ <- increase_poll_votes_if_vote(create_data),
+         {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     else
+      {:quick_insert, true, activity} ->
+        {:ok, activity}
+
       {:fake, true, activity} ->
         {:ok, activity}
 
-- 
cgit v1.2.3


From a1125bd5644a5819d1cfbbc8f6b3bb2dadbbc38a Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 5 Sep 2019 16:01:52 +0300
Subject: generatoin and fetching

---
 lib/load_testing/fetcher.ex           | 126 ++++++++++++++++++++-
 lib/load_testing/generator.ex         | 203 ++++++++++++++++++++++++----------
 lib/load_testing/helper.ex            |   5 -
 lib/mix/tasks/pleroma/load_testing.ex |  47 ++++----
 4 files changed, 289 insertions(+), 92 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
index ed744db9b..825f921e6 100644
--- a/lib/load_testing/fetcher.ex
+++ b/lib/load_testing/fetcher.ex
@@ -51,10 +51,52 @@ defmodule Pleroma.LoadTesting.Fetcher do
         )
       end,
       "User mastodon public timeline" => fn ->
-        ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_public_timeline_params
+        )
       end,
       "User mastodon federated public timeline" => fn ->
-        ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params)
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_federated_timeline_params
+        )
+      end
+    })
+
+    home_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+        [user.ap_id | user.following],
+        home_timeline_params
+      )
+
+    public_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+
+    public_federated_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+        mastodon_federated_timeline_params
+      )
+
+    Benchee.run(%{
+      "Rendering home timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: home_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public federated timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_federated_activities,
+          for: user,
+          as: :activity
+        })
       end
     })
   end
@@ -72,6 +114,27 @@ defmodule Pleroma.LoadTesting.Fetcher do
         Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
       end
     })
+
+    without_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+
+    with_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+
+    Benchee.run(%{
+      "Render notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: without_muted_notifications,
+          for: user
+        })
+      end,
+      "Render notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: with_muted_notifications,
+          for: user
+        })
+      end
+    })
   end
 
   def query_dms(user) do
@@ -96,13 +159,40 @@ defmodule Pleroma.LoadTesting.Fetcher do
         |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
       end
     })
+
+    dms_with_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(params)
+
+    dms_without_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+
+    Benchee.run(%{
+      "Rendering dms with muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_with_muted,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering dms without muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_without_muted,
+          for: user,
+          as: :activity
+        })
+      end
+    })
   end
 
   def query_long_thread(user, activity) do
     IO.puts("\n=================================")
 
     Benchee.run(%{
-      "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end,
+      "Fetch main post" => fn ->
+        Pleroma.Activity.get_by_id_with_object(activity.id)
+      end,
       "Fetch context of main post" => fn ->
         Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
           activity.data["context"],
@@ -114,5 +204,35 @@ defmodule Pleroma.LoadTesting.Fetcher do
         )
       end
     })
+
+    activity = Pleroma.Activity.get_by_id_with_object(activity.id)
+
+    context =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+        activity.data["context"],
+        %{
+          "blocking_user" => user,
+          "user" => user,
+          "exclude_id" => activity.id
+        }
+      )
+
+    Benchee.run(%{
+      "Render status" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{
+          activity: activity,
+          for: user
+        })
+      end,
+      "Render context ancestors" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render(
+          "index.json",
+          for: user,
+          activities: context,
+          as: :activity
+        )
+        |> Enum.reverse()
+      end
+    })
   end
 end
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index 7f50ee68e..6df518aba 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -1,9 +1,11 @@
 defmodule Pleroma.LoadTesting.Generator do
   use Pleroma.LoadTesting.Helper
+  alias Pleroma.Web.CommonAPI
 
   def generate_users(opts) do
     IO.puts("Starting generating #{opts[:users_max]} users...")
     {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
+
     IO.puts("Inserting users take #{to_sec(time)} sec.\n")
   end
 
@@ -37,34 +39,111 @@ defmodule Pleroma.LoadTesting.Generator do
         following: [User.ap_id(user)]
     }
 
-    Pleroma.Repo.insert!(user)
+    Repo.insert!(user)
   end
 
-  def generate_activities(users, opts) do
-    IO.puts("Starting generating #{opts[:activities_max]} activities...")
-    {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end)
-    IO.puts("Inserting activities take #{to_sec(time)} sec.\n")
+  def generate_activities(user, users) do
+    do_generate_activities(user, users)
   end
 
-  defp do_generate_activities(users, opts) do
-    Task.async_stream(
-      1..opts[:activities_max],
-      fn _ ->
-        do_generate_activity(users, opts)
-      end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
+  defp do_generate_activities(user, users) do
+    IO.puts("Starting generating 20000 common activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 20000 activities with mentions...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity_with_mention(user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 10000 activities with threads...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn _ ->
+            do_generate_threads([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_activity(users) do
+    post = %{
+      "status" => "Some status without mention with random user"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  defp do_generate_activity_with_mention(user, users) do
+    mentions_cnt = Enum.random([2, 3, 4, 5])
+    with_user = Enum.random([true, false])
+    users = Enum.shuffle(users)
+    mentions_users = Enum.take(users, mentions_cnt)
+    mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
+
+    mentions_str =
+      Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
+
+    post = %{
+      "status" => mentions_str <> "some status with mentions random users"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
   end
 
-  defp do_generate_activity(users, opts) do
-    status =
-      if opts[:mention],
-        do: "some status with @#{opts[:mention].nickname}",
-        else: "some status"
+  defp do_generate_threads(users) do
+    thread_length = Enum.random([2, 3, 4, 5])
+    actor = Enum.random(users)
+
+    post = %{
+      "status" => "Start of the thread"
+    }
+
+    {:ok, activity} = CommonAPI.post(actor, post)
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{actor.nickname} reply to thread",
+        "in_reply_to_status_id" => activity.id
+      }
 
-    Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status})
+      CommonAPI.post(user, post)
+    end)
   end
 
   def generate_dms(user, users, opts) do
@@ -91,22 +170,21 @@ defmodule Pleroma.LoadTesting.Generator do
       "visibility" => "direct"
     }
 
-    Pleroma.Web.CommonAPI.post(Enum.random(users), post)
+    CommonAPI.post(Enum.random(users), post)
   end
 
   def generate_long_thread(user, users, opts) do
-    IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies")
+    IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
     {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
     IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
     {:ok, activity}
   end
 
   defp do_generate_long_thread(user, users, opts) do
-    {:ok, %{id: id} = activity} =
-      Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"})
+    {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
 
     Task.async_stream(
-      1..opts[:long_thread_length],
+      1..opts[:thread_length],
       fn _ -> do_generate_thread(users, id) end,
       max_concurrency: 10,
       timeout: 30_000
@@ -117,50 +195,63 @@ defmodule Pleroma.LoadTesting.Generator do
   end
 
   defp do_generate_thread(users, activity_id) do
-    Pleroma.Web.CommonAPI.post(Enum.random(users), %{
+    CommonAPI.post(Enum.random(users), %{
       "status" => "reply to main post",
       "in_reply_to_status_id" => activity_id
     })
   end
 
-  def generate_private_thread(users, opts) do
-    IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies")
-    {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end)
-    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
-  end
+  def generate_non_visible_message(user, users) do
+    IO.puts("Starting generating 1000 non visible posts")
 
-  defp do_generate_non_visible_posts(users, opts) do
-    [user1, user2] = Enum.take(users, 2)
-    {:ok, user1} = Pleroma.User.follow(user1, user2)
-    {:ok, user2} = Pleroma.User.follow(user2, user1)
+    {time, _} =
+      :timer.tc(fn ->
+        do_generate_non_visible_posts(user, users)
+      end)
 
-    {:ok, activity} =
-      Pleroma.Web.CommonAPI.post(user1, %{
-        "status" => "Some private post",
-        "visibility" => "private"
-      })
+    IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
+  end
 
-    {:ok, activity_public} =
-      Pleroma.Web.CommonAPI.post(user2, %{
-        "status" => "Some public reply",
-        "in_reply_to_status_id" => activity.id
-      })
+  defp do_generate_non_visible_posts(user, users) do
+    [not_friend | users] = users
 
-    Task.async_stream(
-      1..opts[:non_visible_posts_max],
-      fn _ -> do_generate_non_visible_post(users, activity_public) end,
+    make_friends(user, users)
+
+    Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
       max_concurrency: 10,
       timeout: 30_000
     )
+    |> Stream.run()
   end
 
-  defp do_generate_non_visible_post(users, activity) do
-    visibility = Enum.random(["private", "public"])
+  defp make_friends(_user, []), do: nil
 
-    Pleroma.Web.CommonAPI.post(Enum.random(users), %{
-      "visibility" => visibility,
-      "status" => "Some #{visibility} reply",
-      "in_reply_to_status_id" => activity.id
-    })
+  defp make_friends(user, [friend | users]) do
+    {:ok, _} = User.follow(user, friend)
+    {:ok, _} = User.follow(friend, user)
+    make_friends(user, users)
+  end
+
+  defp do_generate_non_visible_post(not_friend, users) do
+    post = %{
+      "status" => "some non visible post",
+      "visibility" => "private"
+    }
+
+    {:ok, activity} = CommonAPI.post(not_friend, post)
+
+    thread_length = Enum.random([2, 3, 4, 5])
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{not_friend.nickname} reply to non visible post",
+        "in_reply_to_status_id" => activity.id,
+        "visibility" => "private"
+      }
+
+      CommonAPI.post(user, post)
+    end)
   end
 end
diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex
index 338dba323..47b25c65f 100644
--- a/lib/load_testing/helper.ex
+++ b/lib/load_testing/helper.ex
@@ -2,13 +2,8 @@ defmodule Pleroma.LoadTesting.Helper do
   defmacro __using__(_) do
     quote do
       import Ecto.Query
-      alias Pleroma.Activity
-      alias Pleroma.Notification
-      alias Pleroma.Object
       alias Pleroma.Repo
       alias Pleroma.User
-      alias Pleroma.Web.ActivityPub
-      alias Pleroma.Web.CommonAPI
 
       defp to_sec(microseconds), do: microseconds / 1_000_000
     end
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
index d17cf0b3e..83e531abf 100644
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -13,46 +13,37 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
   - activities with notifications
 
   ## Generate data
-      MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000
-      MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000
+      MIX_ENV=benchmark mix pleroma.load_testing --users 10000
+      MIX_ENV=benchmark mix pleroma.load_testing -u 10000
 
   Options:
   - `--users NUMBER` - number of users to generate (default: 10000)
-  - `--activities NUMBER` - number of activities to generate (default: 20000)
   """
 
-  @aliases [u: :users, a: :activities]
+  @aliases [u: :users, d: :dms, t: :thread_length]
   @switches [
     users: :integer,
-    activities: :integer,
     dms: :integer,
-    thread_length: :integer,
-    non_visible_posts: :integer
+    thread_length: :integer
   ]
   @users_default 20_000
-  @activities_default 50_000
-  @dms_default 50_000
+  @dms_default 20_000
   @thread_length_default 2_000
-  @non_visible_posts_default 2_000
 
   def run(args) do
     start_pleroma()
     {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
     users_max = Keyword.get(opts, :users, @users_default)
-    activities_max = Keyword.get(opts, :activities, @activities_default)
     dms_max = Keyword.get(opts, :dms, @dms_default)
-    long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
-    non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default)
+    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
 
     clean_tables()
 
     opts =
       Keyword.put(opts, :users_max, users_max)
-      |> Keyword.put(:activities_max, activities_max)
       |> Keyword.put(:dms_max, dms_max)
-      |> Keyword.put(:long_thread_length, long_thread_length)
-      |> Keyword.put(:non_visible_posts_max, non_visible_posts)
+      |> Keyword.put(:thread_length, thread_length)
 
     generate_users(opts)
 
@@ -60,7 +51,9 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
     IO.puts("Fetching main user...")
 
     {time, user} =
-      :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end)
+      :timer.tc(fn ->
+        Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1))
+      end)
 
     IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
 
@@ -79,25 +72,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
 
     IO.puts("Fetching users take #{to_sec(time)} sec.\n")
 
-    generate_activities(users, opts)
-
-    generate_activities(users, Keyword.put(opts, :mention, user))
+    generate_activities(user, users)
 
     generate_dms(user, users, opts)
 
     {:ok, activity} = generate_long_thread(user, users, opts)
 
-    generate_private_thread(users, opts)
+    generate_non_visible_message(user, users)
 
-    # generate_replies(user, users, activities)
+    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
 
-    # activity = Enum.random(activities)
-    # generate_long_thread(user, users, activity)
+    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}")
 
-    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
-    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}")
-    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}")
-    IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}")
+    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}")
+
+    IO.puts(
+      "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
+    )
 
     fetch_user(user)
     query_timelines(user)
-- 
cgit v1.2.3


From 252e5db45c5d9d63116e5a24cc35c26f91ec7de4 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Fri, 6 Sep 2019 16:37:18 +0300
Subject: docs fixes

---
 lib/load_testing/fetcher.ex           |  9 ---------
 lib/load_testing/generator.ex         |  3 ++-
 lib/mix/tasks/pleroma/load_testing.ex | 16 +++++++++++-----
 3 files changed, 13 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
index 825f921e6..b92b6e04f 100644
--- a/lib/load_testing/fetcher.ex
+++ b/lib/load_testing/fetcher.ex
@@ -2,8 +2,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   use Pleroma.LoadTesting.Helper
 
   def fetch_user(user) do
-    IO.puts("=================================")
-
     Benchee.run(%{
       "By id" => fn -> Repo.get_by(User, id: user.id) end,
       "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
@@ -13,8 +11,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   def query_timelines(user) do
-    IO.puts("\n=================================")
-
     home_timeline_params = %{
       "count" => 20,
       "with_muted" => true,
@@ -102,7 +98,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   def query_notifications(user) do
-    IO.puts("\n=================================")
     without_muted_params = %{"count" => "20", "with_muted" => "false"}
     with_muted_params = %{"count" => "20", "with_muted" => "true"}
 
@@ -138,8 +133,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   def query_dms(user) do
-    IO.puts("\n=================================")
-
     params = %{
       "count" => "20",
       "with_muted" => "true",
@@ -187,8 +180,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   def query_long_thread(user, activity) do
-    IO.puts("\n=================================")
-
     Benchee.run(%{
       "Fetch main post" => fn ->
         Pleroma.Activity.get_by_id_with_object(activity.id)
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index 6df518aba..7ad68dcc1 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -28,7 +28,8 @@ defmodule Pleroma.LoadTesting.Generator do
       nickname: "nick#{i}",
       password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
       bio: "Tester Number #{i}",
-      info: %{}
+      info: %{},
+      local: Enum.random([true, false])
     }
 
     user = %{
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
index 83e531abf..4fed6bf74 100644
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -5,19 +5,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
   import Pleroma.LoadTesting.Generator
   import Pleroma.LoadTesting.Fetcher
 
-  # tODO: remove autovacuum worker until generation is not ended
   @shortdoc "Factory for generation data"
   @moduledoc """
   Generates data like:
   - users
   - activities with notifications
+  - direct messages
+  - long thread
+  - non visible posts
 
   ## Generate data
-      MIX_ENV=benchmark mix pleroma.load_testing --users 10000
-      MIX_ENV=benchmark mix pleroma.load_testing -u 10000
+      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
+      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
 
   Options:
-  - `--users NUMBER` - number of users to generate (default: 10000)
+  - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
+  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
+  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
   """
 
   @aliases [u: :users, d: :dms, t: :thread_length]
@@ -32,6 +36,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
 
   def run(args) do
     start_pleroma()
+    Pleroma.Config.put([:instance, :skip_thread_containment], true)
     {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
     users_max = Keyword.get(opts, :users, @users_default)
@@ -95,11 +100,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
     query_notifications(user)
     query_dms(user)
     query_long_thread(user, activity)
+    Pleroma.Config.put([:instance, :skip_thread_containment], false)
     query_timelines(user)
   end
 
   defp clean_tables do
-    IO.puts("\n\nDeleting old data...\n")
+    IO.puts("Deleting old data...\n")
     Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
     Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
     Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
-- 
cgit v1.2.3


From b3f6f6a4091c048813e367f6a9414eba6fce5109 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Fri, 6 Sep 2019 20:14:02 +0300
Subject: generating remote users

---
 lib/load_testing/fetcher.ex   |  2 +-
 lib/load_testing/generator.ex | 34 ++++++++++++++++++++++++++--------
 lib/pleroma/user.ex           |  4 +++-
 3 files changed, 30 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
index b92b6e04f..0ff2f28c0 100644
--- a/lib/load_testing/fetcher.ex
+++ b/lib/load_testing/fetcher.ex
@@ -215,7 +215,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
           for: user
         })
       end,
-      "Render context ancestors" => fn ->
+      "Render context" => fn ->
         Pleroma.Web.MastodonAPI.StatusView.render(
           "index.json",
           for: user,
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index 7ad68dcc1..61e3d8686 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -22,6 +22,8 @@ defmodule Pleroma.LoadTesting.Generator do
   end
 
   defp generate_user_data(i) do
+    remote = Enum.random([true, false])
+
     user = %User{
       name: "Test テスト User #{i}",
       email: "user#{i}@example.com",
@@ -29,16 +31,32 @@ defmodule Pleroma.LoadTesting.Generator do
       password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
       bio: "Tester Number #{i}",
       info: %{},
-      local: Enum.random([true, false])
+      local: remote
     }
 
-    user = %{
-      user
-      | ap_id: User.ap_id(user),
-        follower_address: User.ap_followers(user),
-        following_address: User.ap_following(user),
-        following: [User.ap_id(user)]
-    }
+    user_urls =
+      if remote do
+        base_url =
+          Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
+
+        ap_id = "#{base_url}/users/#{user.nickname}"
+
+        %{
+          ap_id: ap_id,
+          follower_address: ap_id <> "/followers",
+          following_address: ap_id <> "/following",
+          following: [ap_id]
+        }
+      else
+        %{
+          ap_id: User.ap_id(user),
+          follower_address: User.ap_followers(user),
+          following_address: User.ap_following(user),
+          following: [User.ap_id(user)]
+        }
+      end
+
+    user = Map.merge(user, user_urls)
 
     Repo.insert!(user)
   end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 3aa245f2a..37e59a651 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -429,7 +429,9 @@ defmodule Pleroma.User do
         {:error, "Could not follow user: #{followed.nickname} blocked you."}
 
       true ->
-        if !followed.local && follower.local && !ap_enabled?(followed) do
+        benchmark? = Pleroma.Config.get([:env]) == :benchmark
+
+        if !followed.local && follower.local && !ap_enabled?(followed) && !benchmark? do
           Websub.subscribe(follower, followed)
         end
 
-- 
cgit v1.2.3


From 924d7e6aa60095e27c15b4a5f473a192ab2b42ef Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 19 Sep 2019 12:59:36 +0300
Subject: generating remote activities

---
 lib/load_testing/generator.ex         | 78 ++++++++++++++++++++++++++++++++++-
 lib/mix/tasks/pleroma/load_testing.ex | 29 +++++++++++--
 2 files changed, 102 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index 61e3d8686..5c5a5c122 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -28,7 +28,8 @@ defmodule Pleroma.LoadTesting.Generator do
       name: "Test テスト User #{i}",
       email: "user#{i}@example.com",
       nickname: "nick#{i}",
-      password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
+      password_hash:
+        "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
       bio: "Tester Number #{i}",
       info: %{},
       local: remote
@@ -165,6 +166,81 @@ defmodule Pleroma.LoadTesting.Generator do
     end)
   end
 
+  def generate_remote_activities(user, users) do
+    do_generate_remote_activities(user, users)
+  end
+
+  defp do_generate_remote_activities(user, users) do
+    IO.puts("Starting generating 10000 remote activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn i ->
+            do_generate_remote_activity(i, user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_remote_activity(i, user, users) do
+    actor = Enum.random(users)
+    %{host: host} = URI.parse(actor.ap_id)
+    date = Date.utc_today()
+    datetime = DateTime.utc_now()
+
+    map = %{
+      "actor" => actor.ap_id,
+      "cc" => [actor.follower_address, user.ap_id],
+      "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+      "id" => actor.ap_id <> "/statuses/#{i}/activity",
+      "object" => %{
+        "actor" => actor.ap_id,
+        "atomUri" => actor.ap_id <> "/statuses/#{i}",
+        "attachment" => [],
+        "attributedTo" => actor.ap_id,
+        "bcc" => [],
+        "bto" => [],
+        "cc" => [actor.follower_address, user.ap_id],
+        "content" =>
+          "<p><span class=\"h-card\"><a href=\"" <>
+            user.ap_id <>
+            "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
+        "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "conversation" =>
+          "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "emoji" => %{},
+        "id" => actor.ap_id <> "/statuses/#{i}",
+        "inReplyTo" => nil,
+        "inReplyToAtomUri" => nil,
+        "published" => datetime,
+        "sensitive" => true,
+        "summary" => "cw",
+        "tag" => [
+          %{
+            "href" => user.ap_id,
+            "name" => "@#{user.nickname}@#{host}",
+            "type" => "Mention"
+          }
+        ],
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Note",
+        "url" => "http://#{host}/@#{actor.nickname}/#{i}"
+      },
+      "published" => datetime,
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "type" => "Create"
+    }
+
+    Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
+  end
+
   def generate_dms(user, users, opts) do
     IO.puts("Starting generating #{opts[:dms_max]} DMs")
     {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
index 4fed6bf74..a7057395d 100644
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -53,32 +53,53 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
     generate_users(opts)
 
     # main user for queries
-    IO.puts("Fetching main user...")
+    IO.puts("Fetching local main user...")
 
     {time, user} =
       :timer.tc(fn ->
-        Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1))
+        Repo.one(
+          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
+        )
       end)
 
     IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
 
-    IO.puts("Fetching users...")
+    IO.puts("Fetching local users...")
 
     {time, users} =
       :timer.tc(fn ->
         Repo.all(
           from(u in User,
             where: u.id != ^user.id,
+            where: u.local == true,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching remote users...")
+
+    {time, remote_users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            where: u.local == false,
             order_by: fragment("RANDOM()"),
             limit: 10
           )
         )
       end)
 
-    IO.puts("Fetching users take #{to_sec(time)} sec.\n")
+    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
 
     generate_activities(user, users)
 
+    generate_remote_activities(user, remote_users)
+
     generate_dms(user, users, opts)
 
     {:ok, activity} = generate_long_thread(user, users, opts)
-- 
cgit v1.2.3


From 1d285e6fada841b6959b6516aca1c84d1a2c9b93 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 19 Sep 2019 14:02:27 +0300
Subject: moving to separate dir

---
 lib/load_testing/fetcher.ex           | 229 ----------------------
 lib/load_testing/generator.ex         | 352 ----------------------------------
 lib/load_testing/helper.ex            |  11 --
 lib/mix/tasks/pleroma/load_testing.ex | 134 -------------
 4 files changed, 726 deletions(-)
 delete mode 100644 lib/load_testing/fetcher.ex
 delete mode 100644 lib/load_testing/generator.ex
 delete mode 100644 lib/load_testing/helper.ex
 delete mode 100644 lib/mix/tasks/pleroma/load_testing.ex

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
deleted file mode 100644
index 0ff2f28c0..000000000
--- a/lib/load_testing/fetcher.ex
+++ /dev/null
@@ -1,229 +0,0 @@
-defmodule Pleroma.LoadTesting.Fetcher do
-  use Pleroma.LoadTesting.Helper
-
-  def fetch_user(user) do
-    Benchee.run(%{
-      "By id" => fn -> Repo.get_by(User, id: user.id) end,
-      "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
-      "By email" => fn -> Repo.get_by(User, email: user.email) end,
-      "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
-    })
-  end
-
-  def query_timelines(user) do
-    home_timeline_params = %{
-      "count" => 20,
-      "with_muted" => true,
-      "type" => ["Create", "Announce"],
-      "blocking_user" => user,
-      "muting_user" => user,
-      "user" => user
-    }
-
-    mastodon_public_timeline_params = %{
-      "count" => 20,
-      "local_only" => true,
-      "only_media" => "false",
-      "type" => ["Create", "Announce"],
-      "with_muted" => "true",
-      "blocking_user" => user,
-      "muting_user" => user
-    }
-
-    mastodon_federated_timeline_params = %{
-      "count" => 20,
-      "only_media" => "false",
-      "type" => ["Create", "Announce"],
-      "with_muted" => "true",
-      "blocking_user" => user,
-      "muting_user" => user
-    }
-
-    Benchee.run(%{
-      "User home timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
-          [user.ap_id | user.following],
-          home_timeline_params
-        )
-      end,
-      "User mastodon public timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-          mastodon_public_timeline_params
-        )
-      end,
-      "User mastodon federated public timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-          mastodon_federated_timeline_params
-        )
-      end
-    })
-
-    home_activities =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
-        [user.ap_id | user.following],
-        home_timeline_params
-      )
-
-    public_activities =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
-
-    public_federated_activities =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-        mastodon_federated_timeline_params
-      )
-
-    Benchee.run(%{
-      "Rendering home timeline" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: home_activities,
-          for: user,
-          as: :activity
-        })
-      end,
-      "Rendering public timeline" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: public_activities,
-          for: user,
-          as: :activity
-        })
-      end,
-      "Rendering public federated timeline" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: public_federated_activities,
-          for: user,
-          as: :activity
-        })
-      end
-    })
-  end
-
-  def query_notifications(user) do
-    without_muted_params = %{"count" => "20", "with_muted" => "false"}
-    with_muted_params = %{"count" => "20", "with_muted" => "true"}
-
-    Benchee.run(%{
-      "Notifications without muted" => fn ->
-        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
-      end,
-      "Notifications with muted" => fn ->
-        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
-      end
-    })
-
-    without_muted_notifications =
-      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
-
-    with_muted_notifications =
-      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
-
-    Benchee.run(%{
-      "Render notifications without muted" => fn ->
-        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
-          notifications: without_muted_notifications,
-          for: user
-        })
-      end,
-      "Render notifications with muted" => fn ->
-        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
-          notifications: with_muted_notifications,
-          for: user
-        })
-      end
-    })
-  end
-
-  def query_dms(user) do
-    params = %{
-      "count" => "20",
-      "with_muted" => "true",
-      "type" => "Create",
-      "blocking_user" => user,
-      "user" => user,
-      visibility: "direct"
-    }
-
-    Benchee.run(%{
-      "Direct messages with muted" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-        |> Pleroma.Pagination.fetch_paginated(params)
-      end,
-      "Direct messages without muted" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-        |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
-      end
-    })
-
-    dms_with_muted =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-      |> Pleroma.Pagination.fetch_paginated(params)
-
-    dms_without_muted =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-      |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
-
-    Benchee.run(%{
-      "Rendering dms with muted" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: dms_with_muted,
-          for: user,
-          as: :activity
-        })
-      end,
-      "Rendering dms without muted" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: dms_without_muted,
-          for: user,
-          as: :activity
-        })
-      end
-    })
-  end
-
-  def query_long_thread(user, activity) do
-    Benchee.run(%{
-      "Fetch main post" => fn ->
-        Pleroma.Activity.get_by_id_with_object(activity.id)
-      end,
-      "Fetch context of main post" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
-          activity.data["context"],
-          %{
-            "blocking_user" => user,
-            "user" => user,
-            "exclude_id" => activity.id
-          }
-        )
-      end
-    })
-
-    activity = Pleroma.Activity.get_by_id_with_object(activity.id)
-
-    context =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
-        activity.data["context"],
-        %{
-          "blocking_user" => user,
-          "user" => user,
-          "exclude_id" => activity.id
-        }
-      )
-
-    Benchee.run(%{
-      "Render status" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{
-          activity: activity,
-          for: user
-        })
-      end,
-      "Render context" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render(
-          "index.json",
-          for: user,
-          activities: context,
-          as: :activity
-        )
-        |> Enum.reverse()
-      end
-    })
-  end
-end
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
deleted file mode 100644
index 5c5a5c122..000000000
--- a/lib/load_testing/generator.ex
+++ /dev/null
@@ -1,352 +0,0 @@
-defmodule Pleroma.LoadTesting.Generator do
-  use Pleroma.LoadTesting.Helper
-  alias Pleroma.Web.CommonAPI
-
-  def generate_users(opts) do
-    IO.puts("Starting generating #{opts[:users_max]} users...")
-    {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
-
-    IO.puts("Inserting users take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_users(opts) do
-    max = Keyword.get(opts, :users_max)
-
-    Task.async_stream(
-      1..max,
-      &generate_user_data(&1),
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Enum.to_list()
-  end
-
-  defp generate_user_data(i) do
-    remote = Enum.random([true, false])
-
-    user = %User{
-      name: "Test テスト User #{i}",
-      email: "user#{i}@example.com",
-      nickname: "nick#{i}",
-      password_hash:
-        "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
-      bio: "Tester Number #{i}",
-      info: %{},
-      local: remote
-    }
-
-    user_urls =
-      if remote do
-        base_url =
-          Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
-
-        ap_id = "#{base_url}/users/#{user.nickname}"
-
-        %{
-          ap_id: ap_id,
-          follower_address: ap_id <> "/followers",
-          following_address: ap_id <> "/following",
-          following: [ap_id]
-        }
-      else
-        %{
-          ap_id: User.ap_id(user),
-          follower_address: User.ap_followers(user),
-          following_address: User.ap_following(user),
-          following: [User.ap_id(user)]
-        }
-      end
-
-    user = Map.merge(user, user_urls)
-
-    Repo.insert!(user)
-  end
-
-  def generate_activities(user, users) do
-    do_generate_activities(user, users)
-  end
-
-  defp do_generate_activities(user, users) do
-    IO.puts("Starting generating 20000 common activities...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..20_000,
-          fn _ ->
-            do_generate_activity([user | users])
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
-
-    IO.puts("Starting generating 20000 activities with mentions...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..20_000,
-          fn _ ->
-            do_generate_activity_with_mention(user, users)
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
-
-    IO.puts("Starting generating 10000 activities with threads...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..10_000,
-          fn _ ->
-            do_generate_threads([user | users])
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_activity(users) do
-    post = %{
-      "status" => "Some status without mention with random user"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  defp do_generate_activity_with_mention(user, users) do
-    mentions_cnt = Enum.random([2, 3, 4, 5])
-    with_user = Enum.random([true, false])
-    users = Enum.shuffle(users)
-    mentions_users = Enum.take(users, mentions_cnt)
-    mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
-
-    mentions_str =
-      Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
-
-    post = %{
-      "status" => mentions_str <> "some status with mentions random users"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  defp do_generate_threads(users) do
-    thread_length = Enum.random([2, 3, 4, 5])
-    actor = Enum.random(users)
-
-    post = %{
-      "status" => "Start of the thread"
-    }
-
-    {:ok, activity} = CommonAPI.post(actor, post)
-
-    Enum.each(1..thread_length, fn _ ->
-      user = Enum.random(users)
-
-      post = %{
-        "status" => "@#{actor.nickname} reply to thread",
-        "in_reply_to_status_id" => activity.id
-      }
-
-      CommonAPI.post(user, post)
-    end)
-  end
-
-  def generate_remote_activities(user, users) do
-    do_generate_remote_activities(user, users)
-  end
-
-  defp do_generate_remote_activities(user, users) do
-    IO.puts("Starting generating 10000 remote activities...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..10_000,
-          fn i ->
-            do_generate_remote_activity(i, user, users)
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_remote_activity(i, user, users) do
-    actor = Enum.random(users)
-    %{host: host} = URI.parse(actor.ap_id)
-    date = Date.utc_today()
-    datetime = DateTime.utc_now()
-
-    map = %{
-      "actor" => actor.ap_id,
-      "cc" => [actor.follower_address, user.ap_id],
-      "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-      "id" => actor.ap_id <> "/statuses/#{i}/activity",
-      "object" => %{
-        "actor" => actor.ap_id,
-        "atomUri" => actor.ap_id <> "/statuses/#{i}",
-        "attachment" => [],
-        "attributedTo" => actor.ap_id,
-        "bcc" => [],
-        "bto" => [],
-        "cc" => [actor.follower_address, user.ap_id],
-        "content" =>
-          "<p><span class=\"h-card\"><a href=\"" <>
-            user.ap_id <>
-            "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
-        "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-        "conversation" =>
-          "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-        "emoji" => %{},
-        "id" => actor.ap_id <> "/statuses/#{i}",
-        "inReplyTo" => nil,
-        "inReplyToAtomUri" => nil,
-        "published" => datetime,
-        "sensitive" => true,
-        "summary" => "cw",
-        "tag" => [
-          %{
-            "href" => user.ap_id,
-            "name" => "@#{user.nickname}@#{host}",
-            "type" => "Mention"
-          }
-        ],
-        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-        "type" => "Note",
-        "url" => "http://#{host}/@#{actor.nickname}/#{i}"
-      },
-      "published" => datetime,
-      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-      "type" => "Create"
-    }
-
-    Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
-  end
-
-  def generate_dms(user, users, opts) do
-    IO.puts("Starting generating #{opts[:dms_max]} DMs")
-    {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
-    IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_dms(user, users, opts) do
-    Task.async_stream(
-      1..opts[:dms_max],
-      fn _ ->
-        do_generate_dm(user, users)
-      end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-  end
-
-  defp do_generate_dm(user, users) do
-    post = %{
-      "status" => "@#{user.nickname} some direct message",
-      "visibility" => "direct"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  def generate_long_thread(user, users, opts) do
-    IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
-    {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
-    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
-    {:ok, activity}
-  end
-
-  defp do_generate_long_thread(user, users, opts) do
-    {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
-
-    Task.async_stream(
-      1..opts[:thread_length],
-      fn _ -> do_generate_thread(users, id) end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-
-    activity
-  end
-
-  defp do_generate_thread(users, activity_id) do
-    CommonAPI.post(Enum.random(users), %{
-      "status" => "reply to main post",
-      "in_reply_to_status_id" => activity_id
-    })
-  end
-
-  def generate_non_visible_message(user, users) do
-    IO.puts("Starting generating 1000 non visible posts")
-
-    {time, _} =
-      :timer.tc(fn ->
-        do_generate_non_visible_posts(user, users)
-      end)
-
-    IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_non_visible_posts(user, users) do
-    [not_friend | users] = users
-
-    make_friends(user, users)
-
-    Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-  end
-
-  defp make_friends(_user, []), do: nil
-
-  defp make_friends(user, [friend | users]) do
-    {:ok, _} = User.follow(user, friend)
-    {:ok, _} = User.follow(friend, user)
-    make_friends(user, users)
-  end
-
-  defp do_generate_non_visible_post(not_friend, users) do
-    post = %{
-      "status" => "some non visible post",
-      "visibility" => "private"
-    }
-
-    {:ok, activity} = CommonAPI.post(not_friend, post)
-
-    thread_length = Enum.random([2, 3, 4, 5])
-
-    Enum.each(1..thread_length, fn _ ->
-      user = Enum.random(users)
-
-      post = %{
-        "status" => "@#{not_friend.nickname} reply to non visible post",
-        "in_reply_to_status_id" => activity.id,
-        "visibility" => "private"
-      }
-
-      CommonAPI.post(user, post)
-    end)
-  end
-end
diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex
deleted file mode 100644
index 47b25c65f..000000000
--- a/lib/load_testing/helper.ex
+++ /dev/null
@@ -1,11 +0,0 @@
-defmodule Pleroma.LoadTesting.Helper do
-  defmacro __using__(_) do
-    quote do
-      import Ecto.Query
-      alias Pleroma.Repo
-      alias Pleroma.User
-
-      defp to_sec(microseconds), do: microseconds / 1_000_000
-    end
-  end
-end
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
deleted file mode 100644
index a7057395d..000000000
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ /dev/null
@@ -1,134 +0,0 @@
-defmodule Mix.Tasks.Pleroma.LoadTesting do
-  use Mix.Task
-  use Pleroma.LoadTesting.Helper
-  import Mix.Pleroma
-  import Pleroma.LoadTesting.Generator
-  import Pleroma.LoadTesting.Fetcher
-
-  @shortdoc "Factory for generation data"
-  @moduledoc """
-  Generates data like:
-  - users
-  - activities with notifications
-  - direct messages
-  - long thread
-  - non visible posts
-
-  ## Generate data
-      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
-      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
-
-  Options:
-  - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
-  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
-  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
-  """
-
-  @aliases [u: :users, d: :dms, t: :thread_length]
-  @switches [
-    users: :integer,
-    dms: :integer,
-    thread_length: :integer
-  ]
-  @users_default 20_000
-  @dms_default 20_000
-  @thread_length_default 2_000
-
-  def run(args) do
-    start_pleroma()
-    Pleroma.Config.put([:instance, :skip_thread_containment], true)
-    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
-
-    users_max = Keyword.get(opts, :users, @users_default)
-    dms_max = Keyword.get(opts, :dms, @dms_default)
-    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
-
-    clean_tables()
-
-    opts =
-      Keyword.put(opts, :users_max, users_max)
-      |> Keyword.put(:dms_max, dms_max)
-      |> Keyword.put(:thread_length, thread_length)
-
-    generate_users(opts)
-
-    # main user for queries
-    IO.puts("Fetching local main user...")
-
-    {time, user} =
-      :timer.tc(fn ->
-        Repo.one(
-          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
-        )
-      end)
-
-    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
-
-    IO.puts("Fetching local users...")
-
-    {time, users} =
-      :timer.tc(fn ->
-        Repo.all(
-          from(u in User,
-            where: u.id != ^user.id,
-            where: u.local == true,
-            order_by: fragment("RANDOM()"),
-            limit: 10
-          )
-        )
-      end)
-
-    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
-
-    IO.puts("Fetching remote users...")
-
-    {time, remote_users} =
-      :timer.tc(fn ->
-        Repo.all(
-          from(u in User,
-            where: u.id != ^user.id,
-            where: u.local == false,
-            order_by: fragment("RANDOM()"),
-            limit: 10
-          )
-        )
-      end)
-
-    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
-
-    generate_activities(user, users)
-
-    generate_remote_activities(user, remote_users)
-
-    generate_dms(user, users, opts)
-
-    {:ok, activity} = generate_long_thread(user, users, opts)
-
-    generate_non_visible_message(user, users)
-
-    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
-
-    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}")
-
-    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}")
-
-    IO.puts(
-      "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
-    )
-
-    fetch_user(user)
-    query_timelines(user)
-    query_notifications(user)
-    query_dms(user)
-    query_long_thread(user, activity)
-    Pleroma.Config.put([:instance, :skip_thread_containment], false)
-    query_timelines(user)
-  end
-
-  defp clean_tables do
-    IO.puts("Deleting old data...\n")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
-  end
-end
-- 
cgit v1.2.3


From ad42837244ba4c945b76c5addaffe47353cf62a8 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Wed, 9 Oct 2019 17:03:54 +0300
Subject: Ability to toggle activation status and permission group for a group
 of users

---
 lib/pleroma/moderation_log.ex                     | 60 +++++++--------
 lib/pleroma/user.ex                               | 16 +++-
 lib/pleroma/web/admin_api/admin_api_controller.ex | 89 ++++++++++-------------
 lib/pleroma/web/admin_api/views/account_view.ex   |  6 ++
 lib/pleroma/web/router.ex                         | 14 +---
 5 files changed, 95 insertions(+), 90 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex
index 352cad433..42649ff02 100644
--- a/lib/pleroma/moderation_log.ex
+++ b/lib/pleroma/moderation_log.ex
@@ -86,18 +86,18 @@ defmodule Pleroma.ModerationLog do
     parsed_datetime
   end
 
-  @spec insert_log(%{actor: User, subject: User, action: String.t(), permission: String.t()}) ::
+  @spec insert_log(%{actor: User, subject: [User], action: String.t(), permission: String.t()}) ::
           {:ok, ModerationLog} | {:error, any}
   def insert_log(%{
         actor: %User{} = actor,
-        subject: %User{} = subject,
+        subject: subjects,
         action: action,
         permission: permission
       }) do
     %ModerationLog{
       data: %{
         "actor" => user_to_map(actor),
-        "subject" => user_to_map(subject),
+        "subject" => user_to_map(subjects),
         "action" => action,
         "permission" => permission,
         "message" => ""
@@ -303,13 +303,16 @@ defmodule Pleroma.ModerationLog do
   end
 
   @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any}
-
   defp insert_log_entry_with_message(entry) do
     entry.data["message"]
     |> put_in(get_log_entry_message(entry))
     |> Repo.insert()
   end
 
+  defp user_to_map(users) when is_list(users) do
+    users |> Enum.map(&user_to_map/1)
+  end
+
   defp user_to_map(%User{} = user) do
     user
     |> Map.from_struct()
@@ -363,12 +366,7 @@ defmodule Pleroma.ModerationLog do
           "subjects" => subjects
         }
       }) do
-    nicknames =
-      subjects
-      |> Enum.map(&"@#{&1["nickname"]}")
-      |> Enum.join(", ")
-
-    "@#{actor_nickname} created users: #{nicknames}"
+    "@#{actor_nickname} created users: #{users_to_nicknames_string(subjects)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -376,10 +374,10 @@ defmodule Pleroma.ModerationLog do
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "activate",
-          "subject" => %{"nickname" => subject_nickname, "type" => "user"}
+          "subject" => users
         }
       }) do
-    "@#{actor_nickname} activated user @#{subject_nickname}"
+    "@#{actor_nickname} activated users: #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -387,10 +385,10 @@ defmodule Pleroma.ModerationLog do
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "deactivate",
-          "subject" => %{"nickname" => subject_nickname, "type" => "user"}
+          "subject" => users
         }
       }) do
-    "@#{actor_nickname} deactivated user @#{subject_nickname}"
+    "@#{actor_nickname} deactivated users: #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -402,14 +400,9 @@ defmodule Pleroma.ModerationLog do
           "action" => "tag"
         }
       }) do
-    nicknames_string =
-      nicknames
-      |> Enum.map(&"@#{&1}")
-      |> Enum.join(", ")
-
     tags_string = tags |> Enum.join(", ")
 
-    "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_string}"
+    "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_to_string(nicknames)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -421,14 +414,9 @@ defmodule Pleroma.ModerationLog do
           "action" => "untag"
         }
       }) do
-    nicknames_string =
-      nicknames
-      |> Enum.map(&"@#{&1}")
-      |> Enum.join(", ")
-
     tags_string = tags |> Enum.join(", ")
 
-    "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_string}"
+    "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_to_string(nicknames)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -436,11 +424,11 @@ defmodule Pleroma.ModerationLog do
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "grant",
-          "subject" => %{"nickname" => subject_nickname},
+          "subject" => users,
           "permission" => permission
         }
       }) do
-    "@#{actor_nickname} made @#{subject_nickname} #{permission}"
+    "@#{actor_nickname} made #{users_to_nicknames_string(users)} #{permission}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -448,11 +436,11 @@ defmodule Pleroma.ModerationLog do
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "revoke",
-          "subject" => %{"nickname" => subject_nickname},
+          "subject" => users,
           "permission" => permission
         }
       }) do
-    "@#{actor_nickname} revoked #{permission} role from @#{subject_nickname}"
+    "@#{actor_nickname} revoked #{permission} role from #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -551,4 +539,16 @@ defmodule Pleroma.ModerationLog do
       }) do
     "@#{actor_nickname} deleted status ##{subject_id}"
   end
+
+  defp nicknames_to_string(nicknames) do
+    nicknames
+    |> Enum.map(&"@#{&1}")
+    |> Enum.join(", ")
+  end
+
+  defp users_to_nicknames_string(users) do
+    users
+    |> Enum.map(&"@#{&1["nickname"]}")
+    |> Enum.join(", ")
+  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 2cfb13a8c..a76a5ad70 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1059,7 +1059,15 @@ defmodule Pleroma.User do
     BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status})
   end
 
-  def deactivate(%User{} = user, status \\ true) do
+  def deactivate(user, status \\ true)
+
+  def deactivate(users, status) when is_list(users) do
+    Repo.transaction(fn ->
+      for user <- users, do: deactivate(user, status)
+    end)
+  end
+
+  def deactivate(%User{} = user, status) do
     with {:ok, user} <- update_info(user, &User.Info.set_activation_status(&1, status)) do
       Enum.each(get_followers(user), &invalidate_cache/1)
       Enum.each(get_friends(user), &update_follower_count/1)
@@ -1625,6 +1633,12 @@ defmodule Pleroma.User do
 
   `fun` is called with the `user.info`.
   """
+  def update_info(users, fun) when is_list(users) do
+    Repo.transaction(fn ->
+      for user <- users, do: update_info(user, fun)
+    end)
+  end
+
   def update_info(user, fun) do
     user
     |> change_info(fun)
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 513bae800..d825a5d28 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -231,22 +231,34 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
-  def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
-    user = User.get_cached_by_nickname(nickname)
+  def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, false)
 
-    {:ok, updated_user} = User.deactivate(user, !user.info.deactivated)
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "activate"
+    })
 
-    action = if user.info.deactivated, do: "activate", else: "deactivate"
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, true)
 
     ModerationLog.insert_log(%{
       actor: admin,
-      subject: user,
-      action: action
+      subject: users,
+      action: "deactivate"
     })
 
     conn
     |> put_view(AccountView)
-    |> render("show.json", %{user: updated_user})
+    |> render("index.json", %{users: Keyword.values(updated_users)})
   end
 
   def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do
@@ -315,20 +327,19 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   def right_add(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
-        "nickname" => nickname
+        "nicknames" => nicknames
       })
       when permission_group in ["moderator", "admin"] do
     info = Map.put(%{}, "is_" <> permission_group, true)
 
-    {:ok, user} =
-      nickname
-      |> User.get_cached_by_nickname()
-      |> User.update_info(&User.Info.admin_api_update(&1, info))
+    users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
+
+    User.update_info(users, &User.Info.admin_api_update(&1, info))
 
     ModerationLog.insert_log(%{
       action: "grant",
       actor: admin,
-      subject: user,
+      subject: users,
       permission: permission_group
     })
 
@@ -349,58 +360,38 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     })
   end
 
-  def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do
-    render_error(conn, :forbidden, "You can't revoke your own admin status.")
-  end
-
   def right_delete(
-        %{assigns: %{user: admin}} = conn,
+        %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn,
         %{
           "permission_group" => permission_group,
-          "nickname" => nickname
+          "nicknames" => nicknames
         }
       )
       when permission_group in ["moderator", "admin"] do
-    info = Map.put(%{}, "is_" <> permission_group, false)
+    with false <- Enum.member?(nicknames, admin_nickname) do
+      info = Map.put(%{}, "is_" <> permission_group, false)
 
-    {:ok, user} =
-      nickname
-      |> User.get_cached_by_nickname()
-      |> User.update_info(&User.Info.admin_api_update(&1, info))
+      users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
 
-    ModerationLog.insert_log(%{
-      action: "revoke",
-      actor: admin,
-      subject: user,
-      permission: permission_group
-    })
-
-    json(conn, info)
-  end
-
-  def right_delete(conn, _) do
-    render_error(conn, :not_found, "No such permission_group")
-  end
-
-  def set_activation_status(%{assigns: %{user: admin}} = conn, %{
-        "nickname" => nickname,
-        "status" => status
-      }) do
-    with {:ok, status} <- Ecto.Type.cast(:boolean, status),
-         %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, _} <- User.deactivate(user, !status) do
-      action = if(user.info.deactivated, do: "activate", else: "deactivate")
+      User.update_info(users, &User.Info.admin_api_update(&1, info))
 
       ModerationLog.insert_log(%{
+        action: "revoke",
         actor: admin,
-        subject: user,
-        action: action
+        subject: users,
+        permission: permission_group
       })
 
-      json_response(conn, :no_content, "")
+      json(conn, info)
+    else
+      _ -> render_error(conn, :forbidden, "You can't revoke your own admin/moderator status.")
     end
   end
 
+  def right_delete(conn, _) do
+    render_error(conn, :not_found, "No such permission_group")
+  end
+
   def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do
     with {:ok, _message} <- Relay.follow(target) do
       ModerationLog.insert_log(%{
diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index a96affd40..441269162 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -19,6 +19,12 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
     }
   end
 
+  def render("index.json", %{users: users}) do
+    %{
+      users: render_many(users, AccountView, "show.json", as: :user)
+    }
+  end
+
   def render("show.json", %{user: user}) do
     avatar = User.avatar_url(user) |> MediaProxy.url()
     display_name = HTML.strip_tags(user.name || user.nickname)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index ae799b8ac..894375357 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -136,21 +136,15 @@ defmodule Pleroma.Web.Router do
 
     delete("/users", AdminAPIController, :user_delete)
     post("/users", AdminAPIController, :users_create)
-    patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
+    patch("/users/activate", AdminAPIController, :user_activate)
+    patch("/users/deactivate", AdminAPIController, :user_deactivate)
     put("/users/tag", AdminAPIController, :tag_users)
     delete("/users/tag", AdminAPIController, :untag_users)
 
     get("/users/:nickname/permission_group", AdminAPIController, :right_get)
     get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get)
-    post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add)
-
-    delete(
-      "/users/:nickname/permission_group/:permission_group",
-      AdminAPIController,
-      :right_delete
-    )
-
-    put("/users/:nickname/activation_status", AdminAPIController, :set_activation_status)
+    post("/users/permission_group/:permission_group", AdminAPIController, :right_add)
+    delete("/users/permission_group/:permission_group", AdminAPIController, :right_delete)
 
     post("/relay", AdminAPIController, :relay_follow)
     delete("/relay", AdminAPIController, :relay_unfollow)
-- 
cgit v1.2.3


From 29647dfd09a78fa66a605ff97119a21779d15020 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 10 Oct 2019 17:17:33 +0200
Subject: Transmogrifier: Save correct ids for incoming deletes.

---
 lib/pleroma/web/activity_pub/activity_pub.ex   | 8 ++++++--
 lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++--
 2 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 9f29087df..3a0c382ca 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -409,7 +409,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do
+  def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ []) do
+    local = Keyword.get(options, :local, true)
+    activity_id = Keyword.get(options, :activity_id, nil)
+
     user = User.get_cached_by_ap_id(actor)
     to = (object.data["to"] || []) ++ (object.data["cc"] || [])
 
@@ -420,7 +423,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            "object" => id,
            "to" => to,
            "deleted_activity_id" => activity && activity.id
-         },
+         }
+         |> maybe_put("id", activity_id),
          {:ok, activity} <- insert(data, local, false),
          stream_out_participations(object, user),
          _ <- decrease_replies_count_if_reply(object),
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 872ed0eb2..620b54d3b 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -637,7 +637,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   # an error or a tombstone.  This would allow us to verify that a deletion actually took
   # place.
   def handle_incoming(
-        %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data,
+        %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => id} = data,
         _options
       ) do
     object_id = Utils.get_ap_id(object_id)
@@ -646,7 +646,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          :ok <- Containment.contain_origin(actor.ap_id, object.data),
-         {:ok, activity} <- ActivityPub.delete(object, false) do
+         {:ok, activity} <- ActivityPub.delete(object, local: false, activity_id: id) do
       {:ok, activity}
     else
       nil ->
-- 
cgit v1.2.3


From f5104f36bbec7d49d4ff5acee4b9d28223c6474d Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 11 Oct 2019 00:24:31 +0300
Subject: Deprecate /api/pleroma/admin/users/:nickname/toggle_activation
 instead of deleting it

---
 lib/pleroma/web/admin_api/admin_api_controller.ex | 20 ++++++++++++++++++++
 lib/pleroma/web/router.ex                         |  1 +
 2 files changed, 21 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index d825a5d28..5b513bd7c 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -46,6 +46,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            :user_delete,
            :users_create,
            :user_toggle_activation,
+           :user_activate,
+           :user_deactivate,
            :tag_users,
            :untag_users,
            :right_add,
@@ -231,6 +233,24 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    user = User.get_cached_by_nickname(nickname)
+
+    {:ok, updated_user} = User.deactivate(user, !user.info.deactivated)
+
+    action = if user.info.deactivated, do: "activate", else: "deactivate"
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: [user],
+      action: action
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("show.json", %{user: updated_user})
+  end
+
   def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
     users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
     {:ok, updated_users} = User.deactivate(users, false)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 894375357..a79df51a2 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -136,6 +136,7 @@ defmodule Pleroma.Web.Router do
 
     delete("/users", AdminAPIController, :user_delete)
     post("/users", AdminAPIController, :users_create)
+    patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
     patch("/users/activate", AdminAPIController, :user_activate)
     patch("/users/deactivate", AdminAPIController, :user_deactivate)
     put("/users/tag", AdminAPIController, :tag_users)
-- 
cgit v1.2.3


From 9b963064eb4c8cb740ffa128f491f2ee581fdb8b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 11 Oct 2019 11:25:45 +0200
Subject: Transmogrifier: Actually store who deleted a note.

---
 lib/pleroma/web/activity_pub/activity_pub.ex   | 18 ++++++++++--------
 lib/pleroma/web/activity_pub/transmogrifier.ex |  3 ++-
 2 files changed, 12 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 3a0c382ca..f78574455 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -412,19 +412,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ []) do
     local = Keyword.get(options, :local, true)
     activity_id = Keyword.get(options, :activity_id, nil)
+    actor = Keyword.get(options, :actor, actor)
 
     user = User.get_cached_by_ap_id(actor)
     to = (object.data["to"] || []) ++ (object.data["cc"] || [])
 
     with {:ok, object, activity} <- Object.delete(object),
-         data <- %{
-           "type" => "Delete",
-           "actor" => actor,
-           "object" => id,
-           "to" => to,
-           "deleted_activity_id" => activity && activity.id
-         }
-         |> maybe_put("id", activity_id),
+         data <-
+           %{
+             "type" => "Delete",
+             "actor" => actor,
+             "object" => id,
+             "to" => to,
+             "deleted_activity_id" => activity && activity.id
+           }
+           |> maybe_put("id", activity_id),
          {:ok, activity} <- insert(data, local, false),
          stream_out_participations(object, user),
          _ <- decrease_replies_count_if_reply(object),
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 620b54d3b..64fbb9fa4 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -646,7 +646,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          :ok <- Containment.contain_origin(actor.ap_id, object.data),
-         {:ok, activity} <- ActivityPub.delete(object, local: false, activity_id: id) do
+         {:ok, activity} <-
+           ActivityPub.delete(object, local: false, activity_id: id, actor: actor.ap_id) do
       {:ok, activity}
     else
       nil ->
-- 
cgit v1.2.3


From 37812740c4c03f0a79d17ff186db644a60414ff7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 11 Oct 2019 11:48:58 +0200
Subject: Transmogrifier: Correctly save incoming ids for Accept/Reject.

---
 lib/pleroma/web/activity_pub/activity_pub.ex   | 23 +++++++++++------------
 lib/pleroma/web/activity_pub/transmogrifier.ex | 10 ++++++----
 2 files changed, 17 insertions(+), 16 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index f78574455..364452b5d 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -269,22 +269,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def accept(%{to: to, actor: actor, object: object} = params) do
-    # only accept false as false value
-    local = !(params[:local] == false)
+  def accept(params) do
+    accept_or_reject("Accept", params)
+  end
 
-    with data <- %{"to" => to, "type" => "Accept", "actor" => actor.ap_id, "object" => object},
-         {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity}
-    end
+  def reject(params) do
+    accept_or_reject("Reject", params)
   end
 
-  def reject(%{to: to, actor: actor, object: object} = params) do
-    # only accept false as false value
-    local = !(params[:local] == false)
+  def accept_or_reject(type, %{to: to, actor: actor, object: object} = params) do
+    local = Map.get(params, :local, true)
+    activity_id = Map.get(params, :activity_id, nil)
 
-    with data <- %{"to" => to, "type" => "Reject", "actor" => actor.ap_id, "object" => object},
+    with data <-
+           %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object}
+           |> Utils.maybe_put("id", activity_id),
          {:ok, activity} <- insert(data, local),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 64fbb9fa4..b56343beb 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -514,7 +514,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data,
+        %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => id} = data,
         _options
       ) do
     with actor <- Containment.get_actor(data),
@@ -528,7 +528,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         type: "Accept",
         actor: followed,
         object: follow_activity.data["id"],
-        local: false
+        local: false,
+        activity_id: id
       })
     else
       _e -> :error
@@ -536,7 +537,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data,
+        %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => id} = data,
         _options
       ) do
     with actor <- Containment.get_actor(data),
@@ -550,7 +551,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
              type: "Reject",
              actor: followed,
              object: follow_activity.data["id"],
-             local: false
+             local: false,
+             activity_id: id
            }) do
       User.unfollow(follower, followed)
 
-- 
cgit v1.2.3


From 422aa6befee0cbcbf71d3cacd96d90c1be3263ba Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 11 Oct 2019 12:53:09 +0200
Subject: Ostatus DeleteHandler: Fix for new option format.

---
 lib/pleroma/web/ostatus/handlers/delete_handler.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex
index b2f9f3946..ac2dc115c 100644
--- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex
+++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do
   def handle_delete(entry, _doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),
          %Object{} = object <- Object.normalize(id),
-         {:ok, delete} <- ActivityPub.delete(object, false) do
+         {:ok, delete} <- ActivityPub.delete(object, local: false) do
       delete
     end
   end
-- 
cgit v1.2.3


From aaa4252f416fbad099f95232de4cf6eab11dd7d2 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 11 Oct 2019 15:58:45 +0300
Subject: Deprecate POST/DELETE
 /api/pleroma/admin/users/:nickname/permission_group/:permission_group instead
 of deleting it

---
 lib/pleroma/web/admin_api/admin_api_controller.ex | 61 +++++++++++++++++++++--
 lib/pleroma/web/router.ex                         | 18 ++++++-
 2 files changed, 74 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 5b513bd7c..33e2180ec 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -345,7 +345,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> Enum.into(%{}, &{&1, true})
   end
 
-  def right_add(%{assigns: %{user: admin}} = conn, %{
+  def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
         "nicknames" => nicknames
       })
@@ -366,6 +366,32 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     json(conn, info)
   end
 
+  def right_add_multiple(conn, _) do
+    render_error(conn, :not_found, "No such permission_group")
+  end
+
+  def right_add(%{assigns: %{user: admin}} = conn, %{
+        "permission_group" => permission_group,
+        "nickname" => nickname
+      })
+      when permission_group in ["moderator", "admin"] do
+    info = Map.put(%{}, "is_" <> permission_group, true)
+
+    {:ok, user} =
+      nickname
+      |> User.get_cached_by_nickname()
+      |> User.update_info(&User.Info.admin_api_update(&1, info))
+
+    ModerationLog.insert_log(%{
+      action: "grant",
+      actor: admin,
+      subject: [user],
+      permission: permission_group
+    })
+
+    json(conn, info)
+  end
+
   def right_add(conn, _) do
     render_error(conn, :not_found, "No such permission_group")
   end
@@ -380,7 +406,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     })
   end
 
-  def right_delete(
+  def right_delete_multiple(
         %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn,
         %{
           "permission_group" => permission_group,
@@ -408,10 +434,39 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
-  def right_delete(conn, _) do
+  def right_delete_multiple(conn, _) do
     render_error(conn, :not_found, "No such permission_group")
   end
 
+  def right_delete(
+        %{assigns: %{user: admin}} = conn,
+        %{
+          "permission_group" => permission_group,
+          "nickname" => nickname
+        }
+      )
+      when permission_group in ["moderator", "admin"] do
+    info = Map.put(%{}, "is_" <> permission_group, false)
+
+    {:ok, user} =
+      nickname
+      |> User.get_cached_by_nickname()
+      |> User.update_info(&User.Info.admin_api_update(&1, info))
+
+    ModerationLog.insert_log(%{
+      action: "revoke",
+      actor: admin,
+      subject: [user],
+      permission: permission_group
+    })
+
+    json(conn, info)
+  end
+
+  def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do
+    render_error(conn, :forbidden, "You can't revoke your own admin status.")
+  end
+
   def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do
     with {:ok, _message} <- Relay.follow(target) do
       ModerationLog.insert_log(%{
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index a79df51a2..80651f3ff 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -144,8 +144,22 @@ defmodule Pleroma.Web.Router do
 
     get("/users/:nickname/permission_group", AdminAPIController, :right_get)
     get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get)
-    post("/users/permission_group/:permission_group", AdminAPIController, :right_add)
-    delete("/users/permission_group/:permission_group", AdminAPIController, :right_delete)
+
+    post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add)
+
+    delete(
+      "/users/:nickname/permission_group/:permission_group",
+      AdminAPIController,
+      :right_delete
+    )
+
+    post("/users/permission_group/:permission_group", AdminAPIController, :right_add_multiple)
+
+    delete(
+      "/users/permission_group/:permission_group",
+      AdminAPIController,
+      :right_delete_multiple
+    )
 
     post("/relay", AdminAPIController, :relay_follow)
     delete("/relay", AdminAPIController, :relay_unfollow)
-- 
cgit v1.2.3


From 9bdbf0811b362d8eb9d2456e9c87c9dde1b35d9b Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 11 Oct 2019 22:52:38 +0300
Subject: Make MediaProxy failure tracking less brutal

The current failure tracking mechanism will never request anything that
didn't respond with a success, 403, 404, or 5xx codes. This is causing
issues when using in real fediverse because of weird status codes
some software has and timeouts being frequent. This patch changes
failure tracking mechanism to only never request the url again if it
responded with 400, 204, or the body is too large, otherwise it can be
re-requested in 60 seconds.
---
 lib/pleroma/reverse_proxy/reverse_proxy.ex | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex
index 78144cae3..2ed719315 100644
--- a/lib/pleroma/reverse_proxy/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex
@@ -401,11 +401,9 @@ defmodule Pleroma.ReverseProxy do
 
   defp client, do: Pleroma.ReverseProxy.Client
 
-  defp track_failed_url(url, code, opts) do
-    code = to_string(code)
-
+  defp track_failed_url(url, error, opts) do
     ttl =
-      if code in ["403", "404"] or String.starts_with?(code, "5") do
+      unless error in [:body_too_large, 400, 204] do
         Keyword.get(opts, :failed_request_ttl, @failed_request_ttl)
       else
         nil
-- 
cgit v1.2.3


From a97b642289659a5fccb5943c54caa1ecdce5fd2f Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Tue, 8 Oct 2019 23:05:57 +0300
Subject: Mastodon API: Add `exclude_visibilities` parameter to the timeline
 and notification endpoints

---
 lib/pleroma/notification.ex                  | 104 ++++++++++++++++++++-------
 lib/pleroma/web/activity_pub/activity_pub.ex |  44 ++++++++++++
 lib/pleroma/web/mastodon_api/mastodon_api.ex |   1 +
 3 files changed, 122 insertions(+), 27 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index d94ae5971..d145f8d5b 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -17,6 +17,7 @@ defmodule Pleroma.Notification do
 
   import Ecto.Query
   import Ecto.Changeset
+  require Logger
 
   @type t :: %__MODULE__{}
 
@@ -34,43 +35,92 @@ defmodule Pleroma.Notification do
   end
 
   def for_user_query(user, opts \\ []) do
-    query =
-      Notification
-      |> where(user_id: ^user.id)
-      |> where(
-        [n, a],
+    Notification
+    |> where(user_id: ^user.id)
+    |> where(
+      [n, a],
+      fragment(
+        "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
+        a.actor
+      )
+    )
+    |> join(:inner, [n], activity in assoc(n, :activity))
+    |> join(:left, [n, a], object in Object,
+      on:
         fragment(
-          "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
-          a.actor
+          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          object.data,
+          a.data
         )
-      )
-      |> join(:inner, [n], activity in assoc(n, :activity))
-      |> join(:left, [n, a], object in Object,
-        on:
-          fragment(
-            "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
-            object.data,
-            a.data
-          )
-      )
-      |> preload([n, a, o], activity: {a, object: o})
+    )
+    |> preload([n, a, o], activity: {a, object: o})
+    |> exclude_muted(user, opts)
+    |> exclude_visibility(opts)
+  end
+
+  defp exclude_muted(query, _, %{with_muted: true}) do
+    query
+  end
+
+  defp exclude_muted(query, user, _opts) do
+    query
+    |> where([n, a], a.actor not in ^user.info.muted_notifications)
+    |> where([n, a], a.actor not in ^user.info.blocks)
+    |> where(
+      [n, a],
+      fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks
+    )
+    |> join(:left, [n, a], tm in Pleroma.ThreadMute,
+      on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
+    )
+    |> where([n, a, o, tm], is_nil(tm.user_id))
+  end
 
-    if opts[:with_muted] do
+  @valid_visibilities ~w[direct unlisted public private]
+
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when is_list(visibility) do
+    if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
       query
-    else
-      where(query, [n, a], a.actor not in ^user.info.muted_notifications)
-      |> where([n, a], a.actor not in ^user.info.blocks)
       |> where(
         [n, a],
-        fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks
-      )
-      |> join(:left, [n, a], tm in Pleroma.ThreadMute,
-        on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
+        not fragment(
+          "activity_visibility(?, ?, ?) = ANY (?)",
+          a.actor,
+          a.recipients,
+          a.data,
+          ^visibility
+        )
       )
-      |> where([n, a, o, tm], is_nil(tm.user_id))
+    else
+      Logger.error("Could not exclude visibility to #{visibility}")
+      query
     end
   end
 
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when visibility in @valid_visibilities do
+    query
+    |> where(
+      [n, a],
+      not fragment(
+        "activity_visibility(?, ?, ?) = (?)",
+        a.actor,
+        a.recipients,
+        a.data,
+        ^visibility
+      )
+    )
+  end
+
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when visibility not in @valid_visibilities do
+    Logger.error("Could not exclude visibility to #{visibility}")
+    query
+  end
+
+  defp exclude_visibility(query, _visibility), do: query
+
   def for_user(user, opts \\ %{}) do
     user
     |> for_user_query(opts)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 364452b5d..1d34c4d7e 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -596,6 +596,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_visibility(query, _visibility), do: query
 
+  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+       when is_list(visibility) do
+    if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
+      from(
+        a in query,
+        where:
+          not fragment(
+            "activity_visibility(?, ?, ?) = ANY (?)",
+            a.actor,
+            a.recipients,
+            a.data,
+            ^visibility
+          )
+      )
+    else
+      Logger.error("Could not exclude visibility to #{visibility}")
+      query
+    end
+  end
+
+  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+       when visibility in @valid_visibilities do
+    from(
+      a in query,
+      where:
+        not fragment(
+          "activity_visibility(?, ?, ?) = ?",
+          a.actor,
+          a.recipients,
+          a.data,
+          ^visibility
+        )
+    )
+  end
+
+  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+       when visibility not in @valid_visibilities do
+    Logger.error("Could not exclude visibility to #{visibility}")
+    query
+  end
+
+  defp exclude_visibility(query, _visibility), do: query
+
   defp restrict_thread_visibility(query, _, %{skip_thread_containment: true} = _),
     do: query
 
@@ -960,6 +1003,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_muted_reblogs(opts)
     |> Activity.restrict_deactivated_users()
     |> exclude_poll_votes(opts)
+    |> exclude_visibility(opts)
   end
 
   def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex
index ac01d1ff3..d875a5788 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex
@@ -71,6 +71,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do
   defp cast_params(params) do
     param_types = %{
       exclude_types: {:array, :string},
+      exclude_visibilities: {:array, :string},
       reblogs: :boolean,
       with_muted: :boolean
     }
-- 
cgit v1.2.3


From 5bd0717de286a918e20c5ef2409e5f4cbcea8524 Mon Sep 17 00:00:00 2001
From: kPherox <admin@mail.kr-kp.com>
Date: Tue, 15 Oct 2019 19:10:22 +0900
Subject: Add `Sec-WebSocket-Protocol` to response header

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 3c26eb406..1a757363f 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -35,6 +35,13 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
          {_, stream} <- List.keyfind(params, "stream", 0),
          {:ok, user} <- allow_request(stream, [access_token, sec_websocket]),
          topic when is_binary(topic) <- expand_topic(stream, params) do
+      req =
+        if sec_websocket != nil do
+          :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
+        else
+          req
+        end
+
       {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}}
     else
       {:error, code} ->
-- 
cgit v1.2.3


From e7bb762ec256bb5dc607797c042aa9432a9aa993 Mon Sep 17 00:00:00 2001
From: Alexander <alex.strizhakov@gmail.com>
Date: Tue, 15 Oct 2019 15:16:17 +0300
Subject: don't stream in benchmark env

---
 lib/pleroma/web/streamer/streamer.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 8cf719277..2fc7ac8cf 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -49,7 +49,7 @@ defmodule Pleroma.Web.Streamer do
     end
   end
 
-  defp handle_should_send(_) do
-    true
-  end
+  defp handle_should_send(:benchmark), do: false
+
+  defp handle_should_send(_), do: true
 end
-- 
cgit v1.2.3


From c10ce113d487d71c4daa6fabcc641a5caa0d04cb Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 16 Oct 2019 12:52:47 +0300
Subject: User search: Remove trigram and refactor the module

- Remove trigram as it tends to rank garbage results highly, resulting
in it prioritized above fts, which gives actually decent results. ACKed
by kaniini and lain on irc.
- Remove a test for handling misspelled requests, since we no longer have
trigram
- Remove a test for searching users with `nil` display names, because it
is unrealistic, we don't accept usernames that are not >1 char strings
- Make rank boosting for followers/followees sane again, previous values
resulted in garbage matches getting on top just because the users are
followers/followees
---
 lib/pleroma/user/search.ex | 152 +++++++++++++++------------------------------
 1 file changed, 50 insertions(+), 102 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index 6fb2c2352..fb2f3fedb 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -4,7 +4,6 @@
 
 defmodule Pleroma.User.Search do
   alias Pleroma.Pagination
-  alias Pleroma.Repo
   alias Pleroma.User
   import Ecto.Query
 
@@ -23,18 +22,10 @@ defmodule Pleroma.User.Search do
 
     maybe_resolve(resolve, for_user, query_string)
 
-    {:ok, results} =
-      Repo.transaction(fn ->
-        Ecto.Adapters.SQL.query(
-          Repo,
-          "select set_limit(#{@similarity_threshold})",
-          []
-        )
-
-        query_string
-        |> search_query(for_user, following)
-        |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
-      end)
+    results =
+      query_string
+      |> search_query(for_user, following)
+      |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
 
     results
   end
@@ -56,15 +47,52 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_blocked_domains(for_user)
-    |> search_subqueries(query_string)
-    |> union_subqueries
-    |> distinct_query()
-    |> boost_search_rank_query(for_user)
+    |> fts_subquery(query_string)
     |> subquery()
+    |> where([u], u.search_rank > @similarity_threshold)
+    |> boost_search_rank(for_user)
     |> order_by(desc: :search_rank)
     |> maybe_restrict_local(for_user)
   end
 
+  @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/
+  defp fts_subquery(query, query_string) do
+    {nickname_weight, name_weight} =
+      if String.match?(query_string, @nickname_regex) do
+        {"A", "B"}
+      else
+        {"B", "A"}
+      end
+
+    query_string = to_tsquery(query_string)
+
+    from(
+      u in query,
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+            ts_rank_cd((setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)), to_tsquery('simple', ?))
+            """,
+            u.name,
+            ^name_weight,
+            u.nickname,
+            ^nickname_weight,
+            ^query_string
+          )
+      }
+    )
+  end
+
+  defp to_tsquery(query_string) do
+    String.trim_trailing(query_string, "@" <> local_domain())
+    |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
+    |> String.trim()
+    |> String.split()
+    |> Enum.map(&(&1 <> ":*"))
+    |> Enum.join(" | ")
+  end
+
   defp base_query(_user, false), do: User
   defp base_query(user, true), do: User.get_followers_query(user)
 
@@ -87,21 +115,6 @@ defmodule Pleroma.User.Search do
 
   defp filter_blocked_domains(query, _), do: query
 
-  defp union_subqueries({fts_subquery, trigram_subquery}) do
-    from(s in trigram_subquery, union_all: ^fts_subquery)
-  end
-
-  defp search_subqueries(base_query, query_string) do
-    {
-      fts_search_subquery(base_query, query_string),
-      trigram_search_subquery(base_query, query_string)
-    }
-  end
-
-  defp distinct_query(q) do
-    from(s in subquery(q), order_by: s.search_type, distinct: s.id)
-  end
-
   defp maybe_resolve(true, user, query) do
     case {limit(), user} do
       {:all, _} -> :noop
@@ -126,9 +139,9 @@ defmodule Pleroma.User.Search do
 
   defp restrict_local(q), do: where(q, [u], u.local == true)
 
-  defp boost_search_rank_query(query, nil), do: query
+  defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
 
-  defp boost_search_rank_query(query, for_user) do
+  defp boost_search_rank(query, %User{} = for_user) do
     friends_ids = User.get_friends_ids(for_user)
     followers_ids = User.get_followers_ids(for_user)
 
@@ -137,8 +150,8 @@ defmodule Pleroma.User.Search do
         search_rank:
           fragment(
             """
-             CASE WHEN (?) THEN 0.5 + (?) * 1.3
-             WHEN (?) THEN 0.5 + (?) * 1.2
+             CASE WHEN (?) THEN (?) * 1.5
+             WHEN (?) THEN (?) * 1.3
              WHEN (?) THEN (?) * 1.1
              ELSE (?) END
             """,
@@ -154,70 +167,5 @@ defmodule Pleroma.User.Search do
     )
   end
 
-  @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
-  defp fts_search_subquery(query, term) do
-    processed_query =
-      String.trim_trailing(term, "@" <> local_domain())
-      |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
-      |> String.trim()
-      |> String.split()
-      |> Enum.map(&(&1 <> ":*"))
-      |> Enum.join(" | ")
-
-    from(
-      u in query,
-      select_merge: %{
-        search_type: ^0,
-        search_rank:
-          fragment(
-            """
-            ts_rank_cd(
-              setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
-              setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
-              to_tsquery('simple', ?),
-              32
-            )
-            """,
-            u.nickname,
-            u.name,
-            ^processed_query
-          )
-      },
-      where:
-        fragment(
-          """
-            (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
-            setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
-          """,
-          u.nickname,
-          u.name,
-          ^processed_query
-        )
-    )
-    |> User.restrict_deactivated()
-  end
-
-  @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
-  defp trigram_search_subquery(query, term) do
-    term = String.trim_trailing(term, "@" <> local_domain())
-
-    from(
-      u in query,
-      select_merge: %{
-        # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
-        search_type: fragment("?", 1),
-        search_rank:
-          fragment(
-            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
-            ^term,
-            u.nickname,
-            u.name
-          )
-      },
-      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
-    )
-    |> User.restrict_deactivated()
-  end
-
-  defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+  defp boost_search_rank(query, _for_user), do: query
 end
-- 
cgit v1.2.3


From 0a5175ecbb796cf3c192a42dc561debd73640a54 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 16 Oct 2019 13:49:33 +0300
Subject: Order fts results by trigram

---
 lib/pleroma/user/search.ex | 48 +++++++++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 18 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index fb2f3fedb..0d697fe3d 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -7,7 +7,6 @@ defmodule Pleroma.User.Search do
   alias Pleroma.User
   import Ecto.Query
 
-  @similarity_threshold 0.25
   @limit 20
 
   def search(query_string, opts \\ []) do
@@ -47,16 +46,16 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_blocked_domains(for_user)
-    |> fts_subquery(query_string)
-    |> subquery()
-    |> where([u], u.search_rank > @similarity_threshold)
+    |> fts_search(query_string)
+    |> trigram_rank(query_string)
     |> boost_search_rank(for_user)
+    |> subquery()
     |> order_by(desc: :search_rank)
     |> maybe_restrict_local(for_user)
   end
 
   @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/
-  defp fts_subquery(query, query_string) do
+  defp fts_search(query, query_string) do
     {nickname_weight, name_weight} =
       if String.match?(query_string, @nickname_regex) do
         {"A", "B"}
@@ -68,19 +67,17 @@ defmodule Pleroma.User.Search do
 
     from(
       u in query,
-      select_merge: %{
-        search_rank:
-          fragment(
-            """
-            ts_rank_cd((setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)), to_tsquery('simple', ?))
-            """,
-            u.name,
-            ^name_weight,
-            u.nickname,
-            ^nickname_weight,
-            ^query_string
-          )
-      }
+      where:
+        fragment(
+          """
+          (setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)) @@ to_tsquery('simple', ?)
+          """,
+          u.name,
+          ^name_weight,
+          u.nickname,
+          ^nickname_weight,
+          ^query_string
+        )
     )
   end
 
@@ -93,6 +90,21 @@ defmodule Pleroma.User.Search do
     |> Enum.join(" | ")
   end
 
+  defp trigram_rank(query, query_string) do
+    from(
+      u in query,
+      select_merge: %{
+        search_rank:
+          fragment(
+            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
+            ^query_string,
+            u.nickname,
+            u.name
+          )
+      }
+    )
+  end
+
   defp base_query(_user, false), do: User
   defp base_query(user, true), do: User.get_followers_query(user)
 
-- 
cgit v1.2.3


From 359dd1890e6afcf80584021eaa2421336614dd07 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Thu, 17 Oct 2019 15:25:15 +0300
Subject: Mastodon API: Mark the conversation as read for the author when they
 send a new direct message

---
 lib/pleroma/conversation/participation.ex    |  6 ++++++
 lib/pleroma/web/activity_pub/activity_pub.ex | 23 +++++++++++++++++------
 2 files changed, 23 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex
index ab81f3217..e17f49e58 100644
--- a/lib/pleroma/conversation/participation.ex
+++ b/lib/pleroma/conversation/participation.ex
@@ -48,6 +48,12 @@ defmodule Pleroma.Conversation.Participation do
     |> validate_required([:read])
   end
 
+  def mark_as_read(%User{} = user, %Conversation{} = conversation) do
+    with %__MODULE__{} = participation <- for_user_and_conversation(user, conversation) do
+      mark_as_read(participation)
+    end
+  end
+
   def mark_as_read(participation) do
     participation
     |> read_cng(%{read: true})
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 091ec2588..d391732a2 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity.Ir.Topics
   alias Pleroma.Config
   alias Pleroma.Conversation
+  alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -153,11 +154,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
       Notification.create_notifications(activity)
 
-      participations =
-        activity
-        |> Conversation.create_or_bump_for()
-        |> get_participations()
-
+      conversation = create_or_bump_conversation(activity, map["actor"])
+      participations = get_participations(conversation)
       stream_out(activity)
       stream_out_participations(participations)
       {:ok, activity}
@@ -182,7 +180,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  defp get_participations({:ok, %{participations: participations}}), do: participations
+  defp create_or_bump_conversation(activity, actor) do
+    with {:ok, conversation} <- Conversation.create_or_bump_for(activity),
+         %User{} = user <- User.get_cached_by_ap_id(actor),
+         Participation.mark_as_read(user, conversation) do
+      {:ok, conversation}
+    end
+  end
+
+  defp get_participations({:ok, conversation}) do
+    conversation
+    |> Repo.preload(:participations, force: true)
+    |> Map.get(:participations)
+  end
+
   defp get_participations(_), do: []
 
   def stream_out_participations(participations) do
-- 
cgit v1.2.3


From 733b73b71c2a13a2bfbf4c0a0d35ddfa140a1ce0 Mon Sep 17 00:00:00 2001
From: kaniini <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 04:36:37 +0000
Subject: Apply suggestion to lib/pleroma/web/mastodon_api/websocket_handler.ex

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 1a757363f..a400d1c8d 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -36,7 +36,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
          {:ok, user} <- allow_request(stream, [access_token, sec_websocket]),
          topic when is_binary(topic) <- expand_topic(stream, params) do
       req =
-        if sec_websocket != nil do
+        if sec_websocket do
           :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
         else
           req
-- 
cgit v1.2.3