From db690bede92e9ba65b1afaa8605aa5ddb3c992a4 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Fri, 2 Aug 2019 21:33:12 +0300
Subject: temp commit

---
 lib/load_testing/fetcher.ex           | 116 ++++++++++++++++++++++++++++++++++
 lib/load_testing/generator.ex         |  83 ++++++++++++++++++++++++
 lib/load_testing/helper.ex            |  16 +++++
 lib/mix/tasks/pleroma/load_testing.ex | 100 +++++++++++++++++++++++++++++
 4 files changed, 315 insertions(+)
 create mode 100644 lib/load_testing/fetcher.ex
 create mode 100644 lib/load_testing/generator.ex
 create mode 100644 lib/load_testing/helper.ex
 create mode 100644 lib/mix/tasks/pleroma/load_testing.ex

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
new file mode 100644
index 000000000..70c0fcd0c
--- /dev/null
+++ b/lib/load_testing/fetcher.ex
@@ -0,0 +1,116 @@
+defmodule Pleroma.LoadTesting.Fetcher do
+  use Pleroma.LoadTesting.Helper
+
+  def fetch_user(user) do
+    IO.puts("=================================")
+
+    {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end)
+
+    IO.puts("Query user by id: #{to_sec(time)} sec.")
+
+    {time, _value} =
+      :timer.tc(fn ->
+        Repo.get_by(User, ap_id: user.ap_id)
+      end)
+
+    IO.puts("Query user by ap_id: #{to_sec(time)} sec.")
+
+    {time, _value} =
+      :timer.tc(fn ->
+        Repo.get_by(User, email: user.email)
+      end)
+
+    IO.puts("Query user by email: #{to_sec(time)} sec.")
+
+    {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end)
+
+    IO.puts("Query user by nickname: #{to_sec(time)} sec.")
+  end
+
+  def query_timelines(user) do
+    IO.puts("\n=================================")
+
+    params = %{
+      "count" => 20,
+      "with_muted" => true,
+      "type" => ["Create", "Announce"],
+      "blocking_user" => user,
+      "muting_user" => user,
+      "user" => user
+    }
+
+    {time, _} =
+      :timer.tc(fn ->
+        ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params)
+      end)
+
+    IO.puts("Query user home timeline: #{to_sec(time)} sec.")
+
+    params = %{
+      "count" => 20,
+      "local_only" => true,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    {time, _} =
+      :timer.tc(fn ->
+        ActivityPub.ActivityPub.fetch_public_activities(params)
+      end)
+
+    IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.")
+
+    params = %{
+      "count" => 20,
+      "only_media" => "false",
+      "type" => ["Create", "Announce"],
+      "with_muted" => "true",
+      "blocking_user" => user,
+      "muting_user" => user
+    }
+
+    {time, _} =
+      :timer.tc(fn ->
+        ActivityPub.ActivityPub.fetch_public_activities(params)
+      end)
+
+    IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.")
+  end
+
+  def query_notifications(user) do
+    IO.puts("\n=================================")
+    params = %{"count" => "20", "with_muted" => "false"}
+
+    {time, _} =
+      :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end)
+
+    IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.")
+
+    params = %{"count" => "20", "with_muted" => "true"}
+
+    {time, _} =
+      :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end)
+
+    IO.puts("Query user notifications with muted: #{to_sec(time)} sec.")
+  end
+
+  def query_long_thread(user, activity) do
+    IO.puts("\n=================================")
+
+    {time, replies} =
+      :timer.tc(fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+          activity.data["context"],
+          %{
+            "blocking_user" => user,
+            "user" => user
+          }
+        )
+      end)
+
+    IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.")
+  end
+end
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
new file mode 100644
index 000000000..a9016b9e8
--- /dev/null
+++ b/lib/load_testing/generator.ex
@@ -0,0 +1,83 @@
+defmodule Pleroma.LoadTesting.Generator do
+  use Pleroma.LoadTesting.Helper
+
+  def generate_users(opts) do
+    IO.puts("Starting generating #{opts[:users_max]} users...")
+    {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
+    IO.puts("Inserting users take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_users(opts) do
+    min = Keyword.get(opts, :users_min, 1)
+    max = Keyword.get(opts, :users_max)
+
+    query =
+      "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\",
+      \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n"
+
+    users =
+      Task.async_stream(
+        min..max,
+        &generate_user_data(&1),
+        max_concurrency: 10,
+        timeout: 30_000
+      )
+      |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end)
+
+    query = query <> String.replace_trailing(users, ", \n", ";")
+
+    Ecto.Adapters.SQL.query!(Repo, query)
+  end
+
+  defp generate_user_data(i) do
+    user = %User{
+      name: "Test テスト User #{i}",
+      email: "user#{i}@example.com",
+      nickname: "nick#{i}",
+      password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
+      bio: "Tester Number #{i}",
+      info: %{}
+    }
+
+    user = %{
+      user
+      | ap_id: User.ap_id(user),
+        follower_address: User.ap_followers(user),
+        following_address: User.ap_following(user),
+        following: [User.ap_id(user)]
+    }
+
+    "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{
+      user.following
+    }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{
+      user.name
+    }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())"
+  end
+
+  def generate_activities(users, opts) do
+    IO.puts("Starting generating #{opts[:activities_max]} activities...")
+    {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end)
+    IO.puts("Inserting activities take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_activities(users, opts) do
+    Task.async_stream(
+      1..opts[:activities_max],
+      fn _ ->
+        do_generate_activity(users, opts)
+      end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp do_generate_activity(users, opts) do
+    status =
+      if opts[:mention],
+        do: "some status with @#{opts[:mention].nickname}",
+        else: "some status"
+
+    Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status})
+  end
+end
diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex
new file mode 100644
index 000000000..338dba323
--- /dev/null
+++ b/lib/load_testing/helper.ex
@@ -0,0 +1,16 @@
+defmodule Pleroma.LoadTesting.Helper do
+  defmacro __using__(_) do
+    quote do
+      import Ecto.Query
+      alias Pleroma.Activity
+      alias Pleroma.Notification
+      alias Pleroma.Object
+      alias Pleroma.Repo
+      alias Pleroma.User
+      alias Pleroma.Web.ActivityPub
+      alias Pleroma.Web.CommonAPI
+
+      defp to_sec(microseconds), do: microseconds / 1_000_000
+    end
+  end
+end
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
new file mode 100644
index 000000000..9ed30db7e
--- /dev/null
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -0,0 +1,100 @@
+defmodule Mix.Tasks.Pleroma.LoadTesting do
+  use Mix.Task
+  use Pleroma.LoadTesting.Helper
+  import Mix.Pleroma
+  import Pleroma.LoadTesting.Generator
+  import Pleroma.LoadTesting.Fetcher
+
+  # tODO: remove autovacuum worker until generation is not ended
+  @shortdoc "Factory for generation data"
+  @moduledoc """
+  Generates data like:
+  - users
+  - activities with notifications
+
+  ## Generate data
+      MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000
+      MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000
+
+  Options:
+  - `--users NUMBER` - number of users to generate (default: 10000)
+  - `--activities NUMBER` - number of activities to generate (default: 20000)
+  """
+
+  @aliases [u: :users, a: :activities, d: :delete]
+  @switches [users: :integer, activities: :integer, delete: :boolean]
+  @users_default 20_000
+  @activities_default 50_000
+
+  def run(args) do
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
+    start_pleroma()
+
+    current_max = Keyword.get(opts, :users, @users_default)
+    activities_max = Keyword.get(opts, :activities, @activities_default)
+
+    {users_min, users_max} =
+      if opts[:delete] do
+        clean_tables()
+        {1, current_max}
+      else
+        current_count = Repo.aggregate(from(u in User), :count, :id) + 1
+        {current_count, current_max + current_count}
+      end
+
+    opts =
+      Keyword.put(opts, :users_min, users_min)
+      |> Keyword.put(:users_max, users_max)
+      |> Keyword.put(:activities_max, activities_max)
+
+    generate_users(opts)
+
+    # main user for queries
+    IO.puts("Fetching main user...")
+
+    {time, user} =
+      :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end)
+
+    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching users...")
+
+    {time, users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching users take #{to_sec(time)} sec.\n")
+
+    generate_activities(users, opts)
+
+    generate_activities(users, Keyword.put(opts, :mention, user))
+
+    # generate_replies(user, users, activities)
+
+    # activity = Enum.random(activities)
+    # generate_long_thread(user, users, activity)
+
+    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
+    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}")
+    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}")
+    IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}")
+
+    query_timelines(user)
+    query_notifications(user)
+    # query_long_thread(user, activity)
+  end
+
+  defp clean_tables do
+    IO.puts("\n\nDeleting old data...\n")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
+    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
+  end
+end
-- 
cgit v1.2.3


From 79dde5804427ab24c5bc9aa1b5a44d2259cecc6d Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Wed, 4 Sep 2019 20:18:11 +0300
Subject: one more temp commit

---
 lib/load_testing/fetcher.ex                  | 122 ++++++++++++-------------
 lib/load_testing/generator.ex                | 127 ++++++++++++++++++++++-----
 lib/mix/tasks/pleroma/load_testing.ex        |  46 ++++++----
 lib/pleroma/web/activity_pub/activity_pub.ex |   5 ++
 4 files changed, 203 insertions(+), 97 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
index 70c0fcd0c..ed744db9b 100644
--- a/lib/load_testing/fetcher.ex
+++ b/lib/load_testing/fetcher.ex
@@ -4,33 +4,18 @@ defmodule Pleroma.LoadTesting.Fetcher do
   def fetch_user(user) do
     IO.puts("=================================")
 
-    {time, _value} = :timer.tc(fn -> Repo.get_by(User, id: user.id) end)
-
-    IO.puts("Query user by id: #{to_sec(time)} sec.")
-
-    {time, _value} =
-      :timer.tc(fn ->
-        Repo.get_by(User, ap_id: user.ap_id)
-      end)
-
-    IO.puts("Query user by ap_id: #{to_sec(time)} sec.")
-
-    {time, _value} =
-      :timer.tc(fn ->
-        Repo.get_by(User, email: user.email)
-      end)
-
-    IO.puts("Query user by email: #{to_sec(time)} sec.")
-
-    {time, _value} = :timer.tc(fn -> Repo.get_by(User, nickname: user.nickname) end)
-
-    IO.puts("Query user by nickname: #{to_sec(time)} sec.")
+    Benchee.run(%{
+      "By id" => fn -> Repo.get_by(User, id: user.id) end,
+      "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
+      "By email" => fn -> Repo.get_by(User, email: user.email) end,
+      "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
+    })
   end
 
   def query_timelines(user) do
     IO.puts("\n=================================")
 
-    params = %{
+    home_timeline_params = %{
       "count" => 20,
       "with_muted" => true,
       "type" => ["Create", "Announce"],
@@ -39,14 +24,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
       "user" => user
     }
 
-    {time, _} =
-      :timer.tc(fn ->
-        ActivityPub.ActivityPub.fetch_activities([user.ap_id | user.following], params)
-      end)
-
-    IO.puts("Query user home timeline: #{to_sec(time)} sec.")
-
-    params = %{
+    mastodon_public_timeline_params = %{
       "count" => 20,
       "local_only" => true,
       "only_media" => "false",
@@ -56,14 +34,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
       "muting_user" => user
     }
 
-    {time, _} =
-      :timer.tc(fn ->
-        ActivityPub.ActivityPub.fetch_public_activities(params)
-      end)
-
-    IO.puts("Query user mastodon public timeline: #{to_sec(time)} sec.")
-
-    params = %{
+    mastodon_federated_timeline_params = %{
       "count" => 20,
       "only_media" => "false",
       "type" => ["Create", "Announce"],
@@ -72,45 +43,76 @@ defmodule Pleroma.LoadTesting.Fetcher do
       "muting_user" => user
     }
 
-    {time, _} =
-      :timer.tc(fn ->
-        ActivityPub.ActivityPub.fetch_public_activities(params)
-      end)
-
-    IO.puts("Query user mastodon federated public timeline: #{to_sec(time)} sec.")
+    Benchee.run(%{
+      "User home timeline" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+          [user.ap_id | user.following],
+          home_timeline_params
+        )
+      end,
+      "User mastodon public timeline" => fn ->
+        ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+      end,
+      "User mastodon federated public timeline" => fn ->
+        ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params)
+      end
+    })
   end
 
   def query_notifications(user) do
     IO.puts("\n=================================")
-    params = %{"count" => "20", "with_muted" => "false"}
-
-    {time, _} =
-      :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end)
-
-    IO.puts("Query user notifications with out muted: #{to_sec(time)} sec.")
+    without_muted_params = %{"count" => "20", "with_muted" => "false"}
+    with_muted_params = %{"count" => "20", "with_muted" => "true"}
+
+    Benchee.run(%{
+      "Notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+      end,
+      "Notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+      end
+    })
+  end
 
-    params = %{"count" => "20", "with_muted" => "true"}
+  def query_dms(user) do
+    IO.puts("\n=================================")
 
-    {time, _} =
-      :timer.tc(fn -> Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, params) end)
+    params = %{
+      "count" => "20",
+      "with_muted" => "true",
+      "type" => "Create",
+      "blocking_user" => user,
+      "user" => user,
+      visibility: "direct"
+    }
 
-    IO.puts("Query user notifications with muted: #{to_sec(time)} sec.")
+    Benchee.run(%{
+      "Direct messages with muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(params)
+      end,
+      "Direct messages without muted" => fn ->
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+        |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+      end
+    })
   end
 
   def query_long_thread(user, activity) do
     IO.puts("\n=================================")
 
-    {time, replies} =
-      :timer.tc(fn ->
+    Benchee.run(%{
+      "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end,
+      "Fetch context of main post" => fn ->
         Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
           activity.data["context"],
           %{
             "blocking_user" => user,
-            "user" => user
+            "user" => user,
+            "exclude_id" => activity.id
           }
         )
-      end)
-
-    IO.puts("Query long thread with #{length(replies)} replies: #{to_sec(time)} sec.")
+      end
+    })
   end
 end
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index a9016b9e8..7f50ee68e 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -8,25 +8,15 @@ defmodule Pleroma.LoadTesting.Generator do
   end
 
   defp do_generate_users(opts) do
-    min = Keyword.get(opts, :users_min, 1)
     max = Keyword.get(opts, :users_max)
 
-    query =
-      "INSERT INTO \"users\" (\"ap_id\",\"bio\",\"email\",\"follower_address\",\"following\",\"following_address\",\"info\",
-      \"local\",\"name\",\"nickname\",\"password_hash\",\"tags\",\"id\",\"inserted_at\",\"updated_at\") VALUES \n"
-
-    users =
-      Task.async_stream(
-        min..max,
-        &generate_user_data(&1),
-        max_concurrency: 10,
-        timeout: 30_000
-      )
-      |> Enum.reduce("", fn {:ok, data}, acc -> acc <> data <> ", \n" end)
-
-    query = query <> String.replace_trailing(users, ", \n", ";")
-
-    Ecto.Adapters.SQL.query!(Repo, query)
+    Task.async_stream(
+      1..max,
+      &generate_user_data(&1),
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Enum.to_list()
   end
 
   defp generate_user_data(i) do
@@ -47,11 +37,7 @@ defmodule Pleroma.LoadTesting.Generator do
         following: [User.ap_id(user)]
     }
 
-    "('#{user.ap_id}', '#{user.bio}', '#{user.email}', '#{user.follower_address}', '{#{
-      user.following
-    }}', '#{user.following_address}', '#{Jason.encode!(user.info)}', '#{user.local}', '#{
-      user.name
-    }', '#{user.nickname}', '#{user.password_hash}', '{#{user.tags}}', uuid_generate_v4(), NOW(), NOW())"
+    Pleroma.Repo.insert!(user)
   end
 
   def generate_activities(users, opts) do
@@ -80,4 +66,101 @@ defmodule Pleroma.LoadTesting.Generator do
 
     Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status})
   end
+
+  def generate_dms(user, users, opts) do
+    IO.puts("Starting generating #{opts[:dms_max]} DMs")
+    {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
+    IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_dms(user, users, opts) do
+    Task.async_stream(
+      1..opts[:dms_max],
+      fn _ ->
+        do_generate_dm(user, users)
+      end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+  end
+
+  defp do_generate_dm(user, users) do
+    post = %{
+      "status" => "@#{user.nickname} some direct message",
+      "visibility" => "direct"
+    }
+
+    Pleroma.Web.CommonAPI.post(Enum.random(users), post)
+  end
+
+  def generate_long_thread(user, users, opts) do
+    IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies")
+    {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
+    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
+    {:ok, activity}
+  end
+
+  defp do_generate_long_thread(user, users, opts) do
+    {:ok, %{id: id} = activity} =
+      Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"})
+
+    Task.async_stream(
+      1..opts[:long_thread_length],
+      fn _ -> do_generate_thread(users, id) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+    |> Stream.run()
+
+    activity
+  end
+
+  defp do_generate_thread(users, activity_id) do
+    Pleroma.Web.CommonAPI.post(Enum.random(users), %{
+      "status" => "reply to main post",
+      "in_reply_to_status_id" => activity_id
+    })
+  end
+
+  def generate_private_thread(users, opts) do
+    IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies")
+    {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end)
+    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_non_visible_posts(users, opts) do
+    [user1, user2] = Enum.take(users, 2)
+    {:ok, user1} = Pleroma.User.follow(user1, user2)
+    {:ok, user2} = Pleroma.User.follow(user2, user1)
+
+    {:ok, activity} =
+      Pleroma.Web.CommonAPI.post(user1, %{
+        "status" => "Some private post",
+        "visibility" => "private"
+      })
+
+    {:ok, activity_public} =
+      Pleroma.Web.CommonAPI.post(user2, %{
+        "status" => "Some public reply",
+        "in_reply_to_status_id" => activity.id
+      })
+
+    Task.async_stream(
+      1..opts[:non_visible_posts_max],
+      fn _ -> do_generate_non_visible_post(users, activity_public) end,
+      max_concurrency: 10,
+      timeout: 30_000
+    )
+  end
+
+  defp do_generate_non_visible_post(users, activity) do
+    visibility = Enum.random(["private", "public"])
+
+    Pleroma.Web.CommonAPI.post(Enum.random(users), %{
+      "visibility" => visibility,
+      "status" => "Some #{visibility} reply",
+      "in_reply_to_status_id" => activity.id
+    })
+  end
 end
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
index 9ed30db7e..d17cf0b3e 100644
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -21,31 +21,38 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
   - `--activities NUMBER` - number of activities to generate (default: 20000)
   """
 
-  @aliases [u: :users, a: :activities, d: :delete]
-  @switches [users: :integer, activities: :integer, delete: :boolean]
+  @aliases [u: :users, a: :activities]
+  @switches [
+    users: :integer,
+    activities: :integer,
+    dms: :integer,
+    thread_length: :integer,
+    non_visible_posts: :integer
+  ]
   @users_default 20_000
   @activities_default 50_000
+  @dms_default 50_000
+  @thread_length_default 2_000
+  @non_visible_posts_default 2_000
 
   def run(args) do
-    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
     start_pleroma()
+    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
-    current_max = Keyword.get(opts, :users, @users_default)
+    users_max = Keyword.get(opts, :users, @users_default)
     activities_max = Keyword.get(opts, :activities, @activities_default)
+    dms_max = Keyword.get(opts, :dms, @dms_default)
+    long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
+    non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default)
 
-    {users_min, users_max} =
-      if opts[:delete] do
-        clean_tables()
-        {1, current_max}
-      else
-        current_count = Repo.aggregate(from(u in User), :count, :id) + 1
-        {current_count, current_max + current_count}
-      end
+    clean_tables()
 
     opts =
-      Keyword.put(opts, :users_min, users_min)
-      |> Keyword.put(:users_max, users_max)
+      Keyword.put(opts, :users_max, users_max)
       |> Keyword.put(:activities_max, activities_max)
+      |> Keyword.put(:dms_max, dms_max)
+      |> Keyword.put(:long_thread_length, long_thread_length)
+      |> Keyword.put(:non_visible_posts_max, non_visible_posts)
 
     generate_users(opts)
 
@@ -76,6 +83,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
 
     generate_activities(users, Keyword.put(opts, :mention, user))
 
+    generate_dms(user, users, opts)
+
+    {:ok, activity} = generate_long_thread(user, users, opts)
+
+    generate_private_thread(users, opts)
+
     # generate_replies(user, users, activities)
 
     # activity = Enum.random(activities)
@@ -86,9 +99,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
     IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}")
     IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}")
 
+    fetch_user(user)
     query_timelines(user)
     query_notifications(user)
-    # query_long_thread(user, activity)
+    query_dms(user)
+    query_long_thread(user, activity)
+    query_timelines(user)
   end
 
   defp clean_tables do
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index eeb826814..f2b322314 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -249,6 +249,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     # only accept false as false value
     local = !(params[:local] == false)
     published = params[:published]
+    quick_insert? = Pleroma.Config.get([:env]) == :benchmark
 
     with create_data <-
            make_create_data(
@@ -259,12 +260,16 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:fake, false, activity} <- {:fake, fake, activity},
          _ <- increase_replies_count_if_reply(create_data),
          _ <- increase_poll_votes_if_vote(create_data),
+         {:quick_insert, false, activity} <- {:quick_insert, quick_insert?, activity},
          # Changing note count prior to enqueuing federation task in order to avoid
          # race conditions on updating user.info
          {:ok, _actor} <- increase_note_count_if_public(actor, activity),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
     else
+      {:quick_insert, true, activity} ->
+        {:ok, activity}
+
       {:fake, true, activity} ->
         {:ok, activity}
 
-- 
cgit v1.2.3


From a1125bd5644a5819d1cfbbc8f6b3bb2dadbbc38a Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 5 Sep 2019 16:01:52 +0300
Subject: generatoin and fetching

---
 lib/load_testing/fetcher.ex           | 126 ++++++++++++++++++++-
 lib/load_testing/generator.ex         | 203 ++++++++++++++++++++++++----------
 lib/load_testing/helper.ex            |   5 -
 lib/mix/tasks/pleroma/load_testing.ex |  47 ++++----
 4 files changed, 289 insertions(+), 92 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
index ed744db9b..825f921e6 100644
--- a/lib/load_testing/fetcher.ex
+++ b/lib/load_testing/fetcher.ex
@@ -51,10 +51,52 @@ defmodule Pleroma.LoadTesting.Fetcher do
         )
       end,
       "User mastodon public timeline" => fn ->
-        ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_public_timeline_params
+        )
       end,
       "User mastodon federated public timeline" => fn ->
-        ActivityPub.ActivityPub.fetch_public_activities(mastodon_federated_timeline_params)
+        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+          mastodon_federated_timeline_params
+        )
+      end
+    })
+
+    home_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
+        [user.ap_id | user.following],
+        home_timeline_params
+      )
+
+    public_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
+
+    public_federated_activities =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
+        mastodon_federated_timeline_params
+      )
+
+    Benchee.run(%{
+      "Rendering home timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: home_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_activities,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering public federated timeline" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: public_federated_activities,
+          for: user,
+          as: :activity
+        })
       end
     })
   end
@@ -72,6 +114,27 @@ defmodule Pleroma.LoadTesting.Fetcher do
         Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
       end
     })
+
+    without_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
+
+    with_muted_notifications =
+      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
+
+    Benchee.run(%{
+      "Render notifications without muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: without_muted_notifications,
+          for: user
+        })
+      end,
+      "Render notifications with muted" => fn ->
+        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
+          notifications: with_muted_notifications,
+          for: user
+        })
+      end
+    })
   end
 
   def query_dms(user) do
@@ -96,13 +159,40 @@ defmodule Pleroma.LoadTesting.Fetcher do
         |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
       end
     })
+
+    dms_with_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(params)
+
+    dms_without_muted =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
+      |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
+
+    Benchee.run(%{
+      "Rendering dms with muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_with_muted,
+          for: user,
+          as: :activity
+        })
+      end,
+      "Rendering dms without muted" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
+          activities: dms_without_muted,
+          for: user,
+          as: :activity
+        })
+      end
+    })
   end
 
   def query_long_thread(user, activity) do
     IO.puts("\n=================================")
 
     Benchee.run(%{
-      "Fetch main post" => fn -> Activity.get_by_id_with_object(activity.id) end,
+      "Fetch main post" => fn ->
+        Pleroma.Activity.get_by_id_with_object(activity.id)
+      end,
       "Fetch context of main post" => fn ->
         Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
           activity.data["context"],
@@ -114,5 +204,35 @@ defmodule Pleroma.LoadTesting.Fetcher do
         )
       end
     })
+
+    activity = Pleroma.Activity.get_by_id_with_object(activity.id)
+
+    context =
+      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
+        activity.data["context"],
+        %{
+          "blocking_user" => user,
+          "user" => user,
+          "exclude_id" => activity.id
+        }
+      )
+
+    Benchee.run(%{
+      "Render status" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{
+          activity: activity,
+          for: user
+        })
+      end,
+      "Render context ancestors" => fn ->
+        Pleroma.Web.MastodonAPI.StatusView.render(
+          "index.json",
+          for: user,
+          activities: context,
+          as: :activity
+        )
+        |> Enum.reverse()
+      end
+    })
   end
 end
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index 7f50ee68e..6df518aba 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -1,9 +1,11 @@
 defmodule Pleroma.LoadTesting.Generator do
   use Pleroma.LoadTesting.Helper
+  alias Pleroma.Web.CommonAPI
 
   def generate_users(opts) do
     IO.puts("Starting generating #{opts[:users_max]} users...")
     {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
+
     IO.puts("Inserting users take #{to_sec(time)} sec.\n")
   end
 
@@ -37,34 +39,111 @@ defmodule Pleroma.LoadTesting.Generator do
         following: [User.ap_id(user)]
     }
 
-    Pleroma.Repo.insert!(user)
+    Repo.insert!(user)
   end
 
-  def generate_activities(users, opts) do
-    IO.puts("Starting generating #{opts[:activities_max]} activities...")
-    {time, _} = :timer.tc(fn -> do_generate_activities(users, opts) end)
-    IO.puts("Inserting activities take #{to_sec(time)} sec.\n")
+  def generate_activities(user, users) do
+    do_generate_activities(user, users)
   end
 
-  defp do_generate_activities(users, opts) do
-    Task.async_stream(
-      1..opts[:activities_max],
-      fn _ ->
-        do_generate_activity(users, opts)
-      end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
+  defp do_generate_activities(user, users) do
+    IO.puts("Starting generating 20000 common activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 20000 activities with mentions...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..20_000,
+          fn _ ->
+            do_generate_activity_with_mention(user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
+
+    IO.puts("Starting generating 10000 activities with threads...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn _ ->
+            do_generate_threads([user | users])
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_activity(users) do
+    post = %{
+      "status" => "Some status without mention with random user"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
+  end
+
+  defp do_generate_activity_with_mention(user, users) do
+    mentions_cnt = Enum.random([2, 3, 4, 5])
+    with_user = Enum.random([true, false])
+    users = Enum.shuffle(users)
+    mentions_users = Enum.take(users, mentions_cnt)
+    mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
+
+    mentions_str =
+      Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
+
+    post = %{
+      "status" => mentions_str <> "some status with mentions random users"
+    }
+
+    CommonAPI.post(Enum.random(users), post)
   end
 
-  defp do_generate_activity(users, opts) do
-    status =
-      if opts[:mention],
-        do: "some status with @#{opts[:mention].nickname}",
-        else: "some status"
+  defp do_generate_threads(users) do
+    thread_length = Enum.random([2, 3, 4, 5])
+    actor = Enum.random(users)
+
+    post = %{
+      "status" => "Start of the thread"
+    }
+
+    {:ok, activity} = CommonAPI.post(actor, post)
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{actor.nickname} reply to thread",
+        "in_reply_to_status_id" => activity.id
+      }
 
-    Pleroma.Web.CommonAPI.post(Enum.random(users), %{"status" => status})
+      CommonAPI.post(user, post)
+    end)
   end
 
   def generate_dms(user, users, opts) do
@@ -91,22 +170,21 @@ defmodule Pleroma.LoadTesting.Generator do
       "visibility" => "direct"
     }
 
-    Pleroma.Web.CommonAPI.post(Enum.random(users), post)
+    CommonAPI.post(Enum.random(users), post)
   end
 
   def generate_long_thread(user, users, opts) do
-    IO.puts("Starting generating long thread with #{opts[:long_thread_length]} replies")
+    IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
     {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
     IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
     {:ok, activity}
   end
 
   defp do_generate_long_thread(user, users, opts) do
-    {:ok, %{id: id} = activity} =
-      Pleroma.Web.CommonAPI.post(user, %{"status" => "Start of long thread"})
+    {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
 
     Task.async_stream(
-      1..opts[:long_thread_length],
+      1..opts[:thread_length],
       fn _ -> do_generate_thread(users, id) end,
       max_concurrency: 10,
       timeout: 30_000
@@ -117,50 +195,63 @@ defmodule Pleroma.LoadTesting.Generator do
   end
 
   defp do_generate_thread(users, activity_id) do
-    Pleroma.Web.CommonAPI.post(Enum.random(users), %{
+    CommonAPI.post(Enum.random(users), %{
       "status" => "reply to main post",
       "in_reply_to_status_id" => activity_id
     })
   end
 
-  def generate_private_thread(users, opts) do
-    IO.puts("Starting generating long thread with #{opts[:non_visible_posts_max]} replies")
-    {time, _} = :timer.tc(fn -> do_generate_non_visible_posts(users, opts) end)
-    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
-  end
+  def generate_non_visible_message(user, users) do
+    IO.puts("Starting generating 1000 non visible posts")
 
-  defp do_generate_non_visible_posts(users, opts) do
-    [user1, user2] = Enum.take(users, 2)
-    {:ok, user1} = Pleroma.User.follow(user1, user2)
-    {:ok, user2} = Pleroma.User.follow(user2, user1)
+    {time, _} =
+      :timer.tc(fn ->
+        do_generate_non_visible_posts(user, users)
+      end)
 
-    {:ok, activity} =
-      Pleroma.Web.CommonAPI.post(user1, %{
-        "status" => "Some private post",
-        "visibility" => "private"
-      })
+    IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
+  end
 
-    {:ok, activity_public} =
-      Pleroma.Web.CommonAPI.post(user2, %{
-        "status" => "Some public reply",
-        "in_reply_to_status_id" => activity.id
-      })
+  defp do_generate_non_visible_posts(user, users) do
+    [not_friend | users] = users
 
-    Task.async_stream(
-      1..opts[:non_visible_posts_max],
-      fn _ -> do_generate_non_visible_post(users, activity_public) end,
+    make_friends(user, users)
+
+    Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
       max_concurrency: 10,
       timeout: 30_000
     )
+    |> Stream.run()
   end
 
-  defp do_generate_non_visible_post(users, activity) do
-    visibility = Enum.random(["private", "public"])
+  defp make_friends(_user, []), do: nil
 
-    Pleroma.Web.CommonAPI.post(Enum.random(users), %{
-      "visibility" => visibility,
-      "status" => "Some #{visibility} reply",
-      "in_reply_to_status_id" => activity.id
-    })
+  defp make_friends(user, [friend | users]) do
+    {:ok, _} = User.follow(user, friend)
+    {:ok, _} = User.follow(friend, user)
+    make_friends(user, users)
+  end
+
+  defp do_generate_non_visible_post(not_friend, users) do
+    post = %{
+      "status" => "some non visible post",
+      "visibility" => "private"
+    }
+
+    {:ok, activity} = CommonAPI.post(not_friend, post)
+
+    thread_length = Enum.random([2, 3, 4, 5])
+
+    Enum.each(1..thread_length, fn _ ->
+      user = Enum.random(users)
+
+      post = %{
+        "status" => "@#{not_friend.nickname} reply to non visible post",
+        "in_reply_to_status_id" => activity.id,
+        "visibility" => "private"
+      }
+
+      CommonAPI.post(user, post)
+    end)
   end
 end
diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex
index 338dba323..47b25c65f 100644
--- a/lib/load_testing/helper.ex
+++ b/lib/load_testing/helper.ex
@@ -2,13 +2,8 @@ defmodule Pleroma.LoadTesting.Helper do
   defmacro __using__(_) do
     quote do
       import Ecto.Query
-      alias Pleroma.Activity
-      alias Pleroma.Notification
-      alias Pleroma.Object
       alias Pleroma.Repo
       alias Pleroma.User
-      alias Pleroma.Web.ActivityPub
-      alias Pleroma.Web.CommonAPI
 
       defp to_sec(microseconds), do: microseconds / 1_000_000
     end
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
index d17cf0b3e..83e531abf 100644
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -13,46 +13,37 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
   - activities with notifications
 
   ## Generate data
-      MIX_ENV=test mix pleroma.load_testing --users 10000 --activities 20000
-      MIX_ENV=test mix pleroma.load_testing -u 10000 -a 20000
+      MIX_ENV=benchmark mix pleroma.load_testing --users 10000
+      MIX_ENV=benchmark mix pleroma.load_testing -u 10000
 
   Options:
   - `--users NUMBER` - number of users to generate (default: 10000)
-  - `--activities NUMBER` - number of activities to generate (default: 20000)
   """
 
-  @aliases [u: :users, a: :activities]
+  @aliases [u: :users, d: :dms, t: :thread_length]
   @switches [
     users: :integer,
-    activities: :integer,
     dms: :integer,
-    thread_length: :integer,
-    non_visible_posts: :integer
+    thread_length: :integer
   ]
   @users_default 20_000
-  @activities_default 50_000
-  @dms_default 50_000
+  @dms_default 20_000
   @thread_length_default 2_000
-  @non_visible_posts_default 2_000
 
   def run(args) do
     start_pleroma()
     {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
     users_max = Keyword.get(opts, :users, @users_default)
-    activities_max = Keyword.get(opts, :activities, @activities_default)
     dms_max = Keyword.get(opts, :dms, @dms_default)
-    long_thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
-    non_visible_posts = Keyword.get(opts, :non_visible_posts, @non_visible_posts_default)
+    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
 
     clean_tables()
 
     opts =
       Keyword.put(opts, :users_max, users_max)
-      |> Keyword.put(:activities_max, activities_max)
       |> Keyword.put(:dms_max, dms_max)
-      |> Keyword.put(:long_thread_length, long_thread_length)
-      |> Keyword.put(:non_visible_posts_max, non_visible_posts)
+      |> Keyword.put(:thread_length, thread_length)
 
     generate_users(opts)
 
@@ -60,7 +51,9 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
     IO.puts("Fetching main user...")
 
     {time, user} =
-      :timer.tc(fn -> Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1)) end)
+      :timer.tc(fn ->
+        Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1))
+      end)
 
     IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
 
@@ -79,25 +72,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
 
     IO.puts("Fetching users take #{to_sec(time)} sec.\n")
 
-    generate_activities(users, opts)
-
-    generate_activities(users, Keyword.put(opts, :mention, user))
+    generate_activities(user, users)
 
     generate_dms(user, users, opts)
 
     {:ok, activity} = generate_long_thread(user, users, opts)
 
-    generate_private_thread(users, opts)
+    generate_non_visible_message(user, users)
 
-    # generate_replies(user, users, activities)
+    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
 
-    # activity = Enum.random(activities)
-    # generate_long_thread(user, users, activity)
+    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}")
 
-    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
-    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Activity), :count, :id)}")
-    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Object), :count, :id)}")
-    IO.puts("Notifications in DB: #{Repo.aggregate(from(n in Notification), :count, :id)}")
+    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}")
+
+    IO.puts(
+      "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
+    )
 
     fetch_user(user)
     query_timelines(user)
-- 
cgit v1.2.3


From 252e5db45c5d9d63116e5a24cc35c26f91ec7de4 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Fri, 6 Sep 2019 16:37:18 +0300
Subject: docs fixes

---
 lib/load_testing/fetcher.ex           |  9 ---------
 lib/load_testing/generator.ex         |  3 ++-
 lib/mix/tasks/pleroma/load_testing.ex | 16 +++++++++++-----
 3 files changed, 13 insertions(+), 15 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
index 825f921e6..b92b6e04f 100644
--- a/lib/load_testing/fetcher.ex
+++ b/lib/load_testing/fetcher.ex
@@ -2,8 +2,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   use Pleroma.LoadTesting.Helper
 
   def fetch_user(user) do
-    IO.puts("=================================")
-
     Benchee.run(%{
       "By id" => fn -> Repo.get_by(User, id: user.id) end,
       "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
@@ -13,8 +11,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   def query_timelines(user) do
-    IO.puts("\n=================================")
-
     home_timeline_params = %{
       "count" => 20,
       "with_muted" => true,
@@ -102,7 +98,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   def query_notifications(user) do
-    IO.puts("\n=================================")
     without_muted_params = %{"count" => "20", "with_muted" => "false"}
     with_muted_params = %{"count" => "20", "with_muted" => "true"}
 
@@ -138,8 +133,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   def query_dms(user) do
-    IO.puts("\n=================================")
-
     params = %{
       "count" => "20",
       "with_muted" => "true",
@@ -187,8 +180,6 @@ defmodule Pleroma.LoadTesting.Fetcher do
   end
 
   def query_long_thread(user, activity) do
-    IO.puts("\n=================================")
-
     Benchee.run(%{
       "Fetch main post" => fn ->
         Pleroma.Activity.get_by_id_with_object(activity.id)
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index 6df518aba..7ad68dcc1 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -28,7 +28,8 @@ defmodule Pleroma.LoadTesting.Generator do
       nickname: "nick#{i}",
       password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
       bio: "Tester Number #{i}",
-      info: %{}
+      info: %{},
+      local: Enum.random([true, false])
     }
 
     user = %{
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
index 83e531abf..4fed6bf74 100644
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -5,19 +5,23 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
   import Pleroma.LoadTesting.Generator
   import Pleroma.LoadTesting.Fetcher
 
-  # tODO: remove autovacuum worker until generation is not ended
   @shortdoc "Factory for generation data"
   @moduledoc """
   Generates data like:
   - users
   - activities with notifications
+  - direct messages
+  - long thread
+  - non visible posts
 
   ## Generate data
-      MIX_ENV=benchmark mix pleroma.load_testing --users 10000
-      MIX_ENV=benchmark mix pleroma.load_testing -u 10000
+      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
+      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
 
   Options:
-  - `--users NUMBER` - number of users to generate (default: 10000)
+  - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
+  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
+  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
   """
 
   @aliases [u: :users, d: :dms, t: :thread_length]
@@ -32,6 +36,7 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
 
   def run(args) do
     start_pleroma()
+    Pleroma.Config.put([:instance, :skip_thread_containment], true)
     {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
 
     users_max = Keyword.get(opts, :users, @users_default)
@@ -95,11 +100,12 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
     query_notifications(user)
     query_dms(user)
     query_long_thread(user, activity)
+    Pleroma.Config.put([:instance, :skip_thread_containment], false)
     query_timelines(user)
   end
 
   defp clean_tables do
-    IO.puts("\n\nDeleting old data...\n")
+    IO.puts("Deleting old data...\n")
     Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
     Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
     Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
-- 
cgit v1.2.3


From b3f6f6a4091c048813e367f6a9414eba6fce5109 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Fri, 6 Sep 2019 20:14:02 +0300
Subject: generating remote users

---
 lib/load_testing/fetcher.ex   |  2 +-
 lib/load_testing/generator.ex | 34 ++++++++++++++++++++++++++--------
 lib/pleroma/user.ex           |  4 +++-
 3 files changed, 30 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
index b92b6e04f..0ff2f28c0 100644
--- a/lib/load_testing/fetcher.ex
+++ b/lib/load_testing/fetcher.ex
@@ -215,7 +215,7 @@ defmodule Pleroma.LoadTesting.Fetcher do
           for: user
         })
       end,
-      "Render context ancestors" => fn ->
+      "Render context" => fn ->
         Pleroma.Web.MastodonAPI.StatusView.render(
           "index.json",
           for: user,
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index 7ad68dcc1..61e3d8686 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -22,6 +22,8 @@ defmodule Pleroma.LoadTesting.Generator do
   end
 
   defp generate_user_data(i) do
+    remote = Enum.random([true, false])
+
     user = %User{
       name: "Test テスト User #{i}",
       email: "user#{i}@example.com",
@@ -29,16 +31,32 @@ defmodule Pleroma.LoadTesting.Generator do
       password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
       bio: "Tester Number #{i}",
       info: %{},
-      local: Enum.random([true, false])
+      local: remote
     }
 
-    user = %{
-      user
-      | ap_id: User.ap_id(user),
-        follower_address: User.ap_followers(user),
-        following_address: User.ap_following(user),
-        following: [User.ap_id(user)]
-    }
+    user_urls =
+      if remote do
+        base_url =
+          Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
+
+        ap_id = "#{base_url}/users/#{user.nickname}"
+
+        %{
+          ap_id: ap_id,
+          follower_address: ap_id <> "/followers",
+          following_address: ap_id <> "/following",
+          following: [ap_id]
+        }
+      else
+        %{
+          ap_id: User.ap_id(user),
+          follower_address: User.ap_followers(user),
+          following_address: User.ap_following(user),
+          following: [User.ap_id(user)]
+        }
+      end
+
+    user = Map.merge(user, user_urls)
 
     Repo.insert!(user)
   end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 3aa245f2a..37e59a651 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -429,7 +429,9 @@ defmodule Pleroma.User do
         {:error, "Could not follow user: #{followed.nickname} blocked you."}
 
       true ->
-        if !followed.local && follower.local && !ap_enabled?(followed) do
+        benchmark? = Pleroma.Config.get([:env]) == :benchmark
+
+        if !followed.local && follower.local && !ap_enabled?(followed) && !benchmark? do
           Websub.subscribe(follower, followed)
         end
 
-- 
cgit v1.2.3


From 924d7e6aa60095e27c15b4a5f473a192ab2b42ef Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 19 Sep 2019 12:59:36 +0300
Subject: generating remote activities

---
 lib/load_testing/generator.ex         | 78 ++++++++++++++++++++++++++++++++++-
 lib/mix/tasks/pleroma/load_testing.ex | 29 +++++++++++--
 2 files changed, 102 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
index 61e3d8686..5c5a5c122 100644
--- a/lib/load_testing/generator.ex
+++ b/lib/load_testing/generator.ex
@@ -28,7 +28,8 @@ defmodule Pleroma.LoadTesting.Generator do
       name: "Test テスト User #{i}",
       email: "user#{i}@example.com",
       nickname: "nick#{i}",
-      password_hash: Comeonin.Pbkdf2.hashpwsalt("test"),
+      password_hash:
+        "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
       bio: "Tester Number #{i}",
       info: %{},
       local: remote
@@ -165,6 +166,81 @@ defmodule Pleroma.LoadTesting.Generator do
     end)
   end
 
+  def generate_remote_activities(user, users) do
+    do_generate_remote_activities(user, users)
+  end
+
+  defp do_generate_remote_activities(user, users) do
+    IO.puts("Starting generating 10000 remote activities...")
+
+    {time, _} =
+      :timer.tc(fn ->
+        Task.async_stream(
+          1..10_000,
+          fn i ->
+            do_generate_remote_activity(i, user, users)
+          end,
+          max_concurrency: 10,
+          timeout: 30_000
+        )
+        |> Stream.run()
+      end)
+
+    IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
+  end
+
+  defp do_generate_remote_activity(i, user, users) do
+    actor = Enum.random(users)
+    %{host: host} = URI.parse(actor.ap_id)
+    date = Date.utc_today()
+    datetime = DateTime.utc_now()
+
+    map = %{
+      "actor" => actor.ap_id,
+      "cc" => [actor.follower_address, user.ap_id],
+      "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+      "id" => actor.ap_id <> "/statuses/#{i}/activity",
+      "object" => %{
+        "actor" => actor.ap_id,
+        "atomUri" => actor.ap_id <> "/statuses/#{i}",
+        "attachment" => [],
+        "attributedTo" => actor.ap_id,
+        "bcc" => [],
+        "bto" => [],
+        "cc" => [actor.follower_address, user.ap_id],
+        "content" =>
+          "<p><span class=\"h-card\"><a href=\"" <>
+            user.ap_id <>
+            "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
+        "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "conversation" =>
+          "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
+        "emoji" => %{},
+        "id" => actor.ap_id <> "/statuses/#{i}",
+        "inReplyTo" => nil,
+        "inReplyToAtomUri" => nil,
+        "published" => datetime,
+        "sensitive" => true,
+        "summary" => "cw",
+        "tag" => [
+          %{
+            "href" => user.ap_id,
+            "name" => "@#{user.nickname}@#{host}",
+            "type" => "Mention"
+          }
+        ],
+        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+        "type" => "Note",
+        "url" => "http://#{host}/@#{actor.nickname}/#{i}"
+      },
+      "published" => datetime,
+      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
+      "type" => "Create"
+    }
+
+    Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
+  end
+
   def generate_dms(user, users, opts) do
     IO.puts("Starting generating #{opts[:dms_max]} DMs")
     {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
index 4fed6bf74..a7057395d 100644
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ b/lib/mix/tasks/pleroma/load_testing.ex
@@ -53,32 +53,53 @@ defmodule Mix.Tasks.Pleroma.LoadTesting do
     generate_users(opts)
 
     # main user for queries
-    IO.puts("Fetching main user...")
+    IO.puts("Fetching local main user...")
 
     {time, user} =
       :timer.tc(fn ->
-        Repo.one(from(u in User, order_by: fragment("RANDOM()"), limit: 1))
+        Repo.one(
+          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
+        )
       end)
 
     IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
 
-    IO.puts("Fetching users...")
+    IO.puts("Fetching local users...")
 
     {time, users} =
       :timer.tc(fn ->
         Repo.all(
           from(u in User,
             where: u.id != ^user.id,
+            where: u.local == true,
+            order_by: fragment("RANDOM()"),
+            limit: 10
+          )
+        )
+      end)
+
+    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
+
+    IO.puts("Fetching remote users...")
+
+    {time, remote_users} =
+      :timer.tc(fn ->
+        Repo.all(
+          from(u in User,
+            where: u.id != ^user.id,
+            where: u.local == false,
             order_by: fragment("RANDOM()"),
             limit: 10
           )
         )
       end)
 
-    IO.puts("Fetching users take #{to_sec(time)} sec.\n")
+    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
 
     generate_activities(user, users)
 
+    generate_remote_activities(user, remote_users)
+
     generate_dms(user, users, opts)
 
     {:ok, activity} = generate_long_thread(user, users, opts)
-- 
cgit v1.2.3


From 1d285e6fada841b6959b6516aca1c84d1a2c9b93 Mon Sep 17 00:00:00 2001
From: Alex S <alex.strizhakov@gmail.com>
Date: Thu, 19 Sep 2019 14:02:27 +0300
Subject: moving to separate dir

---
 lib/load_testing/fetcher.ex           | 229 ----------------------
 lib/load_testing/generator.ex         | 352 ----------------------------------
 lib/load_testing/helper.ex            |  11 --
 lib/mix/tasks/pleroma/load_testing.ex | 134 -------------
 4 files changed, 726 deletions(-)
 delete mode 100644 lib/load_testing/fetcher.ex
 delete mode 100644 lib/load_testing/generator.ex
 delete mode 100644 lib/load_testing/helper.ex
 delete mode 100644 lib/mix/tasks/pleroma/load_testing.ex

(limited to 'lib')

diff --git a/lib/load_testing/fetcher.ex b/lib/load_testing/fetcher.ex
deleted file mode 100644
index 0ff2f28c0..000000000
--- a/lib/load_testing/fetcher.ex
+++ /dev/null
@@ -1,229 +0,0 @@
-defmodule Pleroma.LoadTesting.Fetcher do
-  use Pleroma.LoadTesting.Helper
-
-  def fetch_user(user) do
-    Benchee.run(%{
-      "By id" => fn -> Repo.get_by(User, id: user.id) end,
-      "By ap_id" => fn -> Repo.get_by(User, ap_id: user.ap_id) end,
-      "By email" => fn -> Repo.get_by(User, email: user.email) end,
-      "By nickname" => fn -> Repo.get_by(User, nickname: user.nickname) end
-    })
-  end
-
-  def query_timelines(user) do
-    home_timeline_params = %{
-      "count" => 20,
-      "with_muted" => true,
-      "type" => ["Create", "Announce"],
-      "blocking_user" => user,
-      "muting_user" => user,
-      "user" => user
-    }
-
-    mastodon_public_timeline_params = %{
-      "count" => 20,
-      "local_only" => true,
-      "only_media" => "false",
-      "type" => ["Create", "Announce"],
-      "with_muted" => "true",
-      "blocking_user" => user,
-      "muting_user" => user
-    }
-
-    mastodon_federated_timeline_params = %{
-      "count" => 20,
-      "only_media" => "false",
-      "type" => ["Create", "Announce"],
-      "with_muted" => "true",
-      "blocking_user" => user,
-      "muting_user" => user
-    }
-
-    Benchee.run(%{
-      "User home timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
-          [user.ap_id | user.following],
-          home_timeline_params
-        )
-      end,
-      "User mastodon public timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-          mastodon_public_timeline_params
-        )
-      end,
-      "User mastodon federated public timeline" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-          mastodon_federated_timeline_params
-        )
-      end
-    })
-
-    home_activities =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities(
-        [user.ap_id | user.following],
-        home_timeline_params
-      )
-
-    public_activities =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(mastodon_public_timeline_params)
-
-    public_federated_activities =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities(
-        mastodon_federated_timeline_params
-      )
-
-    Benchee.run(%{
-      "Rendering home timeline" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: home_activities,
-          for: user,
-          as: :activity
-        })
-      end,
-      "Rendering public timeline" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: public_activities,
-          for: user,
-          as: :activity
-        })
-      end,
-      "Rendering public federated timeline" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: public_federated_activities,
-          for: user,
-          as: :activity
-        })
-      end
-    })
-  end
-
-  def query_notifications(user) do
-    without_muted_params = %{"count" => "20", "with_muted" => "false"}
-    with_muted_params = %{"count" => "20", "with_muted" => "true"}
-
-    Benchee.run(%{
-      "Notifications without muted" => fn ->
-        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
-      end,
-      "Notifications with muted" => fn ->
-        Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
-      end
-    })
-
-    without_muted_notifications =
-      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, without_muted_params)
-
-    with_muted_notifications =
-      Pleroma.Web.MastodonAPI.MastodonAPI.get_notifications(user, with_muted_params)
-
-    Benchee.run(%{
-      "Render notifications without muted" => fn ->
-        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
-          notifications: without_muted_notifications,
-          for: user
-        })
-      end,
-      "Render notifications with muted" => fn ->
-        Pleroma.Web.MastodonAPI.NotificationView.render("index.json", %{
-          notifications: with_muted_notifications,
-          for: user
-        })
-      end
-    })
-  end
-
-  def query_dms(user) do
-    params = %{
-      "count" => "20",
-      "with_muted" => "true",
-      "type" => "Create",
-      "blocking_user" => user,
-      "user" => user,
-      visibility: "direct"
-    }
-
-    Benchee.run(%{
-      "Direct messages with muted" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-        |> Pleroma.Pagination.fetch_paginated(params)
-      end,
-      "Direct messages without muted" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-        |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
-      end
-    })
-
-    dms_with_muted =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-      |> Pleroma.Pagination.fetch_paginated(params)
-
-    dms_without_muted =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_query([user.ap_id], params)
-      |> Pleroma.Pagination.fetch_paginated(Map.put(params, "with_muted", false))
-
-    Benchee.run(%{
-      "Rendering dms with muted" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: dms_with_muted,
-          for: user,
-          as: :activity
-        })
-      end,
-      "Rendering dms without muted" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-          activities: dms_without_muted,
-          for: user,
-          as: :activity
-        })
-      end
-    })
-  end
-
-  def query_long_thread(user, activity) do
-    Benchee.run(%{
-      "Fetch main post" => fn ->
-        Pleroma.Activity.get_by_id_with_object(activity.id)
-      end,
-      "Fetch context of main post" => fn ->
-        Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
-          activity.data["context"],
-          %{
-            "blocking_user" => user,
-            "user" => user,
-            "exclude_id" => activity.id
-          }
-        )
-      end
-    })
-
-    activity = Pleroma.Activity.get_by_id_with_object(activity.id)
-
-    context =
-      Pleroma.Web.ActivityPub.ActivityPub.fetch_activities_for_context(
-        activity.data["context"],
-        %{
-          "blocking_user" => user,
-          "user" => user,
-          "exclude_id" => activity.id
-        }
-      )
-
-    Benchee.run(%{
-      "Render status" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render("status.json", %{
-          activity: activity,
-          for: user
-        })
-      end,
-      "Render context" => fn ->
-        Pleroma.Web.MastodonAPI.StatusView.render(
-          "index.json",
-          for: user,
-          activities: context,
-          as: :activity
-        )
-        |> Enum.reverse()
-      end
-    })
-  end
-end
diff --git a/lib/load_testing/generator.ex b/lib/load_testing/generator.ex
deleted file mode 100644
index 5c5a5c122..000000000
--- a/lib/load_testing/generator.ex
+++ /dev/null
@@ -1,352 +0,0 @@
-defmodule Pleroma.LoadTesting.Generator do
-  use Pleroma.LoadTesting.Helper
-  alias Pleroma.Web.CommonAPI
-
-  def generate_users(opts) do
-    IO.puts("Starting generating #{opts[:users_max]} users...")
-    {time, _} = :timer.tc(fn -> do_generate_users(opts) end)
-
-    IO.puts("Inserting users take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_users(opts) do
-    max = Keyword.get(opts, :users_max)
-
-    Task.async_stream(
-      1..max,
-      &generate_user_data(&1),
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Enum.to_list()
-  end
-
-  defp generate_user_data(i) do
-    remote = Enum.random([true, false])
-
-    user = %User{
-      name: "Test テスト User #{i}",
-      email: "user#{i}@example.com",
-      nickname: "nick#{i}",
-      password_hash:
-        "$pbkdf2-sha512$160000$bU.OSFI7H/yqWb5DPEqyjw$uKp/2rmXw12QqnRRTqTtuk2DTwZfF8VR4MYW2xMeIlqPR/UX1nT1CEKVUx2CowFMZ5JON8aDvURrZpJjSgqXrg",
-      bio: "Tester Number #{i}",
-      info: %{},
-      local: remote
-    }
-
-    user_urls =
-      if remote do
-        base_url =
-          Enum.random(["https://domain1.com", "https://domain2.com", "https://domain3.com"])
-
-        ap_id = "#{base_url}/users/#{user.nickname}"
-
-        %{
-          ap_id: ap_id,
-          follower_address: ap_id <> "/followers",
-          following_address: ap_id <> "/following",
-          following: [ap_id]
-        }
-      else
-        %{
-          ap_id: User.ap_id(user),
-          follower_address: User.ap_followers(user),
-          following_address: User.ap_following(user),
-          following: [User.ap_id(user)]
-        }
-      end
-
-    user = Map.merge(user, user_urls)
-
-    Repo.insert!(user)
-  end
-
-  def generate_activities(user, users) do
-    do_generate_activities(user, users)
-  end
-
-  defp do_generate_activities(user, users) do
-    IO.puts("Starting generating 20000 common activities...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..20_000,
-          fn _ ->
-            do_generate_activity([user | users])
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting common activities take #{to_sec(time)} sec.\n")
-
-    IO.puts("Starting generating 20000 activities with mentions...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..20_000,
-          fn _ ->
-            do_generate_activity_with_mention(user, users)
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting activities with menthions take #{to_sec(time)} sec.\n")
-
-    IO.puts("Starting generating 10000 activities with threads...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..10_000,
-          fn _ ->
-            do_generate_threads([user | users])
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting activities with threads take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_activity(users) do
-    post = %{
-      "status" => "Some status without mention with random user"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  defp do_generate_activity_with_mention(user, users) do
-    mentions_cnt = Enum.random([2, 3, 4, 5])
-    with_user = Enum.random([true, false])
-    users = Enum.shuffle(users)
-    mentions_users = Enum.take(users, mentions_cnt)
-    mentions_users = if with_user, do: [user | mentions_users], else: mentions_users
-
-    mentions_str =
-      Enum.map(mentions_users, fn user -> "@" <> user.nickname end) |> Enum.join(", ")
-
-    post = %{
-      "status" => mentions_str <> "some status with mentions random users"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  defp do_generate_threads(users) do
-    thread_length = Enum.random([2, 3, 4, 5])
-    actor = Enum.random(users)
-
-    post = %{
-      "status" => "Start of the thread"
-    }
-
-    {:ok, activity} = CommonAPI.post(actor, post)
-
-    Enum.each(1..thread_length, fn _ ->
-      user = Enum.random(users)
-
-      post = %{
-        "status" => "@#{actor.nickname} reply to thread",
-        "in_reply_to_status_id" => activity.id
-      }
-
-      CommonAPI.post(user, post)
-    end)
-  end
-
-  def generate_remote_activities(user, users) do
-    do_generate_remote_activities(user, users)
-  end
-
-  defp do_generate_remote_activities(user, users) do
-    IO.puts("Starting generating 10000 remote activities...")
-
-    {time, _} =
-      :timer.tc(fn ->
-        Task.async_stream(
-          1..10_000,
-          fn i ->
-            do_generate_remote_activity(i, user, users)
-          end,
-          max_concurrency: 10,
-          timeout: 30_000
-        )
-        |> Stream.run()
-      end)
-
-    IO.puts("Inserting remote activities take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_remote_activity(i, user, users) do
-    actor = Enum.random(users)
-    %{host: host} = URI.parse(actor.ap_id)
-    date = Date.utc_today()
-    datetime = DateTime.utc_now()
-
-    map = %{
-      "actor" => actor.ap_id,
-      "cc" => [actor.follower_address, user.ap_id],
-      "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-      "id" => actor.ap_id <> "/statuses/#{i}/activity",
-      "object" => %{
-        "actor" => actor.ap_id,
-        "atomUri" => actor.ap_id <> "/statuses/#{i}",
-        "attachment" => [],
-        "attributedTo" => actor.ap_id,
-        "bcc" => [],
-        "bto" => [],
-        "cc" => [actor.follower_address, user.ap_id],
-        "content" =>
-          "<p><span class=\"h-card\"><a href=\"" <>
-            user.ap_id <>
-            "\" class=\"u-url mention\">@<span>" <> user.nickname <> "</span></a></span></p>",
-        "context" => "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-        "conversation" =>
-          "tag:mastodon.example.org,#{date}:objectId=#{i}:objectType=Conversation",
-        "emoji" => %{},
-        "id" => actor.ap_id <> "/statuses/#{i}",
-        "inReplyTo" => nil,
-        "inReplyToAtomUri" => nil,
-        "published" => datetime,
-        "sensitive" => true,
-        "summary" => "cw",
-        "tag" => [
-          %{
-            "href" => user.ap_id,
-            "name" => "@#{user.nickname}@#{host}",
-            "type" => "Mention"
-          }
-        ],
-        "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-        "type" => "Note",
-        "url" => "http://#{host}/@#{actor.nickname}/#{i}"
-      },
-      "published" => datetime,
-      "to" => ["https://www.w3.org/ns/activitystreams#Public"],
-      "type" => "Create"
-    }
-
-    Pleroma.Web.ActivityPub.ActivityPub.insert(map, false)
-  end
-
-  def generate_dms(user, users, opts) do
-    IO.puts("Starting generating #{opts[:dms_max]} DMs")
-    {time, _} = :timer.tc(fn -> do_generate_dms(user, users, opts) end)
-    IO.puts("Inserting dms take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_dms(user, users, opts) do
-    Task.async_stream(
-      1..opts[:dms_max],
-      fn _ ->
-        do_generate_dm(user, users)
-      end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-  end
-
-  defp do_generate_dm(user, users) do
-    post = %{
-      "status" => "@#{user.nickname} some direct message",
-      "visibility" => "direct"
-    }
-
-    CommonAPI.post(Enum.random(users), post)
-  end
-
-  def generate_long_thread(user, users, opts) do
-    IO.puts("Starting generating long thread with #{opts[:thread_length]} replies")
-    {time, activity} = :timer.tc(fn -> do_generate_long_thread(user, users, opts) end)
-    IO.puts("Inserting long thread replies take #{to_sec(time)} sec.\n")
-    {:ok, activity}
-  end
-
-  defp do_generate_long_thread(user, users, opts) do
-    {:ok, %{id: id} = activity} = CommonAPI.post(user, %{"status" => "Start of long thread"})
-
-    Task.async_stream(
-      1..opts[:thread_length],
-      fn _ -> do_generate_thread(users, id) end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-
-    activity
-  end
-
-  defp do_generate_thread(users, activity_id) do
-    CommonAPI.post(Enum.random(users), %{
-      "status" => "reply to main post",
-      "in_reply_to_status_id" => activity_id
-    })
-  end
-
-  def generate_non_visible_message(user, users) do
-    IO.puts("Starting generating 1000 non visible posts")
-
-    {time, _} =
-      :timer.tc(fn ->
-        do_generate_non_visible_posts(user, users)
-      end)
-
-    IO.puts("Inserting non visible posts take #{to_sec(time)} sec.\n")
-  end
-
-  defp do_generate_non_visible_posts(user, users) do
-    [not_friend | users] = users
-
-    make_friends(user, users)
-
-    Task.async_stream(1..1000, fn _ -> do_generate_non_visible_post(not_friend, users) end,
-      max_concurrency: 10,
-      timeout: 30_000
-    )
-    |> Stream.run()
-  end
-
-  defp make_friends(_user, []), do: nil
-
-  defp make_friends(user, [friend | users]) do
-    {:ok, _} = User.follow(user, friend)
-    {:ok, _} = User.follow(friend, user)
-    make_friends(user, users)
-  end
-
-  defp do_generate_non_visible_post(not_friend, users) do
-    post = %{
-      "status" => "some non visible post",
-      "visibility" => "private"
-    }
-
-    {:ok, activity} = CommonAPI.post(not_friend, post)
-
-    thread_length = Enum.random([2, 3, 4, 5])
-
-    Enum.each(1..thread_length, fn _ ->
-      user = Enum.random(users)
-
-      post = %{
-        "status" => "@#{not_friend.nickname} reply to non visible post",
-        "in_reply_to_status_id" => activity.id,
-        "visibility" => "private"
-      }
-
-      CommonAPI.post(user, post)
-    end)
-  end
-end
diff --git a/lib/load_testing/helper.ex b/lib/load_testing/helper.ex
deleted file mode 100644
index 47b25c65f..000000000
--- a/lib/load_testing/helper.ex
+++ /dev/null
@@ -1,11 +0,0 @@
-defmodule Pleroma.LoadTesting.Helper do
-  defmacro __using__(_) do
-    quote do
-      import Ecto.Query
-      alias Pleroma.Repo
-      alias Pleroma.User
-
-      defp to_sec(microseconds), do: microseconds / 1_000_000
-    end
-  end
-end
diff --git a/lib/mix/tasks/pleroma/load_testing.ex b/lib/mix/tasks/pleroma/load_testing.ex
deleted file mode 100644
index a7057395d..000000000
--- a/lib/mix/tasks/pleroma/load_testing.ex
+++ /dev/null
@@ -1,134 +0,0 @@
-defmodule Mix.Tasks.Pleroma.LoadTesting do
-  use Mix.Task
-  use Pleroma.LoadTesting.Helper
-  import Mix.Pleroma
-  import Pleroma.LoadTesting.Generator
-  import Pleroma.LoadTesting.Fetcher
-
-  @shortdoc "Factory for generation data"
-  @moduledoc """
-  Generates data like:
-  - users
-  - activities with notifications
-  - direct messages
-  - long thread
-  - non visible posts
-
-  ## Generate data
-      MIX_ENV=benchmark mix pleroma.load_testing --users 20000 --dms 20000 --thread_length 2000
-      MIX_ENV=benchmark mix pleroma.load_testing -u 20000 -d 20000 -t 2000
-
-  Options:
-  - `--users NUMBER` - number of users to generate. Defaults to: 20000. Alias: `-u`
-  - `--dms NUMBER` - number of direct messages to generate. Defaults to: 20000. Alias `-d`
-  - `--thread_length` - number of messages in thread. Defaults to: 2000. ALias `-t`
-  """
-
-  @aliases [u: :users, d: :dms, t: :thread_length]
-  @switches [
-    users: :integer,
-    dms: :integer,
-    thread_length: :integer
-  ]
-  @users_default 20_000
-  @dms_default 20_000
-  @thread_length_default 2_000
-
-  def run(args) do
-    start_pleroma()
-    Pleroma.Config.put([:instance, :skip_thread_containment], true)
-    {opts, _} = OptionParser.parse!(args, strict: @switches, aliases: @aliases)
-
-    users_max = Keyword.get(opts, :users, @users_default)
-    dms_max = Keyword.get(opts, :dms, @dms_default)
-    thread_length = Keyword.get(opts, :thread_length, @thread_length_default)
-
-    clean_tables()
-
-    opts =
-      Keyword.put(opts, :users_max, users_max)
-      |> Keyword.put(:dms_max, dms_max)
-      |> Keyword.put(:thread_length, thread_length)
-
-    generate_users(opts)
-
-    # main user for queries
-    IO.puts("Fetching local main user...")
-
-    {time, user} =
-      :timer.tc(fn ->
-        Repo.one(
-          from(u in User, where: u.local == true, order_by: fragment("RANDOM()"), limit: 1)
-        )
-      end)
-
-    IO.puts("Fetching main user take #{to_sec(time)} sec.\n")
-
-    IO.puts("Fetching local users...")
-
-    {time, users} =
-      :timer.tc(fn ->
-        Repo.all(
-          from(u in User,
-            where: u.id != ^user.id,
-            where: u.local == true,
-            order_by: fragment("RANDOM()"),
-            limit: 10
-          )
-        )
-      end)
-
-    IO.puts("Fetching local users take #{to_sec(time)} sec.\n")
-
-    IO.puts("Fetching remote users...")
-
-    {time, remote_users} =
-      :timer.tc(fn ->
-        Repo.all(
-          from(u in User,
-            where: u.id != ^user.id,
-            where: u.local == false,
-            order_by: fragment("RANDOM()"),
-            limit: 10
-          )
-        )
-      end)
-
-    IO.puts("Fetching remote users take #{to_sec(time)} sec.\n")
-
-    generate_activities(user, users)
-
-    generate_remote_activities(user, remote_users)
-
-    generate_dms(user, users, opts)
-
-    {:ok, activity} = generate_long_thread(user, users, opts)
-
-    generate_non_visible_message(user, users)
-
-    IO.puts("Users in DB: #{Repo.aggregate(from(u in User), :count, :id)}")
-
-    IO.puts("Activities in DB: #{Repo.aggregate(from(a in Pleroma.Activity), :count, :id)}")
-
-    IO.puts("Objects in DB: #{Repo.aggregate(from(o in Pleroma.Object), :count, :id)}")
-
-    IO.puts(
-      "Notifications in DB: #{Repo.aggregate(from(n in Pleroma.Notification), :count, :id)}"
-    )
-
-    fetch_user(user)
-    query_timelines(user)
-    query_notifications(user)
-    query_dms(user)
-    query_long_thread(user, activity)
-    Pleroma.Config.put([:instance, :skip_thread_containment], false)
-    query_timelines(user)
-  end
-
-  defp clean_tables do
-    IO.puts("Deleting old data...\n")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE users CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE activities CASCADE;")
-    Ecto.Adapters.SQL.query!(Repo, "TRUNCATE objects CASCADE;")
-  end
-end
-- 
cgit v1.2.3


From 580a67f5ca152fa4454fd9d8ab2e2a5a55af1a2a Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Wed, 9 Oct 2019 06:51:14 +0300
Subject: Mastodon API: Return `pleroma.direct_conversation_id` when viewing a
 status (`GET /api/v1/statuses/:id`)

---
 lib/pleroma/web/mastodon_api/controllers/status_controller.ex | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
index 0c16e9b0f..e5d016f63 100644
--- a/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
+++ b/lib/pleroma/web/mastodon_api/controllers/status_controller.ex
@@ -167,7 +167,11 @@ defmodule Pleroma.Web.MastodonAPI.StatusController do
   def show(%{assigns: %{user: user}} = conn, %{"id" => id}) do
     with %Activity{} = activity <- Activity.get_by_id_with_object(id),
          true <- Visibility.visible_for_user?(activity, user) do
-      try_render(conn, "show.json", activity: activity, for: user)
+      try_render(conn, "show.json",
+        activity: activity,
+        for: user,
+        with_direct_conversation_id: true
+      )
     end
   end
 
-- 
cgit v1.2.3


From d537bfd4e1e3119188519a4b1839fdbe3f6fad39 Mon Sep 17 00:00:00 2001
From: Egor Kislitsyn <egor@kislitsyn.com>
Date: Wed, 9 Oct 2019 13:11:57 +0700
Subject: Add a task to re-count statuses for all users

---
 lib/mix/tasks/pleroma/count_statuses.ex | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)
 create mode 100644 lib/mix/tasks/pleroma/count_statuses.ex

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/count_statuses.ex b/lib/mix/tasks/pleroma/count_statuses.ex
new file mode 100644
index 000000000..e1e8195dd
--- /dev/null
+++ b/lib/mix/tasks/pleroma/count_statuses.ex
@@ -0,0 +1,22 @@
+defmodule Mix.Tasks.Pleroma.CountStatuses do
+  @shortdoc "Re-counts statuses for all users"
+
+  use Mix.Task
+  alias Pleroma.User
+  import Ecto.Query
+
+  def run([]) do
+    Mix.Pleroma.start_pleroma()
+
+    stream =
+      User
+      |> where(local: true)
+      |> Pleroma.Repo.stream()
+
+    Pleroma.Repo.transaction(fn ->
+      Enum.each(stream, &User.update_note_count/1)
+    end)
+
+    Mix.Pleroma.shell_info("Done")
+  end
+end
-- 
cgit v1.2.3


From ad42837244ba4c945b76c5addaffe47353cf62a8 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Wed, 9 Oct 2019 17:03:54 +0300
Subject: Ability to toggle activation status and permission group for a group
 of users

---
 lib/pleroma/moderation_log.ex                     | 60 +++++++--------
 lib/pleroma/user.ex                               | 16 +++-
 lib/pleroma/web/admin_api/admin_api_controller.ex | 89 ++++++++++-------------
 lib/pleroma/web/admin_api/views/account_view.ex   |  6 ++
 lib/pleroma/web/router.ex                         | 14 +---
 5 files changed, 95 insertions(+), 90 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/moderation_log.ex b/lib/pleroma/moderation_log.ex
index 352cad433..42649ff02 100644
--- a/lib/pleroma/moderation_log.ex
+++ b/lib/pleroma/moderation_log.ex
@@ -86,18 +86,18 @@ defmodule Pleroma.ModerationLog do
     parsed_datetime
   end
 
-  @spec insert_log(%{actor: User, subject: User, action: String.t(), permission: String.t()}) ::
+  @spec insert_log(%{actor: User, subject: [User], action: String.t(), permission: String.t()}) ::
           {:ok, ModerationLog} | {:error, any}
   def insert_log(%{
         actor: %User{} = actor,
-        subject: %User{} = subject,
+        subject: subjects,
         action: action,
         permission: permission
       }) do
     %ModerationLog{
       data: %{
         "actor" => user_to_map(actor),
-        "subject" => user_to_map(subject),
+        "subject" => user_to_map(subjects),
         "action" => action,
         "permission" => permission,
         "message" => ""
@@ -303,13 +303,16 @@ defmodule Pleroma.ModerationLog do
   end
 
   @spec insert_log_entry_with_message(ModerationLog) :: {:ok, ModerationLog} | {:error, any}
-
   defp insert_log_entry_with_message(entry) do
     entry.data["message"]
     |> put_in(get_log_entry_message(entry))
     |> Repo.insert()
   end
 
+  defp user_to_map(users) when is_list(users) do
+    users |> Enum.map(&user_to_map/1)
+  end
+
   defp user_to_map(%User{} = user) do
     user
     |> Map.from_struct()
@@ -363,12 +366,7 @@ defmodule Pleroma.ModerationLog do
           "subjects" => subjects
         }
       }) do
-    nicknames =
-      subjects
-      |> Enum.map(&"@#{&1["nickname"]}")
-      |> Enum.join(", ")
-
-    "@#{actor_nickname} created users: #{nicknames}"
+    "@#{actor_nickname} created users: #{users_to_nicknames_string(subjects)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -376,10 +374,10 @@ defmodule Pleroma.ModerationLog do
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "activate",
-          "subject" => %{"nickname" => subject_nickname, "type" => "user"}
+          "subject" => users
         }
       }) do
-    "@#{actor_nickname} activated user @#{subject_nickname}"
+    "@#{actor_nickname} activated users: #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -387,10 +385,10 @@ defmodule Pleroma.ModerationLog do
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "deactivate",
-          "subject" => %{"nickname" => subject_nickname, "type" => "user"}
+          "subject" => users
         }
       }) do
-    "@#{actor_nickname} deactivated user @#{subject_nickname}"
+    "@#{actor_nickname} deactivated users: #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -402,14 +400,9 @@ defmodule Pleroma.ModerationLog do
           "action" => "tag"
         }
       }) do
-    nicknames_string =
-      nicknames
-      |> Enum.map(&"@#{&1}")
-      |> Enum.join(", ")
-
     tags_string = tags |> Enum.join(", ")
 
-    "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_string}"
+    "@#{actor_nickname} added tags: #{tags_string} to users: #{nicknames_to_string(nicknames)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -421,14 +414,9 @@ defmodule Pleroma.ModerationLog do
           "action" => "untag"
         }
       }) do
-    nicknames_string =
-      nicknames
-      |> Enum.map(&"@#{&1}")
-      |> Enum.join(", ")
-
     tags_string = tags |> Enum.join(", ")
 
-    "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_string}"
+    "@#{actor_nickname} removed tags: #{tags_string} from users: #{nicknames_to_string(nicknames)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -436,11 +424,11 @@ defmodule Pleroma.ModerationLog do
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "grant",
-          "subject" => %{"nickname" => subject_nickname},
+          "subject" => users,
           "permission" => permission
         }
       }) do
-    "@#{actor_nickname} made @#{subject_nickname} #{permission}"
+    "@#{actor_nickname} made #{users_to_nicknames_string(users)} #{permission}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -448,11 +436,11 @@ defmodule Pleroma.ModerationLog do
         data: %{
           "actor" => %{"nickname" => actor_nickname},
           "action" => "revoke",
-          "subject" => %{"nickname" => subject_nickname},
+          "subject" => users,
           "permission" => permission
         }
       }) do
-    "@#{actor_nickname} revoked #{permission} role from @#{subject_nickname}"
+    "@#{actor_nickname} revoked #{permission} role from #{users_to_nicknames_string(users)}"
   end
 
   @spec get_log_entry_message(ModerationLog) :: String.t()
@@ -551,4 +539,16 @@ defmodule Pleroma.ModerationLog do
       }) do
     "@#{actor_nickname} deleted status ##{subject_id}"
   end
+
+  defp nicknames_to_string(nicknames) do
+    nicknames
+    |> Enum.map(&"@#{&1}")
+    |> Enum.join(", ")
+  end
+
+  defp users_to_nicknames_string(users) do
+    users
+    |> Enum.map(&"@#{&1["nickname"]}")
+    |> Enum.join(", ")
+  end
 end
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 2cfb13a8c..a76a5ad70 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -1059,7 +1059,15 @@ defmodule Pleroma.User do
     BackgroundWorker.enqueue("deactivate_user", %{"user_id" => user.id, "status" => status})
   end
 
-  def deactivate(%User{} = user, status \\ true) do
+  def deactivate(user, status \\ true)
+
+  def deactivate(users, status) when is_list(users) do
+    Repo.transaction(fn ->
+      for user <- users, do: deactivate(user, status)
+    end)
+  end
+
+  def deactivate(%User{} = user, status) do
     with {:ok, user} <- update_info(user, &User.Info.set_activation_status(&1, status)) do
       Enum.each(get_followers(user), &invalidate_cache/1)
       Enum.each(get_friends(user), &update_follower_count/1)
@@ -1625,6 +1633,12 @@ defmodule Pleroma.User do
 
   `fun` is called with the `user.info`.
   """
+  def update_info(users, fun) when is_list(users) do
+    Repo.transaction(fn ->
+      for user <- users, do: update_info(user, fun)
+    end)
+  end
+
   def update_info(user, fun) do
     user
     |> change_info(fun)
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 513bae800..d825a5d28 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -231,22 +231,34 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
-  def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
-    user = User.get_cached_by_nickname(nickname)
+  def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, false)
 
-    {:ok, updated_user} = User.deactivate(user, !user.info.deactivated)
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: users,
+      action: "activate"
+    })
 
-    action = if user.info.deactivated, do: "activate", else: "deactivate"
+    conn
+    |> put_view(AccountView)
+    |> render("index.json", %{users: Keyword.values(updated_users)})
+  end
+
+  def user_deactivate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
+    users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
+    {:ok, updated_users} = User.deactivate(users, true)
 
     ModerationLog.insert_log(%{
       actor: admin,
-      subject: user,
-      action: action
+      subject: users,
+      action: "deactivate"
     })
 
     conn
     |> put_view(AccountView)
-    |> render("show.json", %{user: updated_user})
+    |> render("index.json", %{users: Keyword.values(updated_users)})
   end
 
   def tag_users(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames, "tags" => tags}) do
@@ -315,20 +327,19 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
 
   def right_add(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
-        "nickname" => nickname
+        "nicknames" => nicknames
       })
       when permission_group in ["moderator", "admin"] do
     info = Map.put(%{}, "is_" <> permission_group, true)
 
-    {:ok, user} =
-      nickname
-      |> User.get_cached_by_nickname()
-      |> User.update_info(&User.Info.admin_api_update(&1, info))
+    users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
+
+    User.update_info(users, &User.Info.admin_api_update(&1, info))
 
     ModerationLog.insert_log(%{
       action: "grant",
       actor: admin,
-      subject: user,
+      subject: users,
       permission: permission_group
     })
 
@@ -349,58 +360,38 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     })
   end
 
-  def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do
-    render_error(conn, :forbidden, "You can't revoke your own admin status.")
-  end
-
   def right_delete(
-        %{assigns: %{user: admin}} = conn,
+        %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn,
         %{
           "permission_group" => permission_group,
-          "nickname" => nickname
+          "nicknames" => nicknames
         }
       )
       when permission_group in ["moderator", "admin"] do
-    info = Map.put(%{}, "is_" <> permission_group, false)
+    with false <- Enum.member?(nicknames, admin_nickname) do
+      info = Map.put(%{}, "is_" <> permission_group, false)
 
-    {:ok, user} =
-      nickname
-      |> User.get_cached_by_nickname()
-      |> User.update_info(&User.Info.admin_api_update(&1, info))
+      users = nicknames |> Enum.map(&User.get_cached_by_nickname/1)
 
-    ModerationLog.insert_log(%{
-      action: "revoke",
-      actor: admin,
-      subject: user,
-      permission: permission_group
-    })
-
-    json(conn, info)
-  end
-
-  def right_delete(conn, _) do
-    render_error(conn, :not_found, "No such permission_group")
-  end
-
-  def set_activation_status(%{assigns: %{user: admin}} = conn, %{
-        "nickname" => nickname,
-        "status" => status
-      }) do
-    with {:ok, status} <- Ecto.Type.cast(:boolean, status),
-         %User{} = user <- User.get_cached_by_nickname(nickname),
-         {:ok, _} <- User.deactivate(user, !status) do
-      action = if(user.info.deactivated, do: "activate", else: "deactivate")
+      User.update_info(users, &User.Info.admin_api_update(&1, info))
 
       ModerationLog.insert_log(%{
+        action: "revoke",
         actor: admin,
-        subject: user,
-        action: action
+        subject: users,
+        permission: permission_group
       })
 
-      json_response(conn, :no_content, "")
+      json(conn, info)
+    else
+      _ -> render_error(conn, :forbidden, "You can't revoke your own admin/moderator status.")
     end
   end
 
+  def right_delete(conn, _) do
+    render_error(conn, :not_found, "No such permission_group")
+  end
+
   def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do
     with {:ok, _message} <- Relay.follow(target) do
       ModerationLog.insert_log(%{
diff --git a/lib/pleroma/web/admin_api/views/account_view.ex b/lib/pleroma/web/admin_api/views/account_view.ex
index a96affd40..441269162 100644
--- a/lib/pleroma/web/admin_api/views/account_view.ex
+++ b/lib/pleroma/web/admin_api/views/account_view.ex
@@ -19,6 +19,12 @@ defmodule Pleroma.Web.AdminAPI.AccountView do
     }
   end
 
+  def render("index.json", %{users: users}) do
+    %{
+      users: render_many(users, AccountView, "show.json", as: :user)
+    }
+  end
+
   def render("show.json", %{user: user}) do
     avatar = User.avatar_url(user) |> MediaProxy.url()
     display_name = HTML.strip_tags(user.name || user.nickname)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index ae799b8ac..894375357 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -136,21 +136,15 @@ defmodule Pleroma.Web.Router do
 
     delete("/users", AdminAPIController, :user_delete)
     post("/users", AdminAPIController, :users_create)
-    patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
+    patch("/users/activate", AdminAPIController, :user_activate)
+    patch("/users/deactivate", AdminAPIController, :user_deactivate)
     put("/users/tag", AdminAPIController, :tag_users)
     delete("/users/tag", AdminAPIController, :untag_users)
 
     get("/users/:nickname/permission_group", AdminAPIController, :right_get)
     get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get)
-    post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add)
-
-    delete(
-      "/users/:nickname/permission_group/:permission_group",
-      AdminAPIController,
-      :right_delete
-    )
-
-    put("/users/:nickname/activation_status", AdminAPIController, :set_activation_status)
+    post("/users/permission_group/:permission_group", AdminAPIController, :right_add)
+    delete("/users/permission_group/:permission_group", AdminAPIController, :right_delete)
 
     post("/relay", AdminAPIController, :relay_follow)
     delete("/relay", AdminAPIController, :relay_unfollow)
-- 
cgit v1.2.3


From 29647dfd09a78fa66a605ff97119a21779d15020 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Thu, 10 Oct 2019 17:17:33 +0200
Subject: Transmogrifier: Save correct ids for incoming deletes.

---
 lib/pleroma/web/activity_pub/activity_pub.ex   | 8 ++++++--
 lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++--
 2 files changed, 8 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 9f29087df..3a0c382ca 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -409,7 +409,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, local \\ true) do
+  def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ []) do
+    local = Keyword.get(options, :local, true)
+    activity_id = Keyword.get(options, :activity_id, nil)
+
     user = User.get_cached_by_ap_id(actor)
     to = (object.data["to"] || []) ++ (object.data["cc"] || [])
 
@@ -420,7 +423,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
            "object" => id,
            "to" => to,
            "deleted_activity_id" => activity && activity.id
-         },
+         }
+         |> maybe_put("id", activity_id),
          {:ok, activity} <- insert(data, local, false),
          stream_out_participations(object, user),
          _ <- decrease_replies_count_if_reply(object),
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 872ed0eb2..620b54d3b 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -637,7 +637,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   # an error or a tombstone.  This would allow us to verify that a deletion actually took
   # place.
   def handle_incoming(
-        %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => _id} = data,
+        %{"type" => "Delete", "object" => object_id, "actor" => actor, "id" => id} = data,
         _options
       ) do
     object_id = Utils.get_ap_id(object_id)
@@ -646,7 +646,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          :ok <- Containment.contain_origin(actor.ap_id, object.data),
-         {:ok, activity} <- ActivityPub.delete(object, false) do
+         {:ok, activity} <- ActivityPub.delete(object, local: false, activity_id: id) do
       {:ok, activity}
     else
       nil ->
-- 
cgit v1.2.3


From f5104f36bbec7d49d4ff5acee4b9d28223c6474d Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 11 Oct 2019 00:24:31 +0300
Subject: Deprecate /api/pleroma/admin/users/:nickname/toggle_activation
 instead of deleting it

---
 lib/pleroma/web/admin_api/admin_api_controller.ex | 20 ++++++++++++++++++++
 lib/pleroma/web/router.ex                         |  1 +
 2 files changed, 21 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index d825a5d28..5b513bd7c 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -46,6 +46,8 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
            :user_delete,
            :users_create,
            :user_toggle_activation,
+           :user_activate,
+           :user_deactivate,
            :tag_users,
            :untag_users,
            :right_add,
@@ -231,6 +233,24 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def user_toggle_activation(%{assigns: %{user: admin}} = conn, %{"nickname" => nickname}) do
+    user = User.get_cached_by_nickname(nickname)
+
+    {:ok, updated_user} = User.deactivate(user, !user.info.deactivated)
+
+    action = if user.info.deactivated, do: "activate", else: "deactivate"
+
+    ModerationLog.insert_log(%{
+      actor: admin,
+      subject: [user],
+      action: action
+    })
+
+    conn
+    |> put_view(AccountView)
+    |> render("show.json", %{user: updated_user})
+  end
+
   def user_activate(%{assigns: %{user: admin}} = conn, %{"nicknames" => nicknames}) do
     users = Enum.map(nicknames, &User.get_cached_by_nickname/1)
     {:ok, updated_users} = User.deactivate(users, false)
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 894375357..a79df51a2 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -136,6 +136,7 @@ defmodule Pleroma.Web.Router do
 
     delete("/users", AdminAPIController, :user_delete)
     post("/users", AdminAPIController, :users_create)
+    patch("/users/:nickname/toggle_activation", AdminAPIController, :user_toggle_activation)
     patch("/users/activate", AdminAPIController, :user_activate)
     patch("/users/deactivate", AdminAPIController, :user_deactivate)
     put("/users/tag", AdminAPIController, :tag_users)
-- 
cgit v1.2.3


From 9b963064eb4c8cb740ffa128f491f2ee581fdb8b Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 11 Oct 2019 11:25:45 +0200
Subject: Transmogrifier: Actually store who deleted a note.

---
 lib/pleroma/web/activity_pub/activity_pub.ex   | 18 ++++++++++--------
 lib/pleroma/web/activity_pub/transmogrifier.ex |  3 ++-
 2 files changed, 12 insertions(+), 9 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 3a0c382ca..f78574455 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -412,19 +412,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def delete(%Object{data: %{"id" => id, "actor" => actor}} = object, options \\ []) do
     local = Keyword.get(options, :local, true)
     activity_id = Keyword.get(options, :activity_id, nil)
+    actor = Keyword.get(options, :actor, actor)
 
     user = User.get_cached_by_ap_id(actor)
     to = (object.data["to"] || []) ++ (object.data["cc"] || [])
 
     with {:ok, object, activity} <- Object.delete(object),
-         data <- %{
-           "type" => "Delete",
-           "actor" => actor,
-           "object" => id,
-           "to" => to,
-           "deleted_activity_id" => activity && activity.id
-         }
-         |> maybe_put("id", activity_id),
+         data <-
+           %{
+             "type" => "Delete",
+             "actor" => actor,
+             "object" => id,
+             "to" => to,
+             "deleted_activity_id" => activity && activity.id
+           }
+           |> maybe_put("id", activity_id),
          {:ok, activity} <- insert(data, local, false),
          stream_out_participations(object, user),
          _ <- decrease_replies_count_if_reply(object),
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 620b54d3b..64fbb9fa4 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -646,7 +646,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
          {:ok, %User{} = actor} <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id),
          :ok <- Containment.contain_origin(actor.ap_id, object.data),
-         {:ok, activity} <- ActivityPub.delete(object, local: false, activity_id: id) do
+         {:ok, activity} <-
+           ActivityPub.delete(object, local: false, activity_id: id, actor: actor.ap_id) do
       {:ok, activity}
     else
       nil ->
-- 
cgit v1.2.3


From 37812740c4c03f0a79d17ff186db644a60414ff7 Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 11 Oct 2019 11:48:58 +0200
Subject: Transmogrifier: Correctly save incoming ids for Accept/Reject.

---
 lib/pleroma/web/activity_pub/activity_pub.ex   | 23 +++++++++++------------
 lib/pleroma/web/activity_pub/transmogrifier.ex | 10 ++++++----
 2 files changed, 17 insertions(+), 16 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index f78574455..364452b5d 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -269,22 +269,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  def accept(%{to: to, actor: actor, object: object} = params) do
-    # only accept false as false value
-    local = !(params[:local] == false)
+  def accept(params) do
+    accept_or_reject("Accept", params)
+  end
 
-    with data <- %{"to" => to, "type" => "Accept", "actor" => actor.ap_id, "object" => object},
-         {:ok, activity} <- insert(data, local),
-         :ok <- maybe_federate(activity) do
-      {:ok, activity}
-    end
+  def reject(params) do
+    accept_or_reject("Reject", params)
   end
 
-  def reject(%{to: to, actor: actor, object: object} = params) do
-    # only accept false as false value
-    local = !(params[:local] == false)
+  def accept_or_reject(type, %{to: to, actor: actor, object: object} = params) do
+    local = Map.get(params, :local, true)
+    activity_id = Map.get(params, :activity_id, nil)
 
-    with data <- %{"to" => to, "type" => "Reject", "actor" => actor.ap_id, "object" => object},
+    with data <-
+           %{"to" => to, "type" => type, "actor" => actor.ap_id, "object" => object}
+           |> Utils.maybe_put("id", activity_id),
          {:ok, activity} <- insert(data, local),
          :ok <- maybe_federate(activity) do
       {:ok, activity}
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 64fbb9fa4..b56343beb 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -514,7 +514,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => _id} = data,
+        %{"type" => "Accept", "object" => follow_object, "actor" => _actor, "id" => id} = data,
         _options
       ) do
     with actor <- Containment.get_actor(data),
@@ -528,7 +528,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
         type: "Accept",
         actor: followed,
         object: follow_activity.data["id"],
-        local: false
+        local: false,
+        activity_id: id
       })
     else
       _e -> :error
@@ -536,7 +537,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   end
 
   def handle_incoming(
-        %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => _id} = data,
+        %{"type" => "Reject", "object" => follow_object, "actor" => _actor, "id" => id} = data,
         _options
       ) do
     with actor <- Containment.get_actor(data),
@@ -550,7 +551,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
              type: "Reject",
              actor: followed,
              object: follow_activity.data["id"],
-             local: false
+             local: false,
+             activity_id: id
            }) do
       User.unfollow(follower, followed)
 
-- 
cgit v1.2.3


From 422aa6befee0cbcbf71d3cacd96d90c1be3263ba Mon Sep 17 00:00:00 2001
From: lain <lain@soykaf.club>
Date: Fri, 11 Oct 2019 12:53:09 +0200
Subject: Ostatus DeleteHandler: Fix for new option format.

---
 lib/pleroma/web/ostatus/handlers/delete_handler.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex
index b2f9f3946..ac2dc115c 100644
--- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex
+++ b/lib/pleroma/web/ostatus/handlers/delete_handler.ex
@@ -11,7 +11,7 @@ defmodule Pleroma.Web.OStatus.DeleteHandler do
   def handle_delete(entry, _doc \\ nil) do
     with id <- XML.string_from_xpath("//id", entry),
          %Object{} = object <- Object.normalize(id),
-         {:ok, delete} <- ActivityPub.delete(object, false) do
+         {:ok, delete} <- ActivityPub.delete(object, local: false) do
       delete
     end
   end
-- 
cgit v1.2.3


From aaa4252f416fbad099f95232de4cf6eab11dd7d2 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 11 Oct 2019 15:58:45 +0300
Subject: Deprecate POST/DELETE
 /api/pleroma/admin/users/:nickname/permission_group/:permission_group instead
 of deleting it

---
 lib/pleroma/web/admin_api/admin_api_controller.ex | 61 +++++++++++++++++++++--
 lib/pleroma/web/router.ex                         | 18 ++++++-
 2 files changed, 74 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 5b513bd7c..33e2180ec 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -345,7 +345,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     |> Enum.into(%{}, &{&1, true})
   end
 
-  def right_add(%{assigns: %{user: admin}} = conn, %{
+  def right_add_multiple(%{assigns: %{user: admin}} = conn, %{
         "permission_group" => permission_group,
         "nicknames" => nicknames
       })
@@ -366,6 +366,32 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     json(conn, info)
   end
 
+  def right_add_multiple(conn, _) do
+    render_error(conn, :not_found, "No such permission_group")
+  end
+
+  def right_add(%{assigns: %{user: admin}} = conn, %{
+        "permission_group" => permission_group,
+        "nickname" => nickname
+      })
+      when permission_group in ["moderator", "admin"] do
+    info = Map.put(%{}, "is_" <> permission_group, true)
+
+    {:ok, user} =
+      nickname
+      |> User.get_cached_by_nickname()
+      |> User.update_info(&User.Info.admin_api_update(&1, info))
+
+    ModerationLog.insert_log(%{
+      action: "grant",
+      actor: admin,
+      subject: [user],
+      permission: permission_group
+    })
+
+    json(conn, info)
+  end
+
   def right_add(conn, _) do
     render_error(conn, :not_found, "No such permission_group")
   end
@@ -380,7 +406,7 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     })
   end
 
-  def right_delete(
+  def right_delete_multiple(
         %{assigns: %{user: %{nickname: admin_nickname} = admin}} = conn,
         %{
           "permission_group" => permission_group,
@@ -408,10 +434,39 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
-  def right_delete(conn, _) do
+  def right_delete_multiple(conn, _) do
     render_error(conn, :not_found, "No such permission_group")
   end
 
+  def right_delete(
+        %{assigns: %{user: admin}} = conn,
+        %{
+          "permission_group" => permission_group,
+          "nickname" => nickname
+        }
+      )
+      when permission_group in ["moderator", "admin"] do
+    info = Map.put(%{}, "is_" <> permission_group, false)
+
+    {:ok, user} =
+      nickname
+      |> User.get_cached_by_nickname()
+      |> User.update_info(&User.Info.admin_api_update(&1, info))
+
+    ModerationLog.insert_log(%{
+      action: "revoke",
+      actor: admin,
+      subject: [user],
+      permission: permission_group
+    })
+
+    json(conn, info)
+  end
+
+  def right_delete(%{assigns: %{user: %{nickname: nickname}}} = conn, %{"nickname" => nickname}) do
+    render_error(conn, :forbidden, "You can't revoke your own admin status.")
+  end
+
   def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do
     with {:ok, _message} <- Relay.follow(target) do
       ModerationLog.insert_log(%{
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index a79df51a2..80651f3ff 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -144,8 +144,22 @@ defmodule Pleroma.Web.Router do
 
     get("/users/:nickname/permission_group", AdminAPIController, :right_get)
     get("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_get)
-    post("/users/permission_group/:permission_group", AdminAPIController, :right_add)
-    delete("/users/permission_group/:permission_group", AdminAPIController, :right_delete)
+
+    post("/users/:nickname/permission_group/:permission_group", AdminAPIController, :right_add)
+
+    delete(
+      "/users/:nickname/permission_group/:permission_group",
+      AdminAPIController,
+      :right_delete
+    )
+
+    post("/users/permission_group/:permission_group", AdminAPIController, :right_add_multiple)
+
+    delete(
+      "/users/permission_group/:permission_group",
+      AdminAPIController,
+      :right_delete_multiple
+    )
 
     post("/relay", AdminAPIController, :relay_follow)
     delete("/relay", AdminAPIController, :relay_unfollow)
-- 
cgit v1.2.3


From cc6875b582df49d2cb780e0940b85d5b04fe0e74 Mon Sep 17 00:00:00 2001
From: Maxim Filippov <colixer@gmail.com>
Date: Fri, 11 Oct 2019 19:12:29 +0300
Subject: Add `GET /api/pleroma/admin/relay` endpoint - lists all followed
 relays

---
 lib/mix/tasks/pleroma/relay.ex                    | 10 +++-------
 lib/pleroma/web/activity_pub/relay.ex             | 14 ++++++++++++++
 lib/pleroma/web/admin_api/admin_api_controller.ex | 10 ++++++++++
 lib/pleroma/web/router.ex                         |  1 +
 4 files changed, 28 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/relay.ex b/lib/mix/tasks/pleroma/relay.ex
index d7a7b599f..7ef5f9678 100644
--- a/lib/mix/tasks/pleroma/relay.ex
+++ b/lib/mix/tasks/pleroma/relay.ex
@@ -5,7 +5,6 @@
 defmodule Mix.Tasks.Pleroma.Relay do
   use Mix.Task
   import Mix.Pleroma
-  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.Relay
 
   @shortdoc "Manages remote relays"
@@ -36,13 +35,10 @@ defmodule Mix.Tasks.Pleroma.Relay do
   def run(["list"]) do
     start_pleroma()
 
-    with %User{following: following} = _user <- Relay.get_actor() do
-      following
-      |> Enum.map(fn entry -> URI.parse(entry).host end)
-      |> Enum.uniq()
-      |> Enum.each(&shell_info(&1))
+    with {:ok, list} <- Relay.list() do
+      list |> Enum.each(&shell_info(&1))
     else
-      e -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
+      {:error, e} -> shell_error("Error while fetching relay subscription list: #{inspect(e)}")
     end
   end
 end
diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex
index c2ac38907..03fc434a9 100644
--- a/lib/pleroma/web/activity_pub/relay.ex
+++ b/lib/pleroma/web/activity_pub/relay.ex
@@ -51,6 +51,20 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def publish(_), do: {:error, "Not implemented"}
 
+  @spec list() :: {:ok, [String.t()]} | {:error, any()}
+  def list do
+    with %User{following: following} = _user <- get_actor() do
+      list =
+        following
+        |> Enum.map(fn entry -> URI.parse(entry).host end)
+        |> Enum.uniq()
+
+      {:ok, list}
+    else
+      error -> format_error(error)
+    end
+  end
+
   defp format_error({:error, error}), do: format_error(error)
 
   defp format_error(error) do
diff --git a/lib/pleroma/web/admin_api/admin_api_controller.ex b/lib/pleroma/web/admin_api/admin_api_controller.ex
index 513bae800..24dda75a9 100644
--- a/lib/pleroma/web/admin_api/admin_api_controller.ex
+++ b/lib/pleroma/web/admin_api/admin_api_controller.ex
@@ -401,6 +401,16 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIController do
     end
   end
 
+  def relay_list(conn, _params) do
+    with {:ok, list} <- Relay.list() do
+      json(conn, %{relays: list})
+    else
+      _ ->
+        conn
+        |> put_status(500)
+    end
+  end
+
   def relay_follow(%{assigns: %{user: admin}} = conn, %{"relay_url" => target}) do
     with {:ok, _message} <- Relay.follow(target) do
       ModerationLog.insert_log(%{
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index ae799b8ac..8cc967af9 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -152,6 +152,7 @@ defmodule Pleroma.Web.Router do
 
     put("/users/:nickname/activation_status", AdminAPIController, :set_activation_status)
 
+    get("/relay", AdminAPIController, :relay_list)
     post("/relay", AdminAPIController, :relay_follow)
     delete("/relay", AdminAPIController, :relay_unfollow)
 
-- 
cgit v1.2.3


From 9bdbf0811b362d8eb9d2456e9c87c9dde1b35d9b Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 11 Oct 2019 22:52:38 +0300
Subject: Make MediaProxy failure tracking less brutal

The current failure tracking mechanism will never request anything that
didn't respond with a success, 403, 404, or 5xx codes. This is causing
issues when using in real fediverse because of weird status codes
some software has and timeouts being frequent. This patch changes
failure tracking mechanism to only never request the url again if it
responded with 400, 204, or the body is too large, otherwise it can be
re-requested in 60 seconds.
---
 lib/pleroma/reverse_proxy/reverse_proxy.ex | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/reverse_proxy/reverse_proxy.ex b/lib/pleroma/reverse_proxy/reverse_proxy.ex
index 78144cae3..2ed719315 100644
--- a/lib/pleroma/reverse_proxy/reverse_proxy.ex
+++ b/lib/pleroma/reverse_proxy/reverse_proxy.ex
@@ -401,11 +401,9 @@ defmodule Pleroma.ReverseProxy do
 
   defp client, do: Pleroma.ReverseProxy.Client
 
-  defp track_failed_url(url, code, opts) do
-    code = to_string(code)
-
+  defp track_failed_url(url, error, opts) do
     ttl =
-      if code in ["403", "404"] or String.starts_with?(code, "5") do
+      unless error in [:body_too_large, 400, 204] do
         Keyword.get(opts, :failed_request_ttl, @failed_request_ttl)
       else
         nil
-- 
cgit v1.2.3


From a97b642289659a5fccb5943c54caa1ecdce5fd2f Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Tue, 8 Oct 2019 23:05:57 +0300
Subject: Mastodon API: Add `exclude_visibilities` parameter to the timeline
 and notification endpoints

---
 lib/pleroma/notification.ex                  | 104 ++++++++++++++++++++-------
 lib/pleroma/web/activity_pub/activity_pub.ex |  44 ++++++++++++
 lib/pleroma/web/mastodon_api/mastodon_api.ex |   1 +
 3 files changed, 122 insertions(+), 27 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex
index d94ae5971..d145f8d5b 100644
--- a/lib/pleroma/notification.ex
+++ b/lib/pleroma/notification.ex
@@ -17,6 +17,7 @@ defmodule Pleroma.Notification do
 
   import Ecto.Query
   import Ecto.Changeset
+  require Logger
 
   @type t :: %__MODULE__{}
 
@@ -34,43 +35,92 @@ defmodule Pleroma.Notification do
   end
 
   def for_user_query(user, opts \\ []) do
-    query =
-      Notification
-      |> where(user_id: ^user.id)
-      |> where(
-        [n, a],
+    Notification
+    |> where(user_id: ^user.id)
+    |> where(
+      [n, a],
+      fragment(
+        "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
+        a.actor
+      )
+    )
+    |> join(:inner, [n], activity in assoc(n, :activity))
+    |> join(:left, [n, a], object in Object,
+      on:
         fragment(
-          "? not in (SELECT ap_id FROM users WHERE info->'deactivated' @> 'true')",
-          a.actor
+          "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
+          object.data,
+          a.data
         )
-      )
-      |> join(:inner, [n], activity in assoc(n, :activity))
-      |> join(:left, [n, a], object in Object,
-        on:
-          fragment(
-            "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)",
-            object.data,
-            a.data
-          )
-      )
-      |> preload([n, a, o], activity: {a, object: o})
+    )
+    |> preload([n, a, o], activity: {a, object: o})
+    |> exclude_muted(user, opts)
+    |> exclude_visibility(opts)
+  end
+
+  defp exclude_muted(query, _, %{with_muted: true}) do
+    query
+  end
+
+  defp exclude_muted(query, user, _opts) do
+    query
+    |> where([n, a], a.actor not in ^user.info.muted_notifications)
+    |> where([n, a], a.actor not in ^user.info.blocks)
+    |> where(
+      [n, a],
+      fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks
+    )
+    |> join(:left, [n, a], tm in Pleroma.ThreadMute,
+      on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
+    )
+    |> where([n, a, o, tm], is_nil(tm.user_id))
+  end
 
-    if opts[:with_muted] do
+  @valid_visibilities ~w[direct unlisted public private]
+
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when is_list(visibility) do
+    if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
       query
-    else
-      where(query, [n, a], a.actor not in ^user.info.muted_notifications)
-      |> where([n, a], a.actor not in ^user.info.blocks)
       |> where(
         [n, a],
-        fragment("substring(? from '.*://([^/]*)')", a.actor) not in ^user.info.domain_blocks
-      )
-      |> join(:left, [n, a], tm in Pleroma.ThreadMute,
-        on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data)
+        not fragment(
+          "activity_visibility(?, ?, ?) = ANY (?)",
+          a.actor,
+          a.recipients,
+          a.data,
+          ^visibility
+        )
       )
-      |> where([n, a, o, tm], is_nil(tm.user_id))
+    else
+      Logger.error("Could not exclude visibility to #{visibility}")
+      query
     end
   end
 
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when visibility in @valid_visibilities do
+    query
+    |> where(
+      [n, a],
+      not fragment(
+        "activity_visibility(?, ?, ?) = (?)",
+        a.actor,
+        a.recipients,
+        a.data,
+        ^visibility
+      )
+    )
+  end
+
+  defp exclude_visibility(query, %{exclude_visibilities: visibility})
+       when visibility not in @valid_visibilities do
+    Logger.error("Could not exclude visibility to #{visibility}")
+    query
+  end
+
+  defp exclude_visibility(query, _visibility), do: query
+
   def for_user(user, opts \\ %{}) do
     user
     |> for_user_query(opts)
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 364452b5d..1d34c4d7e 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -596,6 +596,49 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_visibility(query, _visibility), do: query
 
+  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+       when is_list(visibility) do
+    if Enum.all?(visibility, &(&1 in @valid_visibilities)) do
+      from(
+        a in query,
+        where:
+          not fragment(
+            "activity_visibility(?, ?, ?) = ANY (?)",
+            a.actor,
+            a.recipients,
+            a.data,
+            ^visibility
+          )
+      )
+    else
+      Logger.error("Could not exclude visibility to #{visibility}")
+      query
+    end
+  end
+
+  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+       when visibility in @valid_visibilities do
+    from(
+      a in query,
+      where:
+        not fragment(
+          "activity_visibility(?, ?, ?) = ?",
+          a.actor,
+          a.recipients,
+          a.data,
+          ^visibility
+        )
+    )
+  end
+
+  defp exclude_visibility(query, %{"exclude_visibilities" => visibility})
+       when visibility not in @valid_visibilities do
+    Logger.error("Could not exclude visibility to #{visibility}")
+    query
+  end
+
+  defp exclude_visibility(query, _visibility), do: query
+
   defp restrict_thread_visibility(query, _, %{skip_thread_containment: true} = _),
     do: query
 
@@ -960,6 +1003,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     |> restrict_muted_reblogs(opts)
     |> Activity.restrict_deactivated_users()
     |> exclude_poll_votes(opts)
+    |> exclude_visibility(opts)
   end
 
   def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex
index ac01d1ff3..d875a5788 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex
@@ -71,6 +71,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do
   defp cast_params(params) do
     param_types = %{
       exclude_types: {:array, :string},
+      exclude_visibilities: {:array, :string},
       reblogs: :boolean,
       with_muted: :boolean
     }
-- 
cgit v1.2.3


From fbe5a00475331962d7112ed5a0d5cf8bb5bf3ece Mon Sep 17 00:00:00 2001
From: Ekaterina Vaartis <vaartis@cock.li>
Date: Mon, 30 Sep 2019 16:04:30 +0300
Subject: For pleroma.emoji downloaded packs, generate pack.json instead

---
 lib/mix/tasks/pleroma/emoji.ex | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index 881a6f725..1470b9b72 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -158,19 +158,21 @@ defmodule Mix.Tasks.Pleroma.Emoji do
             file_list: files_to_unzip
           )
 
-        IO.puts(IO.ANSI.format(["Writing emoji.txt for ", :bright, pack_name]))
-
-        emoji_txt_str =
-          Enum.map(
-            files,
-            fn {shortcode, path} ->
-              emojo_path = Path.join("/emoji/#{pack_name}", path)
-              "#{shortcode}, #{emojo_path}"
-            end
-          )
-          |> Enum.join("\n")
-
-        File.write!(Path.join(pack_path, "emoji.txt"), emoji_txt_str)
+        IO.puts(IO.ANSI.format(["Writing pack.json for ", :bright, pack_name]))
+
+        pack_json = %{
+          pack: %{
+            "license" => pack["license"],
+            "homepage" => pack["homepage"],
+            "description" => pack["description"],
+            "fallback-src" => pack["src"],
+            "fallback-src-sha256" => pack["src_sha256"],
+            "share-files" => true
+          },
+          files: files
+        }
+
+        File.write!(Path.join(pack_path, "pack.json"), Jason.encode!(pack_json, pretty: true))
       else
         IO.puts(IO.ANSI.format([:bright, :red, "No pack named \"#{pack_name}\" found"]))
       end
-- 
cgit v1.2.3


From 5bd0717de286a918e20c5ef2409e5f4cbcea8524 Mon Sep 17 00:00:00 2001
From: kPherox <admin@mail.kr-kp.com>
Date: Tue, 15 Oct 2019 19:10:22 +0900
Subject: Add `Sec-WebSocket-Protocol` to response header

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 7 +++++++
 1 file changed, 7 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 3c26eb406..1a757363f 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -35,6 +35,13 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
          {_, stream} <- List.keyfind(params, "stream", 0),
          {:ok, user} <- allow_request(stream, [access_token, sec_websocket]),
          topic when is_binary(topic) <- expand_topic(stream, params) do
+      req =
+        if sec_websocket != nil do
+          :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
+        else
+          req
+        end
+
       {:cowboy_websocket, req, %{user: user, topic: topic}, %{idle_timeout: @timeout}}
     else
       {:error, code} ->
-- 
cgit v1.2.3


From e7bb762ec256bb5dc607797c042aa9432a9aa993 Mon Sep 17 00:00:00 2001
From: Alexander <alex.strizhakov@gmail.com>
Date: Tue, 15 Oct 2019 15:16:17 +0300
Subject: don't stream in benchmark env

---
 lib/pleroma/web/streamer/streamer.ex | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/streamer/streamer.ex b/lib/pleroma/web/streamer/streamer.ex
index 8cf719277..2fc7ac8cf 100644
--- a/lib/pleroma/web/streamer/streamer.ex
+++ b/lib/pleroma/web/streamer/streamer.ex
@@ -49,7 +49,7 @@ defmodule Pleroma.Web.Streamer do
     end
   end
 
-  defp handle_should_send(_) do
-    true
-  end
+  defp handle_should_send(:benchmark), do: false
+
+  defp handle_should_send(_), do: true
 end
-- 
cgit v1.2.3


From c10ce113d487d71c4daa6fabcc641a5caa0d04cb Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 16 Oct 2019 12:52:47 +0300
Subject: User search: Remove trigram and refactor the module

- Remove trigram as it tends to rank garbage results highly, resulting
in it prioritized above fts, which gives actually decent results. ACKed
by kaniini and lain on irc.
- Remove a test for handling misspelled requests, since we no longer have
trigram
- Remove a test for searching users with `nil` display names, because it
is unrealistic, we don't accept usernames that are not >1 char strings
- Make rank boosting for followers/followees sane again, previous values
resulted in garbage matches getting on top just because the users are
followers/followees
---
 lib/pleroma/user/search.ex | 152 +++++++++++++++------------------------------
 1 file changed, 50 insertions(+), 102 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index 6fb2c2352..fb2f3fedb 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -4,7 +4,6 @@
 
 defmodule Pleroma.User.Search do
   alias Pleroma.Pagination
-  alias Pleroma.Repo
   alias Pleroma.User
   import Ecto.Query
 
@@ -23,18 +22,10 @@ defmodule Pleroma.User.Search do
 
     maybe_resolve(resolve, for_user, query_string)
 
-    {:ok, results} =
-      Repo.transaction(fn ->
-        Ecto.Adapters.SQL.query(
-          Repo,
-          "select set_limit(#{@similarity_threshold})",
-          []
-        )
-
-        query_string
-        |> search_query(for_user, following)
-        |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
-      end)
+    results =
+      query_string
+      |> search_query(for_user, following)
+      |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => result_limit}, :offset)
 
     results
   end
@@ -56,15 +47,52 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_blocked_domains(for_user)
-    |> search_subqueries(query_string)
-    |> union_subqueries
-    |> distinct_query()
-    |> boost_search_rank_query(for_user)
+    |> fts_subquery(query_string)
     |> subquery()
+    |> where([u], u.search_rank > @similarity_threshold)
+    |> boost_search_rank(for_user)
     |> order_by(desc: :search_rank)
     |> maybe_restrict_local(for_user)
   end
 
+  @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/
+  defp fts_subquery(query, query_string) do
+    {nickname_weight, name_weight} =
+      if String.match?(query_string, @nickname_regex) do
+        {"A", "B"}
+      else
+        {"B", "A"}
+      end
+
+    query_string = to_tsquery(query_string)
+
+    from(
+      u in query,
+      select_merge: %{
+        search_rank:
+          fragment(
+            """
+            ts_rank_cd((setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)), to_tsquery('simple', ?))
+            """,
+            u.name,
+            ^name_weight,
+            u.nickname,
+            ^nickname_weight,
+            ^query_string
+          )
+      }
+    )
+  end
+
+  defp to_tsquery(query_string) do
+    String.trim_trailing(query_string, "@" <> local_domain())
+    |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
+    |> String.trim()
+    |> String.split()
+    |> Enum.map(&(&1 <> ":*"))
+    |> Enum.join(" | ")
+  end
+
   defp base_query(_user, false), do: User
   defp base_query(user, true), do: User.get_followers_query(user)
 
@@ -87,21 +115,6 @@ defmodule Pleroma.User.Search do
 
   defp filter_blocked_domains(query, _), do: query
 
-  defp union_subqueries({fts_subquery, trigram_subquery}) do
-    from(s in trigram_subquery, union_all: ^fts_subquery)
-  end
-
-  defp search_subqueries(base_query, query_string) do
-    {
-      fts_search_subquery(base_query, query_string),
-      trigram_search_subquery(base_query, query_string)
-    }
-  end
-
-  defp distinct_query(q) do
-    from(s in subquery(q), order_by: s.search_type, distinct: s.id)
-  end
-
   defp maybe_resolve(true, user, query) do
     case {limit(), user} do
       {:all, _} -> :noop
@@ -126,9 +139,9 @@ defmodule Pleroma.User.Search do
 
   defp restrict_local(q), do: where(q, [u], u.local == true)
 
-  defp boost_search_rank_query(query, nil), do: query
+  defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
 
-  defp boost_search_rank_query(query, for_user) do
+  defp boost_search_rank(query, %User{} = for_user) do
     friends_ids = User.get_friends_ids(for_user)
     followers_ids = User.get_followers_ids(for_user)
 
@@ -137,8 +150,8 @@ defmodule Pleroma.User.Search do
         search_rank:
           fragment(
             """
-             CASE WHEN (?) THEN 0.5 + (?) * 1.3
-             WHEN (?) THEN 0.5 + (?) * 1.2
+             CASE WHEN (?) THEN (?) * 1.5
+             WHEN (?) THEN (?) * 1.3
              WHEN (?) THEN (?) * 1.1
              ELSE (?) END
             """,
@@ -154,70 +167,5 @@ defmodule Pleroma.User.Search do
     )
   end
 
-  @spec fts_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
-  defp fts_search_subquery(query, term) do
-    processed_query =
-      String.trim_trailing(term, "@" <> local_domain())
-      |> String.replace(~r/[!-\/|@|[-`|{-~|:-?]+/, " ")
-      |> String.trim()
-      |> String.split()
-      |> Enum.map(&(&1 <> ":*"))
-      |> Enum.join(" | ")
-
-    from(
-      u in query,
-      select_merge: %{
-        search_type: ^0,
-        search_rank:
-          fragment(
-            """
-            ts_rank_cd(
-              setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
-              setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B'),
-              to_tsquery('simple', ?),
-              32
-            )
-            """,
-            u.nickname,
-            u.name,
-            ^processed_query
-          )
-      },
-      where:
-        fragment(
-          """
-            (setweight(to_tsvector('simple', regexp_replace(?, '\\W', ' ', 'g')), 'A') ||
-            setweight(to_tsvector('simple', regexp_replace(coalesce(?, ''), '\\W', ' ', 'g')), 'B')) @@ to_tsquery('simple', ?)
-          """,
-          u.nickname,
-          u.name,
-          ^processed_query
-        )
-    )
-    |> User.restrict_deactivated()
-  end
-
-  @spec trigram_search_subquery(User.t() | Ecto.Query.t(), String.t()) :: Ecto.Query.t()
-  defp trigram_search_subquery(query, term) do
-    term = String.trim_trailing(term, "@" <> local_domain())
-
-    from(
-      u in query,
-      select_merge: %{
-        # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason
-        search_type: fragment("?", 1),
-        search_rank:
-          fragment(
-            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
-            ^term,
-            u.nickname,
-            u.name
-          )
-      },
-      where: fragment("trim(? || ' ' || coalesce(?, '')) % ?", u.nickname, u.name, ^term)
-    )
-    |> User.restrict_deactivated()
-  end
-
-  defp local_domain, do: Pleroma.Config.get([Pleroma.Web.Endpoint, :url, :host])
+  defp boost_search_rank(query, _for_user), do: query
 end
-- 
cgit v1.2.3


From 0a5175ecbb796cf3c192a42dc561debd73640a54 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 16 Oct 2019 13:49:33 +0300
Subject: Order fts results by trigram

---
 lib/pleroma/user/search.ex | 48 +++++++++++++++++++++++++++++-----------------
 1 file changed, 30 insertions(+), 18 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/user/search.ex b/lib/pleroma/user/search.ex
index fb2f3fedb..0d697fe3d 100644
--- a/lib/pleroma/user/search.ex
+++ b/lib/pleroma/user/search.ex
@@ -7,7 +7,6 @@ defmodule Pleroma.User.Search do
   alias Pleroma.User
   import Ecto.Query
 
-  @similarity_threshold 0.25
   @limit 20
 
   def search(query_string, opts \\ []) do
@@ -47,16 +46,16 @@ defmodule Pleroma.User.Search do
     |> base_query(following)
     |> filter_blocked_user(for_user)
     |> filter_blocked_domains(for_user)
-    |> fts_subquery(query_string)
-    |> subquery()
-    |> where([u], u.search_rank > @similarity_threshold)
+    |> fts_search(query_string)
+    |> trigram_rank(query_string)
     |> boost_search_rank(for_user)
+    |> subquery()
     |> order_by(desc: :search_rank)
     |> maybe_restrict_local(for_user)
   end
 
   @nickname_regex ~r/^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~\-@]+$/
-  defp fts_subquery(query, query_string) do
+  defp fts_search(query, query_string) do
     {nickname_weight, name_weight} =
       if String.match?(query_string, @nickname_regex) do
         {"A", "B"}
@@ -68,19 +67,17 @@ defmodule Pleroma.User.Search do
 
     from(
       u in query,
-      select_merge: %{
-        search_rank:
-          fragment(
-            """
-            ts_rank_cd((setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)), to_tsquery('simple', ?))
-            """,
-            u.name,
-            ^name_weight,
-            u.nickname,
-            ^nickname_weight,
-            ^query_string
-          )
-      }
+      where:
+        fragment(
+          """
+          (setweight(to_tsvector('simple', ?), ?) || setweight(to_tsvector('simple', ?), ?)) @@ to_tsquery('simple', ?)
+          """,
+          u.name,
+          ^name_weight,
+          u.nickname,
+          ^nickname_weight,
+          ^query_string
+        )
     )
   end
 
@@ -93,6 +90,21 @@ defmodule Pleroma.User.Search do
     |> Enum.join(" | ")
   end
 
+  defp trigram_rank(query, query_string) do
+    from(
+      u in query,
+      select_merge: %{
+        search_rank:
+          fragment(
+            "similarity(?, trim(? || ' ' || coalesce(?, '')))",
+            ^query_string,
+            u.nickname,
+            u.name
+          )
+      }
+    )
+  end
+
   defp base_query(_user, false), do: User
   defp base_query(user, true), do: User.get_followers_query(user)
 
-- 
cgit v1.2.3


From 359dd1890e6afcf80584021eaa2421336614dd07 Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Thu, 17 Oct 2019 15:25:15 +0300
Subject: Mastodon API: Mark the conversation as read for the author when they
 send a new direct message

---
 lib/pleroma/conversation/participation.ex    |  6 ++++++
 lib/pleroma/web/activity_pub/activity_pub.ex | 23 +++++++++++++++++------
 2 files changed, 23 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex
index ab81f3217..e17f49e58 100644
--- a/lib/pleroma/conversation/participation.ex
+++ b/lib/pleroma/conversation/participation.ex
@@ -48,6 +48,12 @@ defmodule Pleroma.Conversation.Participation do
     |> validate_required([:read])
   end
 
+  def mark_as_read(%User{} = user, %Conversation{} = conversation) do
+    with %__MODULE__{} = participation <- for_user_and_conversation(user, conversation) do
+      mark_as_read(participation)
+    end
+  end
+
   def mark_as_read(participation) do
     participation
     |> read_cng(%{read: true})
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 091ec2588..d391732a2 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -7,6 +7,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.Activity.Ir.Topics
   alias Pleroma.Config
   alias Pleroma.Conversation
+  alias Pleroma.Conversation.Participation
   alias Pleroma.Notification
   alias Pleroma.Object
   alias Pleroma.Object.Containment
@@ -153,11 +154,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
       Notification.create_notifications(activity)
 
-      participations =
-        activity
-        |> Conversation.create_or_bump_for()
-        |> get_participations()
-
+      conversation = create_or_bump_conversation(activity, map["actor"])
+      participations = get_participations(conversation)
       stream_out(activity)
       stream_out_participations(participations)
       {:ok, activity}
@@ -182,7 +180,20 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     end
   end
 
-  defp get_participations({:ok, %{participations: participations}}), do: participations
+  defp create_or_bump_conversation(activity, actor) do
+    with {:ok, conversation} <- Conversation.create_or_bump_for(activity),
+         %User{} = user <- User.get_cached_by_ap_id(actor),
+         Participation.mark_as_read(user, conversation) do
+      {:ok, conversation}
+    end
+  end
+
+  defp get_participations({:ok, conversation}) do
+    conversation
+    |> Repo.preload(:participations, force: true)
+    |> Map.get(:participations)
+  end
+
   defp get_participations(_), do: []
 
   def stream_out_participations(participations) do
-- 
cgit v1.2.3


From 733b73b71c2a13a2bfbf4c0a0d35ddfa140a1ce0 Mon Sep 17 00:00:00 2001
From: kaniini <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 04:36:37 +0000
Subject: Apply suggestion to lib/pleroma/web/mastodon_api/websocket_handler.ex

---
 lib/pleroma/web/mastodon_api/websocket_handler.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/websocket_handler.ex b/lib/pleroma/web/mastodon_api/websocket_handler.ex
index 1a757363f..a400d1c8d 100644
--- a/lib/pleroma/web/mastodon_api/websocket_handler.ex
+++ b/lib/pleroma/web/mastodon_api/websocket_handler.ex
@@ -36,7 +36,7 @@ defmodule Pleroma.Web.MastodonAPI.WebsocketHandler do
          {:ok, user} <- allow_request(stream, [access_token, sec_websocket]),
          topic when is_binary(topic) <- expand_topic(stream, params) do
       req =
-        if sec_websocket != nil do
+        if sec_websocket do
           :cowboy_req.set_resp_header("sec-websocket-protocol", sec_websocket, req)
         else
           req
-- 
cgit v1.2.3


From 3c6fd0bb991c865aad2fb27bc798b2b3c3c248d2 Mon Sep 17 00:00:00 2001
From: "Haelwenn (lanodan) Monnier" <contact@hacktivis.me>
Date: Fri, 18 Oct 2019 12:29:42 +0200
Subject: upload.ex: Remove deprecated configuration

---
 lib/pleroma/upload.ex | 33 +--------------------------------
 1 file changed, 1 insertion(+), 32 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/upload.ex b/lib/pleroma/upload.ex
index 9f0adde5b..2e0986197 100644
--- a/lib/pleroma/upload.ex
+++ b/lib/pleroma/upload.ex
@@ -105,7 +105,7 @@ defmodule Pleroma.Upload do
           {Pleroma.Config.get!([:instance, :upload_limit]), "Document"}
       end
 
-    opts = %{
+    %{
       activity_type: Keyword.get(opts, :activity_type, activity_type),
       size_limit: Keyword.get(opts, :size_limit, size_limit),
       uploader: Keyword.get(opts, :uploader, Pleroma.Config.get([__MODULE__, :uploader])),
@@ -118,37 +118,6 @@ defmodule Pleroma.Upload do
           Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url())
         )
     }
-
-    # TODO: 1.0+ : remove old config compatibility
-    opts =
-      if Pleroma.Config.get([__MODULE__, :strip_exif]) == true &&
-           !Enum.member?(opts.filters, Pleroma.Upload.Filter.Mogrify) do
-        Logger.warn("""
-        Pleroma: configuration `:instance, :strip_exif` is deprecated, please instead set:
-
-          :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Mogrify]]
-
-          :pleroma, Pleroma.Upload.Filter.Mogrify, args: ["strip", "auto-orient"]
-        """)
-
-        Pleroma.Config.put([Pleroma.Upload.Filter.Mogrify], args: ["strip", "auto-orient"])
-        Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Mogrify])
-      else
-        opts
-      end
-
-    if Pleroma.Config.get([:instance, :dedupe_media]) == true &&
-         !Enum.member?(opts.filters, Pleroma.Upload.Filter.Dedupe) do
-      Logger.warn("""
-      Pleroma: configuration `:instance, :dedupe_media` is deprecated, please instead set:
-
-      :pleroma, Pleroma.Upload, [filters: [Pleroma.Upload.Filter.Dedupe]]
-      """)
-
-      Map.put(opts, :filters, opts.filters ++ [Pleroma.Upload.Filter.Dedupe])
-    else
-      opts
-    end
   end
 
   defp prepare_upload(%Plug.Upload{} = file, opts) do
-- 
cgit v1.2.3


From 39e996528c1e7551675c0d0f140dcfa01e671004 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 18 Oct 2019 14:11:30 +0300
Subject: Fix a migration wiping user info of users that don't have any mutes

And introduce safe_jsonb_set
---
 lib/mix/tasks/pleroma/database.ex | 4 ++--
 lib/pleroma/object.ex             | 4 ++--
 lib/pleroma/user.ex               | 6 +++---
 3 files changed, 7 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index cfd9eeada..8a827ca80 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Database do
     Logger.info("Removing embedded objects")
 
     Repo.query!(
-      "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
+      "update activities set data = safe_jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
       [],
       timeout: :infinity
     )
@@ -126,7 +126,7 @@ defmodule Mix.Tasks.Pleroma.Database do
         set: [
           data:
             fragment(
-              "jsonb_set(?, '{likes}', '[]'::jsonb, true)",
+              "safe_jsonb_set(?, '{likes}', '[]'::jsonb, true)",
               object.data
             )
         ]
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index cdfbacb0e..d9b41d710 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -181,7 +181,7 @@ defmodule Pleroma.Object do
         data:
           fragment(
             """
-            jsonb_set(?, '{repliesCount}',
+            safe_jsonb_set(?, '{repliesCount}',
               (coalesce((?->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
             """,
             o.data,
@@ -204,7 +204,7 @@ defmodule Pleroma.Object do
         data:
           fragment(
             """
-            jsonb_set(?, '{repliesCount}',
+            safe_jsonb_set(?, '{repliesCount}',
               (greatest(0, (?->>'repliesCount')::int - 1))::varchar::jsonb, true)
             """,
             o.data,
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index 917bcf2ef..d0d3e1f9a 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -727,7 +727,7 @@ defmodule Pleroma.User do
       set: [
         info:
           fragment(
-            "jsonb_set(?, '{note_count}', ((?->>'note_count')::int + 1)::varchar::jsonb, true)",
+            "safe_jsonb_set(?, '{note_count}', ((?->>'note_count')::int + 1)::varchar::jsonb, true)",
             u.info,
             u.info
           )
@@ -748,7 +748,7 @@ defmodule Pleroma.User do
       set: [
         info:
           fragment(
-            "jsonb_set(?, '{note_count}', (greatest(0, (?->>'note_count')::int - 1))::varchar::jsonb, true)",
+            "safe_jsonb_set(?, '{note_count}', (greatest(0, (?->>'note_count')::int - 1))::varchar::jsonb, true)",
             u.info,
             u.info
           )
@@ -818,7 +818,7 @@ defmodule Pleroma.User do
         set: [
           info:
             fragment(
-              "jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
+              "safe_jsonb_set(?, '{follower_count}', ?::varchar::jsonb, true)",
               u.info,
               s.count
             )
-- 
cgit v1.2.3


From 25b7ff56c371fa3b405fa339aaac6d4f0876ed85 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 22:52:46 +0000
Subject: application: don't start Federator.init/1 anymore

---
 lib/pleroma/application.ex | 10 ----------
 1 file changed, 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index 0bf218bc7..d681eecc8 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -161,11 +161,6 @@ defmodule Pleroma.Application do
         id: :web_push_init,
         start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
         restart: :temporary
-      },
-      %{
-        id: :federator_init,
-        start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
-        restart: :temporary
       }
     ]
   end
@@ -177,11 +172,6 @@ defmodule Pleroma.Application do
         start: {Task, :start_link, [&Pleroma.Web.Push.init/0]},
         restart: :temporary
       },
-      %{
-        id: :federator_init,
-        start: {Task, :start_link, [&Pleroma.Web.Federator.init/0]},
-        restart: :temporary
-      },
       %{
         id: :internal_fetch_init,
         start: {Task, :start_link, [&Pleroma.Web.ActivityPub.InternalFetchActor.init/0]},
-- 
cgit v1.2.3


From b16a460916e0384ec6ed34f80a390b9a4ed4d96d Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 22:53:45 +0000
Subject: federator: remove websub stuff

---
 lib/pleroma/web/federator/federator.ex   | 42 --------------------------------
 lib/pleroma/workers/subscriber_worker.ex | 26 --------------------
 2 files changed, 68 deletions(-)
 delete mode 100644 lib/pleroma/workers/subscriber_worker.ex

(limited to 'lib')

diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 1a2da014a..8227d1a3a 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -11,18 +11,11 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Federator.Publisher
   alias Pleroma.Web.OStatus
-  alias Pleroma.Web.Websub
   alias Pleroma.Workers.PublisherWorker
   alias Pleroma.Workers.ReceiverWorker
-  alias Pleroma.Workers.SubscriberWorker
 
   require Logger
 
-  def init do
-    # To do: consider removing this call in favor of scheduled execution (`quantum`-based)
-    refresh_subscriptions(schedule_in: 60)
-  end
-
   @doc "Addresses [memory leaks on recursive replies fetching](https://git.pleroma.social/pleroma/pleroma/issues/161)"
   # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
   def allowed_incoming_reply_depth?(depth) do
@@ -53,18 +46,6 @@ defmodule Pleroma.Web.Federator do
     PublisherWorker.enqueue("publish", %{"activity_id" => activity.id})
   end
 
-  def verify_websub(websub) do
-    SubscriberWorker.enqueue("verify_websub", %{"websub_id" => websub.id})
-  end
-
-  def request_subscription(websub) do
-    SubscriberWorker.enqueue("request_subscription", %{"websub_id" => websub.id})
-  end
-
-  def refresh_subscriptions(worker_args \\ []) do
-    SubscriberWorker.enqueue("refresh_subscriptions", %{}, worker_args ++ [max_attempts: 1])
-  end
-
   # Job Worker Callbacks
 
   @spec perform(atom(), module(), any()) :: {:ok, any()} | {:error, any()}
@@ -111,29 +92,6 @@ defmodule Pleroma.Web.Federator do
     end
   end
 
-  def perform(:request_subscription, websub) do
-    Logger.debug("Refreshing #{websub.topic}")
-
-    with {:ok, websub} <- Websub.request_subscription(websub) do
-      Logger.debug("Successfully refreshed #{websub.topic}")
-    else
-      _e -> Logger.debug("Couldn't refresh #{websub.topic}")
-    end
-  end
-
-  def perform(:verify_websub, websub) do
-    Logger.debug(fn ->
-      "Running WebSub verification for #{websub.id} (#{websub.topic}, #{websub.callback})"
-    end)
-
-    Websub.verify(websub)
-  end
-
-  def perform(:refresh_subscriptions) do
-    Logger.debug("Federator running refresh subscriptions")
-    Websub.refresh_subscriptions()
-  end
-
   def ap_enabled_actor(id) do
     user = User.get_cached_by_ap_id(id)
 
diff --git a/lib/pleroma/workers/subscriber_worker.ex b/lib/pleroma/workers/subscriber_worker.ex
deleted file mode 100644
index fc490e300..000000000
--- a/lib/pleroma/workers/subscriber_worker.ex
+++ /dev/null
@@ -1,26 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Workers.SubscriberWorker do
-  alias Pleroma.Repo
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.Websub
-
-  use Pleroma.Workers.WorkerHelper, queue: "federator_outgoing"
-
-  @impl Oban.Worker
-  def perform(%{"op" => "refresh_subscriptions"}, _job) do
-    Federator.perform(:refresh_subscriptions)
-  end
-
-  def perform(%{"op" => "request_subscription", "websub_id" => websub_id}, _job) do
-    websub = Repo.get(Websub.WebsubClientSubscription, websub_id)
-    Federator.perform(:request_subscription, websub)
-  end
-
-  def perform(%{"op" => "verify_websub", "websub_id" => websub_id}, _job) do
-    websub = Repo.get(Websub.WebsubServerSubscription, websub_id)
-    Federator.perform(:verify_websub, websub)
-  end
-end
-- 
cgit v1.2.3


From 4f82e42e4e581b32cd25fff862f880f7f5a87b81 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 22:57:37 +0000
Subject: websub: remove entirely

---
 lib/pleroma/user.ex                                |   7 -
 lib/pleroma/web/activity_pub/transmogrifier.ex     |  15 -
 lib/pleroma/web/ostatus/ostatus.ex                 |   5 +-
 lib/pleroma/web/router.ex                          |   3 -
 lib/pleroma/web/templates/feed/feed/feed.xml.eex   |   1 -
 lib/pleroma/web/websub/websub.ex                   | 332 ---------------------
 .../web/websub/websub_client_subscription.ex       |  20 --
 lib/pleroma/web/websub/websub_controller.ex        |  99 ------
 .../web/websub/websub_server_subscription.ex       |  17 --
 9 files changed, 1 insertion(+), 498 deletions(-)
 delete mode 100644 lib/pleroma/web/websub/websub.ex
 delete mode 100644 lib/pleroma/web/websub/websub_client_subscription.ex
 delete mode 100644 lib/pleroma/web/websub/websub_controller.ex
 delete mode 100644 lib/pleroma/web/websub/websub_server_subscription.ex

(limited to 'lib')

diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index d0d3e1f9a..cef011c68 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -28,7 +28,6 @@ defmodule Pleroma.User do
   alias Pleroma.Web.OAuth
   alias Pleroma.Web.OStatus
   alias Pleroma.Web.RelMe
-  alias Pleroma.Web.Websub
   alias Pleroma.Workers.BackgroundWorker
 
   require Logger
@@ -437,12 +436,6 @@ defmodule Pleroma.User do
         {:error, "Could not follow user: #{followed.nickname} blocked you."}
 
       true ->
-        benchmark? = Pleroma.Config.get([:env]) == :benchmark
-
-        if !followed.local && follower.local && !ap_enabled?(followed) && !benchmark? do
-          Websub.subscribe(follower, followed)
-        end
-
         q =
           from(u in User,
             where: u.id == ^follower.id,
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index b56343beb..2c1ce9c55 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -1073,8 +1073,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
     Repo.update_all(q, [])
 
-    maybe_retire_websub(user.ap_id)
-
     q =
       from(
         a in Activity,
@@ -1117,19 +1115,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
     |> User.update_and_set_cache()
   end
 
-  def maybe_retire_websub(ap_id) do
-    # some sanity checks
-    if is_binary(ap_id) && String.length(ap_id) > 8 do
-      q =
-        from(
-          ws in Pleroma.Web.Websub.WebsubClientSubscription,
-          where: fragment("? like ?", ws.topic, ^"#{ap_id}%")
-        )
-
-      Repo.delete_all(q)
-    end
-  end
-
   def maybe_fix_user_url(%{"url" => url} = data) when is_map(url) do
     Map.put(data, "url", url["href"])
   end
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index 5de1ceef3..3dfc9b580 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -19,7 +19,6 @@ defmodule Pleroma.Web.OStatus do
   alias Pleroma.Web.OStatus.NoteHandler
   alias Pleroma.Web.OStatus.UnfollowHandler
   alias Pleroma.Web.WebFinger
-  alias Pleroma.Web.Websub
 
   def is_representable?(%Activity{} = activity) do
     object = Object.normalize(activity)
@@ -314,11 +313,9 @@ defmodule Pleroma.Web.OStatus do
 
   @spec gather_user_info(String.t()) :: {:ok, map()} | {:error, any()}
   def gather_user_info(username) do
-    with {:ok, webfinger_data} <- WebFinger.finger(username),
-         {:ok, feed_data} <- Websub.gather_feed_data(webfinger_data["topic"]) do
+    with {:ok, webfinger_data} <- WebFinger.finger(username) do
       data =
         webfinger_data
-        |> Map.merge(feed_data)
         |> Map.put("fqn", username)
 
       {:ok, data}
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 80651f3ff..77fe938d5 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -509,9 +509,6 @@ defmodule Pleroma.Web.Router do
     get("/users/:nickname", Feed.FeedController, :feed_redirect)
 
     post("/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming)
-    post("/push/hub/:nickname", Websub.WebsubController, :websub_subscription_request)
-    get("/push/subscriptions/:id", Websub.WebsubController, :websub_subscription_confirmation)
-    post("/push/subscriptions/:id", Websub.WebsubController, :websub_incoming)
 
     get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe)
   end
diff --git a/lib/pleroma/web/templates/feed/feed/feed.xml.eex b/lib/pleroma/web/templates/feed/feed/feed.xml.eex
index fbfdc46b5..692cfbd7f 100644
--- a/lib/pleroma/web/templates/feed/feed/feed.xml.eex
+++ b/lib/pleroma/web/templates/feed/feed/feed.xml.eex
@@ -10,7 +10,6 @@
   <title><%= @user.nickname <> "'s timeline" %></title>
   <updated><%= most_recent_update(@activities, @user) %></updated>
   <logo><%= logo(@user) %></logo>
-  <link rel="hub" href="<%= websub_url(@conn, :websub_subscription_request, @user.nickname) %>"/>
   <link rel="salmon" href="<%= o_status_url(@conn, :salmon_incoming, @user.nickname) %>"/>
   <link rel="self" href="<%= '#{feed_url(@conn, :feed, @user.nickname)}.atom' %>" type="application/atom+xml"/>
 
diff --git a/lib/pleroma/web/websub/websub.ex b/lib/pleroma/web/websub/websub.ex
deleted file mode 100644
index b61f388b8..000000000
--- a/lib/pleroma/web/websub/websub.ex
+++ /dev/null
@@ -1,332 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Websub do
-  alias Ecto.Changeset
-  alias Pleroma.Activity
-  alias Pleroma.HTTP
-  alias Pleroma.Instances
-  alias Pleroma.Repo
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Web.Endpoint
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.OStatus.FeedRepresenter
-  alias Pleroma.Web.Router.Helpers
-  alias Pleroma.Web.Websub.WebsubClientSubscription
-  alias Pleroma.Web.Websub.WebsubServerSubscription
-  alias Pleroma.Web.XML
-  require Logger
-
-  import Ecto.Query
-
-  @behaviour Pleroma.Web.Federator.Publisher
-
-  def verify(subscription, getter \\ &HTTP.get/3) do
-    challenge = Base.encode16(:crypto.strong_rand_bytes(8))
-    lease_seconds = NaiveDateTime.diff(subscription.valid_until, subscription.updated_at)
-    lease_seconds = lease_seconds |> to_string
-
-    params = %{
-      "hub.challenge": challenge,
-      "hub.lease_seconds": lease_seconds,
-      "hub.topic": subscription.topic,
-      "hub.mode": "subscribe"
-    }
-
-    url = hd(String.split(subscription.callback, "?"))
-    query = URI.parse(subscription.callback).query || ""
-    params = Map.merge(params, URI.decode_query(query))
-
-    with {:ok, response} <- getter.(url, [], params: params),
-         ^challenge <- response.body do
-      changeset = Changeset.change(subscription, %{state: "active"})
-      Repo.update(changeset)
-    else
-      e ->
-        Logger.debug("Couldn't verify subscription")
-        Logger.debug(inspect(e))
-        {:error, subscription}
-    end
-  end
-
-  @supported_activities [
-    "Create",
-    "Follow",
-    "Like",
-    "Announce",
-    "Undo",
-    "Delete"
-  ]
-
-  def is_representable?(%Activity{data: %{"type" => type}} = activity)
-      when type in @supported_activities,
-      do: Visibility.is_public?(activity)
-
-  def is_representable?(_), do: false
-
-  def publish(topic, user, %{data: %{"type" => type}} = activity)
-      when type in @supported_activities do
-    response =
-      user
-      |> FeedRepresenter.to_simple_form([activity], [user])
-      |> :xmerl.export_simple(:xmerl_xml)
-      |> to_string
-
-    query =
-      from(
-        sub in WebsubServerSubscription,
-        where: sub.topic == ^topic and sub.state == "active",
-        where: fragment("? > (NOW() at time zone 'UTC')", sub.valid_until)
-      )
-
-    subscriptions = Repo.all(query)
-
-    callbacks = Enum.map(subscriptions, & &1.callback)
-    reachable_callbacks_metadata = Instances.filter_reachable(callbacks)
-    reachable_callbacks = Map.keys(reachable_callbacks_metadata)
-
-    subscriptions
-    |> Enum.filter(&(&1.callback in reachable_callbacks))
-    |> Enum.each(fn sub ->
-      data = %{
-        xml: response,
-        topic: topic,
-        callback: sub.callback,
-        secret: sub.secret,
-        unreachable_since: reachable_callbacks_metadata[sub.callback]
-      }
-
-      Publisher.enqueue_one(__MODULE__, data)
-    end)
-  end
-
-  def publish(_, _, _), do: ""
-
-  def publish(actor, activity), do: publish(Pleroma.Web.OStatus.feed_path(actor), actor, activity)
-
-  def sign(secret, doc) do
-    :crypto.hmac(:sha, secret, to_string(doc)) |> Base.encode16() |> String.downcase()
-  end
-
-  def incoming_subscription_request(user, %{"hub.mode" => "subscribe"} = params) do
-    with {:ok, topic} <- valid_topic(params, user),
-         {:ok, lease_time} <- lease_time(params),
-         secret <- params["hub.secret"],
-         callback <- params["hub.callback"] do
-      subscription = get_subscription(topic, callback)
-
-      data = %{
-        state: subscription.state || "requested",
-        topic: topic,
-        secret: secret,
-        callback: callback
-      }
-
-      change = Changeset.change(subscription, data)
-      websub = Repo.insert_or_update!(change)
-
-      change =
-        Changeset.change(websub, %{valid_until: NaiveDateTime.add(websub.updated_at, lease_time)})
-
-      websub = Repo.update!(change)
-
-      Federator.verify_websub(websub)
-
-      {:ok, websub}
-    else
-      {:error, reason} ->
-        Logger.debug("Couldn't create subscription")
-        Logger.debug(inspect(reason))
-
-        {:error, reason}
-    end
-  end
-
-  def incoming_subscription_request(user, params) do
-    Logger.info("Unhandled WebSub request for #{user.nickname}: #{inspect(params)}")
-
-    {:error, "Invalid WebSub request"}
-  end
-
-  defp get_subscription(topic, callback) do
-    Repo.get_by(WebsubServerSubscription, topic: topic, callback: callback) ||
-      %WebsubServerSubscription{}
-  end
-
-  # Temp hack for mastodon.
-  defp lease_time(%{"hub.lease_seconds" => ""}) do
-    # three days
-    {:ok, 60 * 60 * 24 * 3}
-  end
-
-  defp lease_time(%{"hub.lease_seconds" => lease_seconds}) do
-    {:ok, String.to_integer(lease_seconds)}
-  end
-
-  defp lease_time(_) do
-    # three days
-    {:ok, 60 * 60 * 24 * 3}
-  end
-
-  defp valid_topic(%{"hub.topic" => topic}, user) do
-    if topic == OStatus.feed_path(user) do
-      {:ok, OStatus.feed_path(user)}
-    else
-      {:error, "Wrong topic requested, expected #{OStatus.feed_path(user)}, got #{topic}"}
-    end
-  end
-
-  def subscribe(subscriber, subscribed, requester \\ &request_subscription/1) do
-    topic = subscribed.info.topic
-    # FIXME: Race condition, use transactions
-    {:ok, subscription} =
-      with subscription when not is_nil(subscription) <-
-             Repo.get_by(WebsubClientSubscription, topic: topic) do
-        subscribers = [subscriber.ap_id | subscription.subscribers] |> Enum.uniq()
-        change = Ecto.Changeset.change(subscription, %{subscribers: subscribers})
-        Repo.update(change)
-      else
-        _e ->
-          subscription = %WebsubClientSubscription{
-            topic: topic,
-            hub: subscribed.info.hub,
-            subscribers: [subscriber.ap_id],
-            state: "requested",
-            secret: :crypto.strong_rand_bytes(8) |> Base.url_encode64(),
-            user: subscribed
-          }
-
-          Repo.insert(subscription)
-      end
-
-    requester.(subscription)
-  end
-
-  def gather_feed_data(topic, getter \\ &HTTP.get/1) do
-    with {:ok, response} <- getter.(topic),
-         status when status in 200..299 <- response.status,
-         body <- response.body,
-         doc <- XML.parse_document(body),
-         uri when not is_nil(uri) <- XML.string_from_xpath("/feed/author[1]/uri", doc),
-         hub when not is_nil(hub) <- XML.string_from_xpath(~S{/feed/link[@rel="hub"]/@href}, doc) do
-      name = XML.string_from_xpath("/feed/author[1]/name", doc)
-      preferred_username = XML.string_from_xpath("/feed/author[1]/poco:preferredUsername", doc)
-      display_name = XML.string_from_xpath("/feed/author[1]/poco:displayName", doc)
-      avatar = OStatus.make_avatar_object(doc)
-      bio = XML.string_from_xpath("/feed/author[1]/summary", doc)
-
-      {:ok,
-       %{
-         "uri" => uri,
-         "hub" => hub,
-         "nickname" => preferred_username || name,
-         "name" => display_name || name,
-         "host" => URI.parse(uri).host,
-         "avatar" => avatar,
-         "bio" => bio
-       }}
-    else
-      e ->
-        {:error, e}
-    end
-  end
-
-  def request_subscription(websub, poster \\ &HTTP.post/3, timeout \\ 10_000) do
-    data = [
-      "hub.mode": "subscribe",
-      "hub.topic": websub.topic,
-      "hub.secret": websub.secret,
-      "hub.callback": Helpers.websub_url(Endpoint, :websub_subscription_confirmation, websub.id)
-    ]
-
-    # This checks once a second if we are confirmed yet
-    websub_checker = fn ->
-      helper = fn helper ->
-        :timer.sleep(1000)
-        websub = Repo.get_by(WebsubClientSubscription, id: websub.id, state: "accepted")
-        if websub, do: websub, else: helper.(helper)
-      end
-
-      helper.(helper)
-    end
-
-    task = Task.async(websub_checker)
-
-    with {:ok, %{status: 202}} <-
-           poster.(websub.hub, {:form, data}, "Content-type": "application/x-www-form-urlencoded"),
-         {:ok, websub} <- Task.yield(task, timeout) do
-      {:ok, websub}
-    else
-      e ->
-        Task.shutdown(task)
-
-        change = Ecto.Changeset.change(websub, %{state: "rejected"})
-        {:ok, websub} = Repo.update(change)
-
-        Logger.debug(fn -> "Couldn't confirm subscription: #{inspect(websub)}" end)
-        Logger.debug(fn -> "error: #{inspect(e)}" end)
-
-        {:error, websub}
-    end
-  end
-
-  def refresh_subscriptions(delta \\ 60 * 60 * 24) do
-    Logger.debug("Refreshing subscriptions")
-
-    cut_off = NaiveDateTime.add(NaiveDateTime.utc_now(), delta)
-
-    query = from(sub in WebsubClientSubscription, where: sub.valid_until < ^cut_off)
-
-    subs = Repo.all(query)
-
-    Enum.each(subs, fn sub ->
-      Federator.request_subscription(sub)
-    end)
-  end
-
-  def publish_one(%{xml: xml, topic: topic, callback: callback, secret: secret} = params) do
-    signature = sign(secret || "", xml)
-    Logger.info(fn -> "Pushing #{topic} to #{callback}" end)
-
-    with {:ok, %{status: code}} when code in 200..299 <-
-           HTTP.post(
-             callback,
-             xml,
-             [
-               {"Content-Type", "application/atom+xml"},
-               {"X-Hub-Signature", "sha1=#{signature}"}
-             ]
-           ) do
-      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
-        do: Instances.set_reachable(callback)
-
-      Logger.info(fn -> "Pushed to #{callback}, code #{code}" end)
-      {:ok, code}
-    else
-      {_post_result, response} ->
-        unless params[:unreachable_since], do: Instances.set_reachable(callback)
-        Logger.debug(fn -> "Couldn't push to #{callback}, #{inspect(response)}" end)
-        {:error, response}
-    end
-  end
-
-  def gather_webfinger_links(%User{} = user) do
-    [
-      %{
-        "rel" => "http://schemas.google.com/g/2010#updates-from",
-        "type" => "application/atom+xml",
-        "href" => OStatus.feed_path(user)
-      },
-      %{
-        "rel" => "http://ostatus.org/schema/1.0/subscribe",
-        "template" => OStatus.remote_follow_path()
-      }
-    ]
-  end
-
-  def gather_nodeinfo_protocol_names, do: ["ostatus"]
-end
diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex
deleted file mode 100644
index 23a04b87d..000000000
--- a/lib/pleroma/web/websub/websub_client_subscription.ex
+++ /dev/null
@@ -1,20 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Websub.WebsubClientSubscription do
-  use Ecto.Schema
-  alias Pleroma.User
-
-  schema "websub_client_subscriptions" do
-    field(:topic, :string)
-    field(:secret, :string)
-    field(:valid_until, :naive_datetime_usec)
-    field(:state, :string)
-    field(:subscribers, {:array, :string}, default: [])
-    field(:hub, :string)
-    belongs_to(:user, User, type: FlakeId.Ecto.CompatType)
-
-    timestamps()
-  end
-end
diff --git a/lib/pleroma/web/websub/websub_controller.ex b/lib/pleroma/web/websub/websub_controller.ex
deleted file mode 100644
index 9e8b48b80..000000000
--- a/lib/pleroma/web/websub/websub_controller.ex
+++ /dev/null
@@ -1,99 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Websub.WebsubController do
-  use Pleroma.Web, :controller
-
-  alias Pleroma.Repo
-  alias Pleroma.User
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.Websub
-  alias Pleroma.Web.Websub.WebsubClientSubscription
-
-  require Logger
-
-  plug(
-    Pleroma.Web.FederatingPlug
-    when action in [
-           :websub_subscription_request,
-           :websub_subscription_confirmation,
-           :websub_incoming
-         ]
-  )
-
-  def websub_subscription_request(conn, %{"nickname" => nickname} = params) do
-    user = User.get_cached_by_nickname(nickname)
-
-    with {:ok, _websub} <- Websub.incoming_subscription_request(user, params) do
-      conn
-      |> send_resp(202, "Accepted")
-    else
-      {:error, reason} ->
-        conn
-        |> send_resp(500, reason)
-    end
-  end
-
-  # TODO: Extract this into the Websub module
-  def websub_subscription_confirmation(
-        conn,
-        %{
-          "id" => id,
-          "hub.mode" => "subscribe",
-          "hub.challenge" => challenge,
-          "hub.topic" => topic
-        } = params
-      ) do
-    Logger.debug("Got WebSub confirmation")
-    Logger.debug(inspect(params))
-
-    lease_seconds =
-      if params["hub.lease_seconds"] do
-        String.to_integer(params["hub.lease_seconds"])
-      else
-        # Guess 3 days
-        60 * 60 * 24 * 3
-      end
-
-    with %WebsubClientSubscription{} = websub <-
-           Repo.get_by(WebsubClientSubscription, id: id, topic: topic) do
-      valid_until = NaiveDateTime.add(NaiveDateTime.utc_now(), lease_seconds)
-      change = Ecto.Changeset.change(websub, %{state: "accepted", valid_until: valid_until})
-      {:ok, _websub} = Repo.update(change)
-
-      conn
-      |> send_resp(200, challenge)
-    else
-      _e ->
-        conn
-        |> send_resp(500, "Error")
-    end
-  end
-
-  def websub_subscription_confirmation(conn, params) do
-    Logger.info("Invalid WebSub confirmation request: #{inspect(params)}")
-
-    conn
-    |> send_resp(500, "Invalid parameters")
-  end
-
-  def websub_incoming(conn, %{"id" => id}) do
-    with "sha1=" <> signature <- hd(get_req_header(conn, "x-hub-signature")),
-         signature <- String.downcase(signature),
-         %WebsubClientSubscription{} = websub <- Repo.get(WebsubClientSubscription, id),
-         {:ok, body, _conn} = read_body(conn),
-         ^signature <- Websub.sign(websub.secret, body) do
-      Federator.incoming_doc(body)
-
-      conn
-      |> send_resp(200, "OK")
-    else
-      _e ->
-        Logger.debug("Can't handle incoming subscription post")
-
-        conn
-        |> send_resp(500, "Error")
-    end
-  end
-end
diff --git a/lib/pleroma/web/websub/websub_server_subscription.ex b/lib/pleroma/web/websub/websub_server_subscription.ex
deleted file mode 100644
index d0ef548da..000000000
--- a/lib/pleroma/web/websub/websub_server_subscription.ex
+++ /dev/null
@@ -1,17 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Websub.WebsubServerSubscription do
-  use Ecto.Schema
-
-  schema "websub_server_subscriptions" do
-    field(:topic, :string)
-    field(:callback, :string)
-    field(:secret, :string)
-    field(:valid_until, :naive_datetime)
-    field(:state, :string)
-
-    timestamps()
-  end
-end
-- 
cgit v1.2.3


From adb639db56fc40b07edaf9ed8cdf40d6aa2c573b Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 23:05:45 +0000
Subject: publisher: move remote_users() from Salmon module

---
 lib/pleroma/web/activity_pub/publisher.ex |  2 +-
 lib/pleroma/web/federator/publisher.ex    | 26 ++++++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/publisher.ex b/lib/pleroma/web/activity_pub/publisher.ex
index 3866dacee..2aac4e8b9 100644
--- a/lib/pleroma/web/activity_pub/publisher.ex
+++ b/lib/pleroma/web/activity_pub/publisher.ex
@@ -129,7 +129,7 @@ defmodule Pleroma.Web.ActivityPub.Publisher do
           []
       end
 
-    Pleroma.Web.Salmon.remote_users(actor, activity) ++ followers ++ fetchers
+    Pleroma.Web.Federator.Publisher.remote_users(actor, activity) ++ followers ++ fetchers
   end
 
   defp get_cc_ap_ids(ap_id, recipients) do
diff --git a/lib/pleroma/web/federator/publisher.ex b/lib/pleroma/web/federator/publisher.ex
index 937064638..fb9b26649 100644
--- a/lib/pleroma/web/federator/publisher.ex
+++ b/lib/pleroma/web/federator/publisher.ex
@@ -80,4 +80,30 @@ defmodule Pleroma.Web.Federator.Publisher do
       links ++ module.gather_nodeinfo_protocol_names()
     end)
   end
+
+  @doc """
+  Gathers a set of remote users given an IR envelope.
+  """
+  def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do
+    cc = Map.get(data, "cc", [])
+
+    bcc =
+      data
+      |> Map.get("bcc", [])
+      |> Enum.reduce([], fn ap_id, bcc ->
+        case Pleroma.List.get_by_ap_id(ap_id) do
+          %Pleroma.List{user_id: ^user_id} = list ->
+            {:ok, following} = Pleroma.List.get_following(list)
+            bcc ++ Enum.map(following, & &1.ap_id)
+
+          _ ->
+            bcc
+        end
+      end)
+
+    [to, cc, bcc]
+    |> Enum.concat()
+    |> Enum.map(&User.get_cached_by_ap_id/1)
+    |> Enum.filter(fn user -> user && !user.local end)
+  end
 end
-- 
cgit v1.2.3


From c00ae10af8187cb9d54a0bfbbf37a69a94298703 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 23:06:53 +0000
Subject: feed: don't advertise salmon endpoint

---
 lib/pleroma/web/templates/feed/feed/feed.xml.eex | 1 -
 1 file changed, 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/templates/feed/feed/feed.xml.eex b/lib/pleroma/web/templates/feed/feed/feed.xml.eex
index 692cfbd7f..45df9dc09 100644
--- a/lib/pleroma/web/templates/feed/feed/feed.xml.eex
+++ b/lib/pleroma/web/templates/feed/feed/feed.xml.eex
@@ -10,7 +10,6 @@
   <title><%= @user.nickname <> "'s timeline" %></title>
   <updated><%= most_recent_update(@activities, @user) %></updated>
   <logo><%= logo(@user) %></logo>
-  <link rel="salmon" href="<%= o_status_url(@conn, :salmon_incoming, @user.nickname) %>"/>
   <link rel="self" href="<%= '#{feed_url(@conn, :feed, @user.nickname)}.atom' %>" type="application/atom+xml"/>
 
   <%= render @view_module, "_author.xml", assigns %>
-- 
cgit v1.2.3


From a7b92bba68281f3861d1446b85743a0b65fe1115 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 23:07:20 +0000
Subject: webfinger: stop pulling Salmon data out of WebFinger

---
 lib/pleroma/web/web_finger/web_finger.ex | 12 ------------
 1 file changed, 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/web_finger/web_finger.ex b/lib/pleroma/web/web_finger/web_finger.ex
index ecb39ee50..b4cc80179 100644
--- a/lib/pleroma/web/web_finger/web_finger.ex
+++ b/lib/pleroma/web/web_finger/web_finger.ex
@@ -108,7 +108,6 @@ defmodule Pleroma.Web.WebFinger do
              doc
            ),
          subject <- XML.string_from_xpath("//Subject", doc),
-         salmon <- XML.string_from_xpath(~s{//Link[@rel="salmon"]/@href}, doc),
          subscribe_address <-
            XML.string_from_xpath(
              ~s{//Link[@rel="http://ostatus.org/schema/1.0/subscribe"]/@template},
@@ -123,7 +122,6 @@ defmodule Pleroma.Web.WebFinger do
         "magic_key" => magic_key,
         "topic" => topic,
         "subject" => subject,
-        "salmon" => salmon,
         "subscribe_address" => subscribe_address,
         "ap_id" => ap_id
       }
@@ -148,16 +146,6 @@ defmodule Pleroma.Web.WebFinger do
           {"application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"", "self"} ->
             Map.put(data, "ap_id", link["href"])
 
-          {_, "magic-public-key"} ->
-            "data:application/magic-public-key," <> magic_key = link["href"]
-            Map.put(data, "magic_key", magic_key)
-
-          {"application/atom+xml", "http://schemas.google.com/g/2010#updates-from"} ->
-            Map.put(data, "topic", link["href"])
-
-          {_, "salmon"} ->
-            Map.put(data, "salmon", link["href"])
-
           {_, "http://ostatus.org/schema/1.0/subscribe"} ->
             Map.put(data, "subscribe_address", link["template"])
 
-- 
cgit v1.2.3


From beb9861f9df080cd071c34f37c95e89d1b170138 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 23:07:54 +0000
Subject: router: disconnect Salmon

---
 lib/pleroma/web/router.ex | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index 77fe938d5..b3b5ada4e 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -508,8 +508,6 @@ defmodule Pleroma.Web.Router do
     get("/users/:nickname/feed", Feed.FeedController, :feed)
     get("/users/:nickname", Feed.FeedController, :feed_redirect)
 
-    post("/users/:nickname/salmon", OStatus.OStatusController, :salmon_incoming)
-
     get("/mailer/unsubscribe/:token", Mailer.SubscriptionController, :unsubscribe)
   end
 
-- 
cgit v1.2.3


From 835ad5237885cb1c95f678054733b904f20b0bbd Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 23:09:15 +0000
Subject: remove Salmon module

---
 lib/pleroma/user/info.ex                      |   6 -
 lib/pleroma/web/ostatus/feed_representer.ex   |   2 -
 lib/pleroma/web/ostatus/ostatus.ex            |   4 -
 lib/pleroma/web/ostatus/ostatus_controller.ex |  28 ---
 lib/pleroma/web/salmon/salmon.ex              | 254 --------------------------
 5 files changed, 294 deletions(-)
 delete mode 100644 lib/pleroma/web/salmon/salmon.ex

(limited to 'lib')

diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex
index 4b5b43d7f..2d39abcb3 100644
--- a/lib/pleroma/user/info.ex
+++ b/lib/pleroma/user/info.ex
@@ -39,9 +39,6 @@ defmodule Pleroma.User.Info do
     field(:settings, :map, default: nil)
     field(:magic_key, :string, default: nil)
     field(:uri, :string, default: nil)
-    field(:topic, :string, default: nil)
-    field(:hub, :string, default: nil)
-    field(:salmon, :string, default: nil)
     field(:hide_followers_count, :boolean, default: false)
     field(:hide_follows_count, :boolean, default: false)
     field(:hide_followers, :boolean, default: false)
@@ -262,9 +259,6 @@ defmodule Pleroma.User.Info do
       :locked,
       :magic_key,
       :uri,
-      :hub,
-      :topic,
-      :salmon,
       :hide_followers,
       :hide_follows,
       :hide_followers_count,
diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex
index b7b97e505..fa7f7b564 100644
--- a/lib/pleroma/web/ostatus/feed_representer.ex
+++ b/lib/pleroma/web/ostatus/feed_representer.ex
@@ -40,8 +40,6 @@ defmodule Pleroma.Web.OStatus.FeedRepresenter do
           {:title, ['#{user.nickname}\'s timeline']},
           {:updated, h.(most_recent_update)},
           {:logo, [to_charlist(User.avatar_url(user) |> MediaProxy.url())]},
-          {:link, [rel: 'hub', href: h.(OStatus.pubsub_path(user))], []},
-          {:link, [rel: 'salmon', href: h.(OStatus.salmon_path(user))], []},
           {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'],
            []},
           {:author, UserRepresenter.to_simple_form(user)}
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
index 3dfc9b580..a858759d3 100644
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ b/lib/pleroma/web/ostatus/ostatus.ex
@@ -37,10 +37,6 @@ defmodule Pleroma.Web.OStatus do
 
   def feed_path(user), do: "#{user.ap_id}/feed.atom"
 
-  def pubsub_path(user), do: "#{Web.base_url()}/push/hub/#{user.nickname}"
-
-  def salmon_path(user), do: "#{user.ap_id}/salmon"
-
   def remote_follow_path, do: "#{Web.base_url()}/ostatus_subscribe?acct={uri}"
 
   def handle_incoming(xml_string, options \\ []) do
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index 20f2d9ddc..7466dd8ea 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -24,8 +24,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     {:ap_routes, params: ["uuid"]} when action in [:object, :activity]
   )
 
-  plug(Pleroma.Web.FederatingPlug when action in [:salmon_incoming])
-
   plug(
     Pleroma.Plugs.SetFormatPlug
     when action in [:object, :activity, :notice]
@@ -33,32 +31,6 @@ defmodule Pleroma.Web.OStatus.OStatusController do
 
   action_fallback(:errors)
 
-  defp decode_or_retry(body) do
-    with {:ok, magic_key} <- Pleroma.Web.Salmon.fetch_magic_key(body),
-         {:ok, doc} <- Pleroma.Web.Salmon.decode_and_validate(magic_key, body) do
-      {:ok, doc}
-    else
-      _e ->
-        with [decoded | _] <- Pleroma.Web.Salmon.decode(body),
-             doc <- XML.parse_document(decoded),
-             uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc),
-             {:ok, _} <- Pleroma.Web.OStatus.make_user(uri, true),
-             {:ok, magic_key} <- Pleroma.Web.Salmon.fetch_magic_key(body),
-             {:ok, doc} <- Pleroma.Web.Salmon.decode_and_validate(magic_key, body) do
-          {:ok, doc}
-        end
-    end
-  end
-
-  def salmon_incoming(conn, _) do
-    {:ok, body, _conn} = read_body(conn)
-    {:ok, doc} = decode_or_retry(body)
-
-    Federator.incoming_doc(doc)
-
-    send_resp(conn, 200, "")
-  end
-
   def object(%{assigns: %{format: format}} = conn, %{"uuid" => _uuid})
       when format in ["json", "activity+json"] do
     ActivityPubController.call(conn, :object)
diff --git a/lib/pleroma/web/salmon/salmon.ex b/lib/pleroma/web/salmon/salmon.ex
deleted file mode 100644
index 0ffe903cd..000000000
--- a/lib/pleroma/web/salmon/salmon.ex
+++ /dev/null
@@ -1,254 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.Salmon do
-  @behaviour Pleroma.Web.Federator.Publisher
-
-  use Bitwise
-
-  alias Pleroma.Activity
-  alias Pleroma.HTTP
-  alias Pleroma.Instances
-  alias Pleroma.Keys
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.OStatus.ActivityRepresenter
-  alias Pleroma.Web.XML
-
-  require Logger
-
-  def decode(salmon) do
-    doc = XML.parse_document(salmon)
-
-    {:xmlObj, :string, data} = :xmerl_xpath.string('string(//me:data[1])', doc)
-    {:xmlObj, :string, sig} = :xmerl_xpath.string('string(//me:sig[1])', doc)
-    {:xmlObj, :string, alg} = :xmerl_xpath.string('string(//me:alg[1])', doc)
-    {:xmlObj, :string, encoding} = :xmerl_xpath.string('string(//me:encoding[1])', doc)
-    {:xmlObj, :string, type} = :xmerl_xpath.string('string(//me:data[1]/@type)', doc)
-
-    {:ok, data} = Base.url_decode64(to_string(data), ignore: :whitespace)
-    {:ok, sig} = Base.url_decode64(to_string(sig), ignore: :whitespace)
-    alg = to_string(alg)
-    encoding = to_string(encoding)
-    type = to_string(type)
-
-    [data, type, encoding, alg, sig]
-  end
-
-  def fetch_magic_key(salmon) do
-    with [data, _, _, _, _] <- decode(salmon),
-         doc <- XML.parse_document(data),
-         uri when not is_nil(uri) <- XML.string_from_xpath("/entry/author[1]/uri", doc),
-         {:ok, public_key} <- User.get_public_key_for_ap_id(uri),
-         magic_key <- encode_key(public_key) do
-      {:ok, magic_key}
-    end
-  end
-
-  def decode_and_validate(magickey, salmon) do
-    [data, type, encoding, alg, sig] = decode(salmon)
-
-    signed_text =
-      [data, type, encoding, alg]
-      |> Enum.map(&Base.url_encode64/1)
-      |> Enum.join(".")
-
-    key = decode_key(magickey)
-
-    verify = :public_key.verify(signed_text, :sha256, sig, key)
-
-    if verify do
-      {:ok, data}
-    else
-      :error
-    end
-  end
-
-  def decode_key("RSA." <> magickey) do
-    make_integer = fn bin ->
-      list = :erlang.binary_to_list(bin)
-      Enum.reduce(list, 0, fn el, acc -> acc <<< 8 ||| el end)
-    end
-
-    [modulus, exponent] =
-      magickey
-      |> String.split(".")
-      |> Enum.map(fn n -> Base.url_decode64!(n, padding: false) end)
-      |> Enum.map(make_integer)
-
-    {:RSAPublicKey, modulus, exponent}
-  end
-
-  def encode_key({:RSAPublicKey, modulus, exponent}) do
-    modulus_enc = :binary.encode_unsigned(modulus) |> Base.url_encode64()
-    exponent_enc = :binary.encode_unsigned(exponent) |> Base.url_encode64()
-
-    "RSA.#{modulus_enc}.#{exponent_enc}"
-  end
-
-  def encode(private_key, doc) do
-    type = "application/atom+xml"
-    encoding = "base64url"
-    alg = "RSA-SHA256"
-
-    signed_text =
-      [doc, type, encoding, alg]
-      |> Enum.map(&Base.url_encode64/1)
-      |> Enum.join(".")
-
-    signature =
-      signed_text
-      |> :public_key.sign(:sha256, private_key)
-      |> to_string
-      |> Base.url_encode64()
-
-    doc_base64 =
-      doc
-      |> Base.url_encode64()
-
-    # Don't need proper xml building, these strings are safe to leave unescaped
-    salmon = """
-    <?xml version="1.0" encoding="UTF-8"?>
-    <me:env xmlns:me="http://salmon-protocol.org/ns/magic-env">
-      <me:data type="application/atom+xml">#{doc_base64}</me:data>
-      <me:encoding>#{encoding}</me:encoding>
-      <me:alg>#{alg}</me:alg>
-      <me:sig>#{signature}</me:sig>
-    </me:env>
-    """
-
-    {:ok, salmon}
-  end
-
-  def remote_users(%User{id: user_id}, %{data: %{"to" => to} = data}) do
-    cc = Map.get(data, "cc", [])
-
-    bcc =
-      data
-      |> Map.get("bcc", [])
-      |> Enum.reduce([], fn ap_id, bcc ->
-        case Pleroma.List.get_by_ap_id(ap_id) do
-          %Pleroma.List{user_id: ^user_id} = list ->
-            {:ok, following} = Pleroma.List.get_following(list)
-            bcc ++ Enum.map(following, & &1.ap_id)
-
-          _ ->
-            bcc
-        end
-      end)
-
-    [to, cc, bcc]
-    |> Enum.concat()
-    |> Enum.map(&User.get_cached_by_ap_id/1)
-    |> Enum.filter(fn user -> user && !user.local end)
-  end
-
-  @doc "Pushes an activity to remote account."
-  def publish_one(%{recipient: %{info: %{salmon: salmon}}} = params),
-    do: publish_one(Map.put(params, :recipient, salmon))
-
-  def publish_one(%{recipient: url, feed: feed} = params) when is_binary(url) do
-    with {:ok, %{status: code}} when code in 200..299 <-
-           HTTP.post(
-             url,
-             feed,
-             [{"Content-Type", "application/magic-envelope+xml"}]
-           ) do
-      if !Map.has_key?(params, :unreachable_since) || params[:unreachable_since],
-        do: Instances.set_reachable(url)
-
-      Logger.debug(fn -> "Pushed to #{url}, code #{code}" end)
-      {:ok, code}
-    else
-      e ->
-        unless params[:unreachable_since], do: Instances.set_reachable(url)
-        Logger.debug(fn -> "Pushing Salmon to #{url} failed, #{inspect(e)}" end)
-        {:error, "Unreachable instance"}
-    end
-  end
-
-  def publish_one(%{recipient_id: recipient_id} = params) do
-    recipient = User.get_cached_by_id(recipient_id)
-
-    params
-    |> Map.delete(:recipient_id)
-    |> Map.put(:recipient, recipient)
-    |> publish_one()
-  end
-
-  def publish_one(_), do: :noop
-
-  @supported_activities [
-    "Create",
-    "Follow",
-    "Like",
-    "Announce",
-    "Undo",
-    "Delete"
-  ]
-
-  def is_representable?(%Activity{data: %{"type" => type}} = activity)
-      when type in @supported_activities,
-      do: Visibility.is_public?(activity)
-
-  def is_representable?(_), do: false
-
-  @doc """
-  Publishes an activity to remote accounts
-  """
-  @spec publish(User.t(), Pleroma.Activity.t()) :: none
-  def publish(user, activity)
-
-  def publish(%{keys: keys} = user, %{data: %{"type" => type}} = activity)
-      when type in @supported_activities do
-    feed = ActivityRepresenter.to_simple_form(activity, user, true)
-
-    if feed do
-      feed =
-        ActivityRepresenter.wrap_with_entry(feed)
-        |> :xmerl.export_simple(:xmerl_xml)
-        |> to_string
-
-      {:ok, private, _} = Keys.keys_from_pem(keys)
-      {:ok, feed} = encode(private, feed)
-
-      remote_users = remote_users(user, activity)
-
-      salmon_urls = Enum.map(remote_users, & &1.info.salmon)
-      reachable_urls_metadata = Instances.filter_reachable(salmon_urls)
-      reachable_urls = Map.keys(reachable_urls_metadata)
-
-      remote_users
-      |> Enum.filter(&(&1.info.salmon in reachable_urls))
-      |> Enum.each(fn remote_user ->
-        Logger.debug(fn -> "Sending Salmon to #{remote_user.ap_id}" end)
-
-        Publisher.enqueue_one(__MODULE__, %{
-          recipient_id: remote_user.id,
-          feed: feed,
-          unreachable_since: reachable_urls_metadata[remote_user.info.salmon]
-        })
-      end)
-    end
-  end
-
-  def publish(%{id: id}, _), do: Logger.debug(fn -> "Keys missing for user #{id}" end)
-
-  def gather_webfinger_links(%User{} = user) do
-    {:ok, _private, public} = Keys.keys_from_pem(user.keys)
-    magic_key = encode_key(public)
-
-    [
-      %{"rel" => "salmon", "href" => OStatus.salmon_path(user)},
-      %{
-        "rel" => "magic-public-key",
-        "href" => "data:application/magic-public-key,#{magic_key}"
-      }
-    ]
-  end
-
-  def gather_nodeinfo_protocol_names, do: []
-end
-- 
cgit v1.2.3


From 6a1f4c5145c05efdfa1b0d56ba25bf87e51c7f82 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 23:15:09 +0000
Subject: federator: remove OStatus incoming document support

---
 lib/pleroma/web/federator/federator.ex | 9 ---------
 lib/pleroma/workers/receiver_worker.ex | 4 ----
 2 files changed, 13 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 8227d1a3a..8c83f9aea 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -30,10 +30,6 @@ defmodule Pleroma.Web.Federator do
 
   # Client API
 
-  def incoming_doc(doc) do
-    ReceiverWorker.enqueue("incoming_doc", %{"body" => doc})
-  end
-
   def incoming_ap_doc(params) do
     ReceiverWorker.enqueue("incoming_ap_doc", %{"params" => params})
   end
@@ -62,11 +58,6 @@ defmodule Pleroma.Web.Federator do
     end
   end
 
-  def perform(:incoming_doc, doc) do
-    Logger.info("Got document, trying to parse")
-    OStatus.handle_incoming(doc)
-  end
-
   def perform(:incoming_ap_doc, params) do
     Logger.info("Handling incoming AP activity")
 
diff --git a/lib/pleroma/workers/receiver_worker.ex b/lib/pleroma/workers/receiver_worker.ex
index 83d528a66..8ad756b62 100644
--- a/lib/pleroma/workers/receiver_worker.ex
+++ b/lib/pleroma/workers/receiver_worker.ex
@@ -8,10 +8,6 @@ defmodule Pleroma.Workers.ReceiverWorker do
   use Pleroma.Workers.WorkerHelper, queue: "federator_incoming"
 
   @impl Oban.Worker
-  def perform(%{"op" => "incoming_doc", "body" => doc}, _job) do
-    Federator.perform(:incoming_doc, doc)
-  end
-
   def perform(%{"op" => "incoming_ap_doc", "params" => params}, _job) do
     Federator.perform(:incoming_ap_doc, params)
   end
-- 
cgit v1.2.3


From d379b4876927701f0fa9e4886f9fd552fe71d9c9 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Thu, 17 Oct 2019 23:37:21 +0000
Subject: kill almost all of the OStatus module

---
 lib/pleroma/object/fetcher.ex                      |  12 +-
 lib/pleroma/user.ex                                |  26 +-
 lib/pleroma/web/federator/federator.ex             |   1 -
 lib/pleroma/web/ostatus/activity_representer.ex    | 313 -----------------
 lib/pleroma/web/ostatus/feed_representer.ex        |  64 ----
 lib/pleroma/web/ostatus/handlers/delete_handler.ex |  18 -
 lib/pleroma/web/ostatus/handlers/follow_handler.ex |  26 --
 lib/pleroma/web/ostatus/handlers/note_handler.ex   | 168 ---------
 .../web/ostatus/handlers/unfollow_handler.ex       |  22 --
 lib/pleroma/web/ostatus/ostatus.ex                 | 388 ---------------------
 lib/pleroma/web/ostatus/ostatus_controller.ex      |  18 +-
 lib/pleroma/web/ostatus/user_representer.ex        |  41 ---
 12 files changed, 5 insertions(+), 1092 deletions(-)
 delete mode 100644 lib/pleroma/web/ostatus/activity_representer.ex
 delete mode 100644 lib/pleroma/web/ostatus/feed_representer.ex
 delete mode 100644 lib/pleroma/web/ostatus/handlers/delete_handler.ex
 delete mode 100644 lib/pleroma/web/ostatus/handlers/follow_handler.ex
 delete mode 100644 lib/pleroma/web/ostatus/handlers/note_handler.ex
 delete mode 100644 lib/pleroma/web/ostatus/handlers/unfollow_handler.ex
 delete mode 100644 lib/pleroma/web/ostatus/ostatus.ex
 delete mode 100644 lib/pleroma/web/ostatus/user_representer.ex

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 5e064fd87..9436e2730 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -10,7 +10,6 @@ defmodule Pleroma.Object.Fetcher do
   alias Pleroma.Signature
   alias Pleroma.Web.ActivityPub.InternalFetchActor
   alias Pleroma.Web.ActivityPub.Transmogrifier
-  alias Pleroma.Web.OStatus
 
   require Logger
   require Pleroma.Constants
@@ -87,15 +86,8 @@ defmodule Pleroma.Object.Fetcher do
       {:fetch_object, %Object{} = object} ->
         {:ok, object}
 
-      _e ->
-        # Only fallback when receiving a fetch/normalization error with ActivityPub
-        Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
-
-        # FIXME: OStatus Object Containment?
-        case OStatus.fetch_activity_from_url(id) do
-          {:ok, [activity | _]} -> {:ok, Object.normalize(activity, false)}
-          e -> e
-        end
+      e ->
+        e
     end
   end
 
diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex
index cef011c68..ec705b8f6 100644
--- a/lib/pleroma/user.ex
+++ b/lib/pleroma/user.ex
@@ -26,7 +26,6 @@ defmodule Pleroma.User do
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils, as: CommonUtils
   alias Pleroma.Web.OAuth
-  alias Pleroma.Web.OStatus
   alias Pleroma.Web.RelMe
   alias Pleroma.Workers.BackgroundWorker
 
@@ -609,12 +608,7 @@ defmodule Pleroma.User do
     Cachex.fetch!(:user_cache, key, fn -> user_info(user) end)
   end
 
-  def fetch_by_nickname(nickname) do
-    case ActivityPub.make_user_from_nickname(nickname) do
-      {:ok, user} -> {:ok, user}
-      _ -> OStatus.make_user(nickname)
-    end
-  end
+  def fetch_by_nickname(nickname), do: ActivityPub.make_user_from_nickname(nickname)
 
   def get_or_fetch_by_nickname(nickname) do
     with %User{} = user <- get_by_nickname(nickname) do
@@ -1241,18 +1235,7 @@ defmodule Pleroma.User do
 
   def html_filter_policy(_), do: Pleroma.Config.get([:markup, :scrub_policy])
 
-  def fetch_by_ap_id(ap_id) do
-    case ActivityPub.make_user_from_ap_id(ap_id) do
-      {:ok, user} ->
-        {:ok, user}
-
-      _ ->
-        case OStatus.make_user(ap_id) do
-          {:ok, user} -> {:ok, user}
-          _ -> {:error, "Could not fetch by AP id"}
-        end
-    end
-  end
+  def fetch_by_ap_id(ap_id), do: ActivityPub.make_user_from_ap_id(ap_id)
 
   def get_or_fetch_by_ap_id(ap_id) do
     user = get_cached_by_ap_id(ap_id)
@@ -1307,11 +1290,6 @@ defmodule Pleroma.User do
     {:ok, key}
   end
 
-  # OStatus Magic Key
-  def public_key_from_info(%{magic_key: magic_key}) when not is_nil(magic_key) do
-    {:ok, Pleroma.Web.Salmon.decode_key(magic_key)}
-  end
-
   def public_key_from_info(_), do: {:error, "not found key"}
 
   def get_public_key_for_ap_id(ap_id) do
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index 8c83f9aea..e8a56ebd7 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -10,7 +10,6 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.ActivityPub.Transmogrifier
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.Federator.Publisher
-  alias Pleroma.Web.OStatus
   alias Pleroma.Workers.PublisherWorker
   alias Pleroma.Workers.ReceiverWorker
 
diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex
deleted file mode 100644
index 8e55b9f0b..000000000
--- a/lib/pleroma/web/ostatus/activity_representer.ex
+++ /dev/null
@@ -1,313 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.ActivityRepresenter do
-  alias Pleroma.Activity
-  alias Pleroma.Object
-  alias Pleroma.User
-  alias Pleroma.Web.OStatus.UserRepresenter
-
-  require Logger
-  require Pleroma.Constants
-
-  defp get_href(id) do
-    with %Object{data: %{"external_url" => external_url}} <- Object.get_cached_by_ap_id(id) do
-      external_url
-    else
-      _e -> id
-    end
-  end
-
-  defp get_in_reply_to(activity) do
-    with %Object{data: %{"inReplyTo" => in_reply_to}} <- Object.normalize(activity) do
-      [
-        {:"thr:in-reply-to",
-         [ref: to_charlist(in_reply_to), href: to_charlist(get_href(in_reply_to))], []}
-      ]
-    else
-      _ ->
-        []
-    end
-  end
-
-  defp get_mentions(to) do
-    Enum.map(to, fn id ->
-      cond do
-        # Special handling for the AP/Ostatus public collections
-        Pleroma.Constants.as_public() == id ->
-          {:link,
-           [
-             rel: "mentioned",
-             "ostatus:object-type": "http://activitystrea.ms/schema/1.0/collection",
-             href: "http://activityschema.org/collection/public"
-           ], []}
-
-        # Ostatus doesn't handle follower collections, ignore these.
-        Regex.match?(~r/^#{Pleroma.Web.base_url()}.+followers$/, id) ->
-          []
-
-        true ->
-          {:link,
-           [
-             rel: "mentioned",
-             "ostatus:object-type": "http://activitystrea.ms/schema/1.0/person",
-             href: id
-           ], []}
-      end
-    end)
-  end
-
-  defp get_links(%{local: true}, %{"id" => object_id}) do
-    h = fn str -> [to_charlist(str)] end
-
-    [
-      {:link, [type: ['application/atom+xml'], href: h.(object_id), rel: 'self'], []},
-      {:link, [type: ['text/html'], href: h.(object_id), rel: 'alternate'], []}
-    ]
-  end
-
-  defp get_links(%{local: false}, %{"external_url" => external_url}) do
-    h = fn str -> [to_charlist(str)] end
-
-    [
-      {:link, [type: ['text/html'], href: h.(external_url), rel: 'alternate'], []}
-    ]
-  end
-
-  defp get_links(_activity, _object_data), do: []
-
-  defp get_emoji_links(emojis) do
-    Enum.map(emojis, fn {emoji, file} ->
-      {:link, [name: to_charlist(emoji), rel: 'emoji', href: to_charlist(file)], []}
-    end)
-  end
-
-  def to_simple_form(activity, user, with_author \\ false)
-
-  def to_simple_form(%{data: %{"type" => "Create"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    object = Object.normalize(activity)
-
-    updated_at = object.data["published"]
-    inserted_at = object.data["published"]
-
-    attachments =
-      Enum.map(object.data["attachment"] || [], fn attachment ->
-        url = hd(attachment["url"])
-
-        {:link,
-         [rel: 'enclosure', href: to_charlist(url["href"]), type: to_charlist(url["mediaType"])],
-         []}
-      end)
-
-    in_reply_to = get_in_reply_to(activity)
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-    mentions = activity.recipients |> get_mentions
-
-    categories =
-      (object.data["tag"] || [])
-      |> Enum.map(fn tag ->
-        if is_binary(tag) do
-          {:category, [term: to_charlist(tag)], []}
-        else
-          nil
-        end
-      end)
-      |> Enum.filter(& &1)
-
-    emoji_links = get_emoji_links(object.data["emoji"] || %{})
-
-    summary =
-      if object.data["summary"] do
-        [{:summary, [], h.(object.data["summary"])}]
-      else
-        []
-      end
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']},
-      # For notes, federate the object id.
-      {:id, h.(object.data["id"])},
-      {:title, ['New note by #{user.nickname}']},
-      {:content, [type: 'html'], h.(object.data["content"] |> String.replace(~r/[\n\r]/, ""))},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"ostatus:conversation", [ref: h.(activity.data["context"])],
-       h.(activity.data["context"])},
-      {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}
-    ] ++
-      summary ++
-      get_links(activity, object.data) ++
-      categories ++ attachments ++ in_reply_to ++ author ++ mentions ++ emoji_links
-  end
-
-  def to_simple_form(%{data: %{"type" => "Like"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-    mentions = activity.recipients |> get_mentions
-
-    [
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/favorite']},
-      {:id, h.(activity.data["id"])},
-      {:title, ['New favorite by #{user.nickname}']},
-      {:content, [type: 'html'], ['#{user.nickname} favorited something']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"activity:object",
-       [
-         {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
-         # For notes, federate the object id.
-         {:id, h.(activity.data["object"])}
-       ]},
-      {:"ostatus:conversation", [ref: h.(activity.data["context"])],
-       h.(activity.data["context"])},
-      {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
-      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
-      {:"thr:in-reply-to", [ref: to_charlist(activity.data["object"])], []}
-    ] ++ author ++ mentions
-  end
-
-  def to_simple_form(%{data: %{"type" => "Announce"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-
-    retweeted_activity = Activity.get_create_by_object_ap_id(activity.data["object"])
-    retweeted_object = Object.normalize(retweeted_activity)
-    retweeted_user = User.get_cached_by_ap_id(retweeted_activity.data["actor"])
-
-    retweeted_xml = to_simple_form(retweeted_activity, retweeted_user, true)
-
-    mentions =
-      ([retweeted_user.ap_id] ++ activity.recipients)
-      |> Enum.uniq()
-      |> get_mentions()
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/share']},
-      {:id, h.(activity.data["id"])},
-      {:title, ['#{user.nickname} repeated a notice']},
-      {:content, [type: 'html'], ['RT #{retweeted_object.data["content"]}']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"ostatus:conversation", [ref: h.(activity.data["context"])],
-       h.(activity.data["context"])},
-      {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []},
-      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []},
-      {:"activity:object", retweeted_xml}
-    ] ++ mentions ++ author
-  end
-
-  def to_simple_form(%{data: %{"type" => "Follow"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-
-    mentions = (activity.recipients || []) |> get_mentions
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/follow']},
-      {:id, h.(activity.data["id"])},
-      {:title, ['#{user.nickname} started following #{activity.data["object"]}']},
-      {:content, [type: 'html'],
-       ['#{user.nickname} started following #{activity.data["object"]}']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"activity:object",
-       [
-         {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/person']},
-         {:id, h.(activity.data["object"])},
-         {:uri, h.(activity.data["object"])}
-       ]},
-      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}
-    ] ++ mentions ++ author
-  end
-
-  # Only undos of follow for now. Will need to get redone once there are more
-  def to_simple_form(
-        %{data: %{"type" => "Undo", "object" => %{"type" => "Follow"} = follow_activity}} =
-          activity,
-        user,
-        with_author
-      ) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-
-    mentions = (activity.recipients || []) |> get_mentions
-    follow_activity = Activity.normalize(follow_activity)
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/unfollow']},
-      {:id, h.(activity.data["id"])},
-      {:title, ['#{user.nickname} stopped following #{follow_activity.data["object"]}']},
-      {:content, [type: 'html'],
-       ['#{user.nickname} stopped following #{follow_activity.data["object"]}']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)},
-      {:"activity:object",
-       [
-         {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/person']},
-         {:id, h.(follow_activity.data["object"])},
-         {:uri, h.(follow_activity.data["object"])}
-       ]},
-      {:link, [rel: 'self', type: ['application/atom+xml'], href: h.(activity.data["id"])], []}
-    ] ++ mentions ++ author
-  end
-
-  def to_simple_form(%{data: %{"type" => "Delete"}} = activity, user, with_author) do
-    h = fn str -> [to_charlist(str)] end
-
-    updated_at = activity.data["published"]
-    inserted_at = activity.data["published"]
-
-    author = if with_author, do: [{:author, UserRepresenter.to_simple_form(user)}], else: []
-
-    [
-      {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/activity']},
-      {:"activity:verb", ['http://activitystrea.ms/schema/1.0/delete']},
-      {:id, h.(activity.data["object"])},
-      {:title, ['An object was deleted']},
-      {:content, [type: 'html'], ['An object was deleted']},
-      {:published, h.(inserted_at)},
-      {:updated, h.(updated_at)}
-    ] ++ author
-  end
-
-  def to_simple_form(_, _, _), do: nil
-
-  def wrap_with_entry(simple_form) do
-    [
-      {
-        :entry,
-        [
-          xmlns: 'http://www.w3.org/2005/Atom',
-          "xmlns:thr": 'http://purl.org/syndication/thread/1.0',
-          "xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
-          "xmlns:poco": 'http://portablecontacts.net/spec/1.0',
-          "xmlns:ostatus": 'http://ostatus.org/schema/1.0'
-        ],
-        simple_form
-      }
-    ]
-  end
-end
diff --git a/lib/pleroma/web/ostatus/feed_representer.ex b/lib/pleroma/web/ostatus/feed_representer.ex
deleted file mode 100644
index fa7f7b564..000000000
--- a/lib/pleroma/web/ostatus/feed_representer.ex
+++ /dev/null
@@ -1,64 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.FeedRepresenter do
-  alias Pleroma.User
-  alias Pleroma.Web.MediaProxy
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.OStatus.ActivityRepresenter
-  alias Pleroma.Web.OStatus.UserRepresenter
-
-  def to_simple_form(user, activities, _users) do
-    most_recent_update =
-      (List.first(activities) || user).updated_at
-      |> NaiveDateTime.to_iso8601()
-
-    h = fn str -> [to_charlist(str)] end
-
-    last_activity = List.last(activities)
-
-    entries =
-      activities
-      |> Enum.map(fn activity ->
-        {:entry, ActivityRepresenter.to_simple_form(activity, user)}
-      end)
-      |> Enum.filter(fn {_, form} -> form end)
-
-    [
-      {
-        :feed,
-        [
-          xmlns: 'http://www.w3.org/2005/Atom',
-          "xmlns:thr": 'http://purl.org/syndication/thread/1.0',
-          "xmlns:activity": 'http://activitystrea.ms/spec/1.0/',
-          "xmlns:poco": 'http://portablecontacts.net/spec/1.0',
-          "xmlns:ostatus": 'http://ostatus.org/schema/1.0'
-        ],
-        [
-          {:id, h.(OStatus.feed_path(user))},
-          {:title, ['#{user.nickname}\'s timeline']},
-          {:updated, h.(most_recent_update)},
-          {:logo, [to_charlist(User.avatar_url(user) |> MediaProxy.url())]},
-          {:link, [rel: 'self', href: h.(OStatus.feed_path(user)), type: 'application/atom+xml'],
-           []},
-          {:author, UserRepresenter.to_simple_form(user)}
-        ] ++
-          if last_activity do
-            [
-              {:link,
-               [
-                 rel: 'next',
-                 href:
-                   to_charlist(OStatus.feed_path(user)) ++
-                     '?max_id=' ++ to_charlist(last_activity.id),
-                 type: 'application/atom+xml'
-               ], []}
-            ]
-          else
-            []
-          end ++ entries
-      }
-    ]
-  end
-end
diff --git a/lib/pleroma/web/ostatus/handlers/delete_handler.ex b/lib/pleroma/web/ostatus/handlers/delete_handler.ex
deleted file mode 100644
index ac2dc115c..000000000
--- a/lib/pleroma/web/ostatus/handlers/delete_handler.ex
+++ /dev/null
@@ -1,18 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.DeleteHandler do
-  require Logger
-  alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.XML
-
-  def handle_delete(entry, _doc \\ nil) do
-    with id <- XML.string_from_xpath("//id", entry),
-         %Object{} = object <- Object.normalize(id),
-         {:ok, delete} <- ActivityPub.delete(object, local: false) do
-      delete
-    end
-  end
-end
diff --git a/lib/pleroma/web/ostatus/handlers/follow_handler.ex b/lib/pleroma/web/ostatus/handlers/follow_handler.ex
deleted file mode 100644
index 24513972e..000000000
--- a/lib/pleroma/web/ostatus/handlers/follow_handler.ex
+++ /dev/null
@@ -1,26 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.FollowHandler do
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.XML
-
-  def handle(entry, doc) do
-    with {:ok, actor} <- OStatus.find_make_or_update_actor(doc),
-         id when not is_nil(id) <- XML.string_from_xpath("/entry/id", entry),
-         followed_uri when not is_nil(followed_uri) <-
-           XML.string_from_xpath("/entry/activity:object/id", entry),
-         {:ok, followed} <- OStatus.find_or_make_user(followed_uri),
-         {:locked, false} <- {:locked, followed.info.locked},
-         {:ok, activity} <- ActivityPub.follow(actor, followed, id, false) do
-      User.follow(actor, followed)
-      {:ok, activity}
-    else
-      {:locked, true} ->
-        {:error, "It's not possible to follow locked accounts over OStatus"}
-    end
-  end
-end
diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex
deleted file mode 100644
index 7fae14f7b..000000000
--- a/lib/pleroma/web/ostatus/handlers/note_handler.ex
+++ /dev/null
@@ -1,168 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.NoteHandler do
-  require Logger
-  require Pleroma.Constants
-
-  alias Pleroma.Activity
-  alias Pleroma.Object
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Utils
-  alias Pleroma.Web.CommonAPI
-  alias Pleroma.Web.Federator
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.XML
-
-  @doc """
-  Get the context for this note. Uses this:
-  1. The context of the parent activity
-  2. The conversation reference in the ostatus xml
-  3. A newly generated context id.
-  """
-  def get_context(entry, in_reply_to) do
-    context =
-      (XML.string_from_xpath("//ostatus:conversation[1]", entry) ||
-         XML.string_from_xpath("//ostatus:conversation[1]/@ref", entry) || "")
-      |> String.trim()
-
-    with %{data: %{"context" => context}} <- Object.get_cached_by_ap_id(in_reply_to) do
-      context
-    else
-      _e ->
-        if String.length(context) > 0 do
-          context
-        else
-          Utils.generate_context_id()
-        end
-    end
-  end
-
-  def get_people_mentions(entry) do
-    :xmerl_xpath.string(
-      '//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/person"]',
-      entry
-    )
-    |> Enum.map(fn person -> XML.string_from_xpath("@href", person) end)
-  end
-
-  def get_collection_mentions(entry) do
-    transmogrify = fn
-      "http://activityschema.org/collection/public" ->
-        Pleroma.Constants.as_public()
-
-      group ->
-        group
-    end
-
-    :xmerl_xpath.string(
-      '//link[@rel="mentioned" and @ostatus:object-type="http://activitystrea.ms/schema/1.0/collection"]',
-      entry
-    )
-    |> Enum.map(fn collection -> XML.string_from_xpath("@href", collection) |> transmogrify.() end)
-  end
-
-  def get_mentions(entry) do
-    (get_people_mentions(entry) ++ get_collection_mentions(entry))
-    |> Enum.filter(& &1)
-  end
-
-  def get_emoji(entry) do
-    try do
-      :xmerl_xpath.string('//link[@rel="emoji"]', entry)
-      |> Enum.reduce(%{}, fn emoji, acc ->
-        Map.put(acc, XML.string_from_xpath("@name", emoji), XML.string_from_xpath("@href", emoji))
-      end)
-    rescue
-      _e -> nil
-    end
-  end
-
-  def make_to_list(actor, mentions) do
-    [
-      actor.follower_address
-    ] ++ mentions
-  end
-
-  def add_external_url(note, entry) do
-    url = XML.string_from_xpath("//link[@rel='alternate' and @type='text/html']/@href", entry)
-    Map.put(note, "external_url", url)
-  end
-
-  def fetch_replied_to_activity(entry, in_reply_to, options \\ []) do
-    with %Activity{} = activity <- Activity.get_create_by_object_ap_id(in_reply_to) do
-      activity
-    else
-      _e ->
-        with true <- Federator.allowed_incoming_reply_depth?(options[:depth]),
-             in_reply_to_href when not is_nil(in_reply_to_href) <-
-               XML.string_from_xpath("//thr:in-reply-to[1]/@href", entry),
-             {:ok, [activity | _]} <- OStatus.fetch_activity_from_url(in_reply_to_href, options) do
-          activity
-        else
-          _e -> nil
-        end
-    end
-  end
-
-  # TODO: Clean this up a bit.
-  def handle_note(entry, doc \\ nil, options \\ []) do
-    with id <- XML.string_from_xpath("//id", entry),
-         activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id),
-         [author] <- :xmerl_xpath.string('//author[1]', doc),
-         {:ok, actor} <- OStatus.find_make_or_update_actor(author),
-         content_html <- OStatus.get_content(entry),
-         cw <- OStatus.get_cw(entry),
-         in_reply_to <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
-         options <- Keyword.put(options, :depth, (options[:depth] || 0) + 1),
-         in_reply_to_activity <- fetch_replied_to_activity(entry, in_reply_to, options),
-         in_reply_to_object <-
-           (in_reply_to_activity && Object.normalize(in_reply_to_activity)) || nil,
-         in_reply_to <- (in_reply_to_object && in_reply_to_object.data["id"]) || in_reply_to,
-         attachments <- OStatus.get_attachments(entry),
-         context <- get_context(entry, in_reply_to),
-         tags <- OStatus.get_tags(entry),
-         mentions <- get_mentions(entry),
-         to <- make_to_list(actor, mentions),
-         date <- XML.string_from_xpath("//published", entry),
-         unlisted <- XML.string_from_xpath("//mastodon:scope", entry) == "unlisted",
-         cc <- if(unlisted, do: [Pleroma.Constants.as_public()], else: []),
-         note <-
-           CommonAPI.Utils.make_note_data(
-             actor.ap_id,
-             to,
-             context,
-             content_html,
-             attachments,
-             in_reply_to_activity,
-             [],
-             cw
-           ),
-         note <- note |> Map.put("id", id) |> Map.put("tag", tags),
-         note <- note |> Map.put("published", date),
-         note <- note |> Map.put("emoji", get_emoji(entry)),
-         note <- add_external_url(note, entry),
-         note <- note |> Map.put("cc", cc),
-         # TODO: Handle this case in make_note_data
-         note <-
-           if(
-             in_reply_to && !in_reply_to_activity,
-             do: note |> Map.put("inReplyTo", in_reply_to),
-             else: note
-           ) do
-      ActivityPub.create(%{
-        to: to,
-        actor: actor,
-        context: context,
-        object: note,
-        published: date,
-        local: false,
-        additional: %{"cc" => cc}
-      })
-    else
-      %Activity{} = activity -> {:ok, activity}
-      e -> {:error, e}
-    end
-  end
-end
diff --git a/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex b/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex
deleted file mode 100644
index 2062432e3..000000000
--- a/lib/pleroma/web/ostatus/handlers/unfollow_handler.ex
+++ /dev/null
@@ -1,22 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.UnfollowHandler do
-  alias Pleroma.User
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.OStatus
-  alias Pleroma.Web.XML
-
-  def handle(entry, doc) do
-    with {:ok, actor} <- OStatus.find_make_or_update_actor(doc),
-         id when not is_nil(id) <- XML.string_from_xpath("/entry/id", entry),
-         followed_uri when not is_nil(followed_uri) <-
-           XML.string_from_xpath("/entry/activity:object/id", entry),
-         {:ok, followed} <- OStatus.find_or_make_user(followed_uri),
-         {:ok, activity} <- ActivityPub.unfollow(actor, followed, id, false) do
-      User.unfollow(actor, followed)
-      {:ok, activity}
-    end
-  end
-end
diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex
deleted file mode 100644
index a858759d3..000000000
--- a/lib/pleroma/web/ostatus/ostatus.ex
+++ /dev/null
@@ -1,388 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus do
-  import Pleroma.Web.XML
-  require Logger
-
-  alias Pleroma.Activity
-  alias Pleroma.HTTP
-  alias Pleroma.Object
-  alias Pleroma.User
-  alias Pleroma.Web
-  alias Pleroma.Web.ActivityPub.ActivityPub
-  alias Pleroma.Web.ActivityPub.Transmogrifier
-  alias Pleroma.Web.ActivityPub.Visibility
-  alias Pleroma.Web.OStatus.DeleteHandler
-  alias Pleroma.Web.OStatus.FollowHandler
-  alias Pleroma.Web.OStatus.NoteHandler
-  alias Pleroma.Web.OStatus.UnfollowHandler
-  alias Pleroma.Web.WebFinger
-
-  def is_representable?(%Activity{} = activity) do
-    object = Object.normalize(activity)
-
-    cond do
-      is_nil(object) ->
-        false
-
-      Visibility.is_public?(activity) && object.data["type"] == "Note" ->
-        true
-
-      true ->
-        false
-    end
-  end
-
-  def feed_path(user), do: "#{user.ap_id}/feed.atom"
-
-  def remote_follow_path, do: "#{Web.base_url()}/ostatus_subscribe?acct={uri}"
-
-  def handle_incoming(xml_string, options \\ []) do
-    with doc when doc != :error <- parse_document(xml_string) do
-      with {:ok, actor_user} <- find_make_or_update_actor(doc),
-           do: Pleroma.Instances.set_reachable(actor_user.ap_id)
-
-      entries = :xmerl_xpath.string('//entry', doc)
-
-      activities =
-        Enum.map(entries, fn entry ->
-          {:xmlObj, :string, object_type} =
-            :xmerl_xpath.string('string(/entry/activity:object-type[1])', entry)
-
-          {:xmlObj, :string, verb} = :xmerl_xpath.string('string(/entry/activity:verb[1])', entry)
-          Logger.debug("Handling #{verb}")
-
-          try do
-            case verb do
-              'http://activitystrea.ms/schema/1.0/delete' ->
-                with {:ok, activity} <- DeleteHandler.handle_delete(entry, doc), do: activity
-
-              'http://activitystrea.ms/schema/1.0/follow' ->
-                with {:ok, activity} <- FollowHandler.handle(entry, doc), do: activity
-
-              'http://activitystrea.ms/schema/1.0/unfollow' ->
-                with {:ok, activity} <- UnfollowHandler.handle(entry, doc), do: activity
-
-              'http://activitystrea.ms/schema/1.0/share' ->
-                with {:ok, activity, retweeted_activity} <- handle_share(entry, doc),
-                     do: [activity, retweeted_activity]
-
-              'http://activitystrea.ms/schema/1.0/favorite' ->
-                with {:ok, activity, favorited_activity} <- handle_favorite(entry, doc),
-                     do: [activity, favorited_activity]
-
-              _ ->
-                case object_type do
-                  'http://activitystrea.ms/schema/1.0/note' ->
-                    with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options),
-                         do: activity
-
-                  'http://activitystrea.ms/schema/1.0/comment' ->
-                    with {:ok, activity} <- NoteHandler.handle_note(entry, doc, options),
-                         do: activity
-
-                  _ ->
-                    Logger.error("Couldn't parse incoming document")
-                    nil
-                end
-            end
-          rescue
-            e ->
-              Logger.error("Error occured while handling activity")
-              Logger.error(xml_string)
-              Logger.error(inspect(e))
-              nil
-          end
-        end)
-        |> Enum.filter(& &1)
-
-      {:ok, activities}
-    else
-      _e -> {:error, []}
-    end
-  end
-
-  def make_share(entry, doc, retweeted_activity) do
-    with {:ok, actor} <- find_make_or_update_actor(doc),
-         %Object{} = object <- Object.normalize(retweeted_activity),
-         id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
-         {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do
-      {:ok, activity}
-    end
-  end
-
-  def handle_share(entry, doc) do
-    with {:ok, retweeted_activity} <- get_or_build_object(entry),
-         {:ok, activity} <- make_share(entry, doc, retweeted_activity) do
-      {:ok, activity, retweeted_activity}
-    else
-      e -> {:error, e}
-    end
-  end
-
-  def make_favorite(entry, doc, favorited_activity) do
-    with {:ok, actor} <- find_make_or_update_actor(doc),
-         %Object{} = object <- Object.normalize(favorited_activity),
-         id when not is_nil(id) <- string_from_xpath("/entry/id", entry),
-         {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do
-      {:ok, activity}
-    end
-  end
-
-  def get_or_build_object(entry) do
-    with {:ok, activity} <- get_or_try_fetching(entry) do
-      {:ok, activity}
-    else
-      _e ->
-        with [object] <- :xmerl_xpath.string('/entry/activity:object', entry) do
-          NoteHandler.handle_note(object, object)
-        end
-    end
-  end
-
-  def get_or_try_fetching(entry) do
-    Logger.debug("Trying to get entry from db")
-
-    with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry),
-         %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do
-      {:ok, activity}
-    else
-      _ ->
-        Logger.debug("Couldn't get, will try to fetch")
-
-        with href when not is_nil(href) <-
-               string_from_xpath("//activity:object[1]/link[@type=\"text/html\"]/@href", entry),
-             {:ok, [favorited_activity]} <- fetch_activity_from_url(href) do
-          {:ok, favorited_activity}
-        else
-          e -> Logger.debug("Couldn't find href: #{inspect(e)}")
-        end
-    end
-  end
-
-  def handle_favorite(entry, doc) do
-    with {:ok, favorited_activity} <- get_or_try_fetching(entry),
-         {:ok, activity} <- make_favorite(entry, doc, favorited_activity) do
-      {:ok, activity, favorited_activity}
-    else
-      e -> {:error, e}
-    end
-  end
-
-  def get_attachments(entry) do
-    :xmerl_xpath.string('/entry/link[@rel="enclosure"]', entry)
-    |> Enum.map(fn enclosure ->
-      with href when not is_nil(href) <- string_from_xpath("/link/@href", enclosure),
-           type when not is_nil(type) <- string_from_xpath("/link/@type", enclosure) do
-        %{
-          "type" => "Attachment",
-          "url" => [
-            %{
-              "type" => "Link",
-              "mediaType" => type,
-              "href" => href
-            }
-          ]
-        }
-      end
-    end)
-    |> Enum.filter(& &1)
-  end
-
-  @doc """
-    Gets the content from a an entry.
-  """
-  def get_content(entry) do
-    string_from_xpath("//content", entry)
-  end
-
-  @doc """
-    Get the cw that mastodon uses.
-  """
-  def get_cw(entry) do
-    case string_from_xpath("/*/summary", entry) do
-      cw when not is_nil(cw) -> cw
-      _ -> nil
-    end
-  end
-
-  def get_tags(entry) do
-    :xmerl_xpath.string('//category', entry)
-    |> Enum.map(fn category -> string_from_xpath("/category/@term", category) end)
-    |> Enum.filter(& &1)
-    |> Enum.map(&String.downcase/1)
-  end
-
-  def maybe_update(doc, user) do
-    case string_from_xpath("//author[1]/ap_enabled", doc) do
-      "true" ->
-        Transmogrifier.upgrade_user_from_ap_id(user.ap_id)
-
-      _ ->
-        maybe_update_ostatus(doc, user)
-    end
-  end
-
-  def maybe_update_ostatus(doc, user) do
-    old_data = Map.take(user, [:bio, :avatar, :name])
-
-    with false <- user.local,
-         avatar <- make_avatar_object(doc),
-         bio <- string_from_xpath("//author[1]/summary", doc),
-         name <- string_from_xpath("//author[1]/poco:displayName", doc),
-         new_data <- %{
-           avatar: avatar || old_data.avatar,
-           name: name || old_data.name,
-           bio: bio || old_data.bio
-         },
-         false <- new_data == old_data do
-      change = Ecto.Changeset.change(user, new_data)
-      User.update_and_set_cache(change)
-    else
-      _ ->
-        {:ok, user}
-    end
-  end
-
-  def find_make_or_update_actor(doc) do
-    uri = string_from_xpath("//author/uri[1]", doc)
-
-    with {:ok, %User{} = user} <- find_or_make_user(uri),
-         {:ap_enabled, false} <- {:ap_enabled, User.ap_enabled?(user)} do
-      maybe_update(doc, user)
-    else
-      {:ap_enabled, true} ->
-        {:error, :invalid_protocol}
-
-      _ ->
-        {:error, :unknown_user}
-    end
-  end
-
-  @spec find_or_make_user(String.t()) :: {:ok, User.t()}
-  def find_or_make_user(uri) do
-    case User.get_by_ap_id(uri) do
-      %User{} = user -> {:ok, user}
-      _ -> make_user(uri)
-    end
-  end
-
-  @spec make_user(String.t(), boolean()) :: {:ok, User.t()} | {:error, any()}
-  def make_user(uri, update \\ false) do
-    with {:ok, info} <- gather_user_info(uri) do
-      with false <- update,
-           %User{} = user <- User.get_cached_by_ap_id(info["uri"]) do
-        {:ok, user}
-      else
-        _e -> User.insert_or_update_user(build_user_data(info))
-      end
-    end
-  end
-
-  defp build_user_data(info) do
-    %{
-      name: info["name"],
-      nickname: info["nickname"] <> "@" <> info["host"],
-      ap_id: info["uri"],
-      info: info,
-      avatar: info["avatar"],
-      bio: info["bio"]
-    }
-  end
-
-  # TODO: Just takes the first one for now.
-  def make_avatar_object(author_doc, rel \\ "avatar") do
-    href = string_from_xpath("//author[1]/link[@rel=\"#{rel}\"]/@href", author_doc)
-    type = string_from_xpath("//author[1]/link[@rel=\"#{rel}\"]/@type", author_doc)
-
-    if href do
-      %{
-        "type" => "Image",
-        "url" => [%{"type" => "Link", "mediaType" => type, "href" => href}]
-      }
-    else
-      nil
-    end
-  end
-
-  @spec gather_user_info(String.t()) :: {:ok, map()} | {:error, any()}
-  def gather_user_info(username) do
-    with {:ok, webfinger_data} <- WebFinger.finger(username) do
-      data =
-        webfinger_data
-        |> Map.put("fqn", username)
-
-      {:ok, data}
-    else
-      e ->
-        Logger.debug(fn -> "Couldn't gather info for #{username}" end)
-        {:error, e}
-    end
-  end
-
-  # Regex-based 'parsing' so we don't have to pull in a full html parser
-  # It's a hack anyway. Maybe revisit this in the future
-  @mastodon_regex ~r/<link href='(.*)' rel='alternate' type='application\/atom\+xml'>/
-  @gs_regex ~r/<link title=.* href="(.*)" type="application\/atom\+xml" rel="alternate">/
-  @gs_classic_regex ~r/<link rel="alternate" href="(.*)" type="application\/atom\+xml" title=.*>/
-  def get_atom_url(body) do
-    cond do
-      Regex.match?(@mastodon_regex, body) ->
-        [[_, match]] = Regex.scan(@mastodon_regex, body)
-        {:ok, match}
-
-      Regex.match?(@gs_regex, body) ->
-        [[_, match]] = Regex.scan(@gs_regex, body)
-        {:ok, match}
-
-      Regex.match?(@gs_classic_regex, body) ->
-        [[_, match]] = Regex.scan(@gs_classic_regex, body)
-        {:ok, match}
-
-      true ->
-        Logger.debug(fn -> "Couldn't find Atom link in #{inspect(body)}" end)
-        {:error, "Couldn't find the Atom link"}
-    end
-  end
-
-  def fetch_activity_from_atom_url(url, options \\ []) do
-    with true <- String.starts_with?(url, "http"),
-         {:ok, %{body: body, status: code}} when code in 200..299 <-
-           HTTP.get(url, [{:Accept, "application/atom+xml"}]) do
-      Logger.debug("Got document from #{url}, handling...")
-      handle_incoming(body, options)
-    else
-      e ->
-        Logger.debug("Couldn't get #{url}: #{inspect(e)}")
-        e
-    end
-  end
-
-  def fetch_activity_from_html_url(url, options \\ []) do
-    Logger.debug("Trying to fetch #{url}")
-
-    with true <- String.starts_with?(url, "http"),
-         {:ok, %{body: body}} <- HTTP.get(url, []),
-         {:ok, atom_url} <- get_atom_url(body) do
-      fetch_activity_from_atom_url(atom_url, options)
-    else
-      e ->
-        Logger.debug("Couldn't get #{url}: #{inspect(e)}")
-        e
-    end
-  end
-
-  def fetch_activity_from_url(url, options \\ []) do
-    with {:ok, [_ | _] = activities} <- fetch_activity_from_atom_url(url, options) do
-      {:ok, activities}
-    else
-      _e -> fetch_activity_from_html_url(url, options)
-    end
-  rescue
-    e ->
-      Logger.debug("Couldn't get #{url}: #{inspect(e)}")
-      {:error, "Couldn't get #{url}: #{inspect(e)}"}
-  end
-end
diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex
index 7466dd8ea..6958519de 100644
--- a/lib/pleroma/web/ostatus/ostatus_controller.ex
+++ b/lib/pleroma/web/ostatus/ostatus_controller.ex
@@ -13,11 +13,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do
   alias Pleroma.Web.ActivityPub.ObjectView
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Endpoint
-  alias Pleroma.Web.Federator
   alias Pleroma.Web.Metadata.PlayerView
-  alias Pleroma.Web.OStatus.ActivityRepresenter
   alias Pleroma.Web.Router
-  alias Pleroma.Web.XML
 
   plug(
     Pleroma.Plugs.RateLimiter,
@@ -151,23 +148,10 @@ defmodule Pleroma.Web.OStatus.OStatusController do
     |> render("object.json", %{object: object})
   end
 
-  defp represent_activity(_conn, "activity+json", _, _) do
+  defp represent_activity(_conn, _, _, _) do
     {:error, :not_found}
   end
 
-  defp represent_activity(conn, _, activity, user) do
-    response =
-      activity
-      |> ActivityRepresenter.to_simple_form(user, true)
-      |> ActivityRepresenter.wrap_with_entry()
-      |> :xmerl.export_simple(:xmerl_xml)
-      |> to_string
-
-    conn
-    |> put_resp_content_type("application/atom+xml")
-    |> send_resp(200, response)
-  end
-
   def errors(conn, {:error, :not_found}) do
     render_error(conn, :not_found, "Not found")
   end
diff --git a/lib/pleroma/web/ostatus/user_representer.ex b/lib/pleroma/web/ostatus/user_representer.ex
deleted file mode 100644
index 852be6eb4..000000000
--- a/lib/pleroma/web/ostatus/user_representer.ex
+++ /dev/null
@@ -1,41 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Pleroma.Web.OStatus.UserRepresenter do
-  alias Pleroma.User
-
-  def to_simple_form(user) do
-    ap_id = to_charlist(user.ap_id)
-    nickname = to_charlist(user.nickname)
-    name = to_charlist(user.name)
-    bio = to_charlist(user.bio)
-    avatar_url = to_charlist(User.avatar_url(user))
-
-    banner =
-      if banner_url = User.banner_url(user) do
-        [{:link, [rel: 'header', href: banner_url], []}]
-      else
-        []
-      end
-
-    ap_enabled =
-      if user.local do
-        [{:ap_enabled, ['true']}]
-      else
-        []
-      end
-
-    [
-      {:id, [ap_id]},
-      {:"activity:object", ['http://activitystrea.ms/schema/1.0/person']},
-      {:uri, [ap_id]},
-      {:"poco:preferredUsername", [nickname]},
-      {:"poco:displayName", [name]},
-      {:"poco:note", [bio]},
-      {:summary, [bio]},
-      {:name, [nickname]},
-      {:link, [rel: 'avatar', href: avatar_url], []}
-    ] ++ banner ++ ap_enabled
-  end
-end
-- 
cgit v1.2.3


From 48059c03c91b2437779ac42581812c07530c1a34 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 00:30:01 +0000
Subject: fix up some tests

---
 lib/pleroma/object/fetcher.ex                | 4 ++--
 lib/pleroma/web/activity_pub/activity_pub.ex | 4 +++-
 2 files changed, 5 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 9436e2730..8975fb47e 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -162,8 +162,8 @@ defmodule Pleroma.Object.Fetcher do
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, "Object has been deleted"}
 
-      e ->
-        {:error, e}
+      _ ->
+        {:error, "Could not fetch by AP id"}
     end
   end
 
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index d391732a2..d631e43c6 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1219,7 +1219,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          data <- maybe_update_follow_information(data) do
       {:ok, data}
     else
-      e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
+      e ->
+        Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
+        {:error, e}
     end
   end
 
-- 
cgit v1.2.3


From 6f110fc04ca31bf571fb80a2a852ab60ddd496ff Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 02:42:25 +0000
Subject: object fetcher: fix up error handling

---
 lib/pleroma/object/fetcher.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 8975fb47e..9436e2730 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -162,8 +162,8 @@ defmodule Pleroma.Object.Fetcher do
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, "Object has been deleted"}
 
-      _ ->
-        {:error, "Could not fetch by AP id"}
+      e ->
+        {:error, e}
     end
   end
 
-- 
cgit v1.2.3


From bf2107743ffcd3b7b9c32c49242f1dfd89dcfdb6 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 03:26:50 +0000
Subject: object: containment: don't try to contain ostatus objects

---
 lib/pleroma/object/containment.ex | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex
index f077a9f32..cd8623821 100644
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@@ -57,7 +57,9 @@ defmodule Pleroma.Object.Containment do
     id_uri = URI.parse(id)
     other_uri = URI.parse(other_id)
 
-    if id_uri.host == other_uri.host do
+    # We explicitly allow 'tag' URIs through, due to legacy OStatus objects
+    # being present in the ActivityPub network.
+    if id_uri.host == other_uri.host || other_uri.scheme == "tag" do
       :ok
     else
       :error
-- 
cgit v1.2.3


From a177f22e0242f58db665688ed56dd5f1ccd1999c Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 03:41:38 +0000
Subject: object: fetcher: improve error reporting

---
 lib/pleroma/object/fetcher.ex | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 9436e2730..9af2e02ea 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -66,7 +66,7 @@ defmodule Pleroma.Object.Fetcher do
          {:normalize, nil} <- {:normalize, Object.normalize(data, false)},
          params <- prepare_activity_params(data),
          {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)},
-         {:ok, activity} <- Transmogrifier.handle_incoming(params, options),
+         {:transmogrifier, {:ok, activity}} <- {:transmogrifier, Transmogrifier.handle_incoming(params, options)},
          {:object, _data, %Object{} = object} <-
            {:object, data, Object.normalize(activity, false)} do
       {:ok, object}
@@ -74,9 +74,12 @@ defmodule Pleroma.Object.Fetcher do
       {:containment, _} ->
         {:error, "Object containment failed."}
 
-      {:error, {:reject, nil}} ->
+      {:transmogrifier, {:error, {:reject, nil}}} ->
         {:reject, nil}
 
+      {:transmogrifier, _} ->
+        {:error, "Transmogrifier failure."}
+
       {:object, data, nil} ->
         reinject_object(%Object{}, data)
 
@@ -106,7 +109,8 @@ defmodule Pleroma.Object.Fetcher do
     with {:ok, object} <- fetch_object_from_id(id, options) do
       object
     else
-      _e ->
+      e ->
+        Logger.error("Error while fetching #{id}: #{inspect(e)}")
         nil
     end
   end
@@ -153,7 +157,7 @@ defmodule Pleroma.Object.Fetcher do
 
     Logger.debug("Fetch headers: #{inspect(headers)}")
 
-    with true <- String.starts_with?(id, "http"),
+    with {:scheme, true} <- {:scheme, String.starts_with?(id, "http")},
          {:ok, %{body: body, status: code}} when code in 200..299 <- HTTP.get(id, headers),
          {:ok, data} <- Jason.decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
@@ -162,6 +166,9 @@ defmodule Pleroma.Object.Fetcher do
       {:ok, %{status: code}} when code in [404, 410] ->
         {:error, "Object has been deleted"}
 
+      {:scheme, _} ->
+        {:error, "Unsupported URI scheme"}
+
       e ->
         {:error, e}
     end
-- 
cgit v1.2.3


From 7295a05ceee441311bf56513f5fe889908f59bd5 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 03:56:31 +0000
Subject: object: containment: also allow OStatus object IDs through when
 comparing origins

---
 lib/pleroma/object/containment.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex
index cd8623821..6a621ac26 100644
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@@ -41,7 +41,7 @@ defmodule Pleroma.Object.Containment do
     id_uri = URI.parse(id)
     actor_uri = URI.parse(get_actor(params))
 
-    if id_uri.host == actor_uri.host do
+    if id_uri.host == actor_uri.host || id_uri.scheme == "tag" do
       :ok
     else
       :error
-- 
cgit v1.2.3


From bae96de273250a0054d22c132bd847ab83928ca3 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 03:57:32 +0000
Subject: activitypub: tag containment checks for better error tracing

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index d631e43c6..94c467b69 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -132,7 +132,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          {:ok, map} <- MRF.filter(map),
          {recipients, _, _} = get_recipients(map),
          {:fake, false, map, recipients} <- {:fake, fake, map, recipients},
-         :ok <- Containment.contain_child(map),
+         {:containment, :ok} <- {:containment, Containment.contain_child(map)},
          {:ok, map, object} <- insert_full_object(map) do
       {:ok, activity} =
         Repo.insert(%Activity{
-- 
cgit v1.2.3


From 3c785b85a62267f07f5e6bdd4f12c96ca94639c1 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 04:08:25 +0000
Subject: object: fetcher: fix up formatting

---
 lib/pleroma/object/fetcher.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 9af2e02ea..7758cb90b 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -66,7 +66,8 @@ defmodule Pleroma.Object.Fetcher do
          {:normalize, nil} <- {:normalize, Object.normalize(data, false)},
          params <- prepare_activity_params(data),
          {:containment, :ok} <- {:containment, Containment.contain_origin(id, params)},
-         {:transmogrifier, {:ok, activity}} <- {:transmogrifier, Transmogrifier.handle_incoming(params, options)},
+         {:transmogrifier, {:ok, activity}} <-
+           {:transmogrifier, Transmogrifier.handle_incoming(params, options)},
          {:object, _data, %Object{} = object} <-
            {:object, data, Object.normalize(activity, false)} do
       {:ok, object}
-- 
cgit v1.2.3


From e99fdfc32db231391184220eb28024d358821d27 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 15:34:36 +0000
Subject: object: containment: only allow OStatus references in test suite
 environment

---
 lib/pleroma/object/containment.ex | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex
index 6a621ac26..1beb9c83d 100644
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@@ -32,6 +32,23 @@ defmodule Pleroma.Object.Containment do
     get_actor(%{"actor" => actor})
   end
 
+  # TODO: We explicitly allow 'tag' URIs through, due to references to legacy OStatus
+  # objects being present in the test suite environment.  Once these objects are
+  # removed, please also remove this.
+  if Mix.env() == :test do
+    defp compare_uris(_, %URI{scheme: "tag" <> _}), do: :ok
+  end
+
+  defp compare_uris(%URI{} = id_uri, %URI{} = other_uri) do
+    if id_uri.host == other_uri.host do
+      :ok
+    else
+      :error
+    end
+  end
+
+  defp compare_uris(_, _), do: :error
+
   @doc """
   Checks that an imported AP object's actor matches the domain it came from.
   """
@@ -41,11 +58,7 @@ defmodule Pleroma.Object.Containment do
     id_uri = URI.parse(id)
     actor_uri = URI.parse(get_actor(params))
 
-    if id_uri.host == actor_uri.host || id_uri.scheme == "tag" do
-      :ok
-    else
-      :error
-    end
+    compare_uris(actor_uri, id_uri)
   end
 
   def contain_origin(id, %{"attributedTo" => actor} = params),
@@ -57,13 +70,7 @@ defmodule Pleroma.Object.Containment do
     id_uri = URI.parse(id)
     other_uri = URI.parse(other_id)
 
-    # We explicitly allow 'tag' URIs through, due to legacy OStatus objects
-    # being present in the ActivityPub network.
-    if id_uri.host == other_uri.host || other_uri.scheme == "tag" do
-      :ok
-    else
-      :error
-    end
+    compare_uris(id_uri, other_uri)
   end
 
   def contain_child(%{"object" => %{"id" => id, "attributedTo" => _} = object}),
-- 
cgit v1.2.3


From 44e64af5e76e1ace82aa66973da883e334ebfc93 Mon Sep 17 00:00:00 2001
From: Ariadne Conill <ariadne@dereferenced.org>
Date: Fri, 18 Oct 2019 15:39:15 +0000
Subject: object: containment: simplify the pattern match for OStatus testsuite
 hack

---
 lib/pleroma/object/containment.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex
index 1beb9c83d..68535c09e 100644
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@@ -36,7 +36,7 @@ defmodule Pleroma.Object.Containment do
   # objects being present in the test suite environment.  Once these objects are
   # removed, please also remove this.
   if Mix.env() == :test do
-    defp compare_uris(_, %URI{scheme: "tag" <> _}), do: :ok
+    defp compare_uris(_, %URI{scheme: "tag"}), do: :ok
   end
 
   defp compare_uris(%URI{} = id_uri, %URI{} = other_uri) do
-- 
cgit v1.2.3


From 638457ba94931c35268cf5fb4d2a432a0def6bdd Mon Sep 17 00:00:00 2001
From: KokaKiwi <kokakiwi+git@kokakiwi.net>
Date: Fri, 11 Oct 2019 14:48:01 +0200
Subject: MastoFE: Add PWA manifest.

---
 lib/pleroma/web/masto_fe_controller.ex            |  6 ++++++
 lib/pleroma/web/router.ex                         |  6 ++++++
 lib/pleroma/web/templates/masto_fe/index.html.eex |  6 +++++-
 lib/pleroma/web/views/masto_fe_view.ex            | 19 +++++++++++++++++++
 4 files changed, 36 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/masto_fe_controller.ex b/lib/pleroma/web/masto_fe_controller.ex
index 87860f1d5..93b38e8f4 100644
--- a/lib/pleroma/web/masto_fe_controller.ex
+++ b/lib/pleroma/web/masto_fe_controller.ex
@@ -34,6 +34,12 @@ defmodule Pleroma.Web.MastoFEController do
     end
   end
 
+  @doc "GET /web/manifest.json"
+  def manifest(conn, _params) do
+    conn
+    |> render("manifest.json")
+  end
+
   @doc "PUT /api/web/settings"
   def put_settings(%{assigns: %{user: user}} = conn, %{"data" => settings} = _params) do
     with {:ok, _} <- User.update_info(user, &User.Info.mastodon_settings_update(&1, settings)) do
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index d68fb87da..e7d9ee238 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -591,6 +591,12 @@ defmodule Pleroma.Web.Router do
     get("/:version", Nodeinfo.NodeinfoController, :nodeinfo)
   end
 
+  scope "/", Pleroma.Web do
+    pipe_through(:api)
+
+    get("/web/manifest.json", MastoFEController, :manifest)
+  end
+
   scope "/", Pleroma.Web do
     pipe_through(:mastodon_html)
 
diff --git a/lib/pleroma/web/templates/masto_fe/index.html.eex b/lib/pleroma/web/templates/masto_fe/index.html.eex
index feff36fae..c330960fa 100644
--- a/lib/pleroma/web/templates/masto_fe/index.html.eex
+++ b/lib/pleroma/web/templates/masto_fe/index.html.eex
@@ -4,9 +4,13 @@
 <meta charset='utf-8'>
 <meta content='width=device-width, initial-scale=1' name='viewport'>
 <title>
-<%= Pleroma.Config.get([:instance, :name]) %>
+<%= Config.get([:instance, :name]) %>
 </title>
 <link rel="icon" type="image/png" href="/favicon.png"/>
+<link rel="manifest" type="applicaton/manifest+json" href="<%= masto_fe_path(Pleroma.Web.Endpoint, :manifest) %>" />
+
+<meta name="theme-color" content="<%= Config.get([:manifest, :theme_color]) %>" />
+
 <script crossorigin='anonymous' src="/packs/locales.js"></script>
 <script crossorigin='anonymous' src="/packs/locales/glitch/en.js"></script>
 
diff --git a/lib/pleroma/web/views/masto_fe_view.ex b/lib/pleroma/web/views/masto_fe_view.ex
index 21b086d4c..85b164b59 100644
--- a/lib/pleroma/web/views/masto_fe_view.ex
+++ b/lib/pleroma/web/views/masto_fe_view.ex
@@ -99,4 +99,23 @@ defmodule Pleroma.Web.MastoFEView do
   defp present?(nil), do: false
   defp present?(false), do: false
   defp present?(_), do: true
+
+  def render("manifest.json", _params) do
+    %{
+      name: Config.get([:instance, :name]),
+      description: Config.get([:instance, :description]),
+      icons: Config.get([:manifest, :icons]),
+      theme_color: Config.get([:manifest, :theme_color]),
+      background_color: Config.get([:manifest, :background_color]),
+      display: "standalone",
+      scope: Pleroma.Web.base_url(),
+      start_url: masto_fe_path(Pleroma.Web.Endpoint, :index, ["getting-started"]),
+      categories: [
+        "social"
+      ],
+      serviceworker: %{
+        src: "/sw.js"
+      }
+    }
+  end
 end
-- 
cgit v1.2.3


From 52ed2f8f2d962154ba61b31a3f5aab13dc7217fc Mon Sep 17 00:00:00 2001
From: eugenijm <eugenijm@protonmail.com>
Date: Fri, 11 Oct 2019 06:40:58 +0300
Subject: Pleroma API: `POST /api/v1/pleroma/conversations/read` to mark all
 user's conversations as read

---
 lib/pleroma/conversation/participation.ex                   | 13 +++++++++++++
 .../web/pleroma_api/controllers/pleroma_api_controller.ex   |  9 +++++++++
 lib/pleroma/web/router.ex                                   |  1 +
 3 files changed, 23 insertions(+)

(limited to 'lib')

diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex
index e17f49e58..41918fa78 100644
--- a/lib/pleroma/conversation/participation.ex
+++ b/lib/pleroma/conversation/participation.ex
@@ -69,6 +69,19 @@ defmodule Pleroma.Conversation.Participation do
     end
   end
 
+  def mark_all_as_read(user) do
+    {_, participations} =
+      __MODULE__
+      |> where([p], p.user_id == ^user.id)
+      |> where([p], not p.read)
+      |> update([p], set: [read: true])
+      |> select([p], p)
+      |> Repo.update_all([])
+
+    User.set_unread_conversation_count(user)
+    {:ok, participations}
+  end
+
   def mark_as_unread(participation) do
     participation
     |> read_cng(%{read: false})
diff --git a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
index 9d50a7ca9..fc39abf05 100644
--- a/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
+++ b/lib/pleroma/web/pleroma_api/controllers/pleroma_api_controller.ex
@@ -79,6 +79,15 @@ defmodule Pleroma.Web.PleromaAPI.PleromaAPIController do
     end
   end
 
+  def read_conversations(%{assigns: %{user: user}} = conn, _params) do
+    with {:ok, participations} <- Participation.mark_all_as_read(user) do
+      conn
+      |> add_link_headers(participations)
+      |> put_view(ConversationView)
+      |> render("participations.json", participations: participations, for: user)
+    end
+  end
+
   def read_notification(%{assigns: %{user: user}} = conn, %{"id" => notification_id}) do
     with {:ok, notification} <- Notification.read_one(user, notification_id) do
       conn
diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex
index d68fb87da..66e7a2263 100644
--- a/lib/pleroma/web/router.ex
+++ b/lib/pleroma/web/router.ex
@@ -266,6 +266,7 @@ defmodule Pleroma.Web.Router do
 
       get("/conversations/:id/statuses", PleromaAPIController, :conversation_statuses)
       get("/conversations/:id", PleromaAPIController, :conversation)
+      post("/conversations/read", PleromaAPIController, :read_conversations)
     end
 
     scope [] do
-- 
cgit v1.2.3