Merge branch 'develop' of git.pleroma.social:pleroma/pleroma into tusooa/extract-fix

7 files changed, 160 insertions, 123 deletions

diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex
deleted file mode 100644
index f32492169..000000000
--- a/lib/mix/tasks/pleroma/benchmark.ex
+++ /dev/null
@@ -1,113 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.Benchmark do
-  import Mix.Pleroma
-  use Mix.Task
-
-  def run(["search"]) do
-    start_pleroma()
-
-    Benchee.run(%{
-      "search" => fn ->
-        Pleroma.Activity.search(nil, "cofe")
-      end
-    })
-  end
-
-  def run(["tag"]) do
-    start_pleroma()
-
-    Benchee.run(%{
-      "tag" => fn ->
-        %{"type" => "Create", "tag" => "cofe"}
-        |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
-      end
-    })
-  end
-
-  def run(["render_timeline", nickname | _] = args) do
-    start_pleroma()
-    user = Pleroma.User.get_by_nickname(nickname)
-
-    activities =
-      %{}
-      |> Map.put("type", ["Create", "Announce"])
-      |> Map.put("blocking_user", user)
-      |> Map.put("muting_user", user)
-      |> Map.put("user", user)
-      |> Map.put("limit", 4096)
-      |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
-      |> Enum.reverse()
-
-    inputs = %{
-      "1 activity" => Enum.take_random(activities, 1),
-      "10 activities" => Enum.take_random(activities, 10),
-      "20 activities" => Enum.take_random(activities, 20),
-      "40 activities" => Enum.take_random(activities, 40),
-      "80 activities" => Enum.take_random(activities, 80)
-    }
-
-    inputs =
-      if Enum.at(args, 2) == "extended" do
-        Map.merge(inputs, %{
-          "200 activities" => Enum.take_random(activities, 200),
-          "500 activities" => Enum.take_random(activities, 500),
-          "2000 activities" => Enum.take_random(activities, 2000),
-          "4096 activities" => Enum.take_random(activities, 4096)
-        })
-      else
-        inputs
-      end
-
-    Benchee.run(
-      %{
-        "Standart rendering" => fn activities ->
-          Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
-            activities: activities,
-            for: user,
-            as: :activity
-          })
-        end
-      },
-      inputs: inputs
-    )
-  end
-
-  def run(["adapters"]) do
-    start_pleroma()
-
-    :ok =
-      Pleroma.Gun.Conn.open(
-        "https://httpbin.org/stream-bytes/1500",
-        :gun_connections
-      )
-
-    Process.sleep(1_500)
-
-    Benchee.run(
-      %{
-        "Without conn and without pool" => fn ->
-          {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
-              pool: :no_pool,
-              receive_conn: false
-            )
-        end,
-        "Without conn and with pool" => fn ->
-          {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false)
-        end,
-        "With reused conn and without pool" => fn ->
-          {:ok, %Tesla.Env{}} =
-            Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool)
-        end,
-        "With reused conn and with pool" => fn ->
-          {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
-        end
-      },
-      parallel: 10
-    )
-  end
-end
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index ed560c177..93ee57dc3 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -193,7 +193,7 @@ defmodule Mix.Tasks.Pleroma.Database do
         "ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';"
       )
 
-    # non-exist config will not raise excpetion but only give >0 messages
+    # non-exist config will not raise exception but only give >0 messages
     if length(msg) > 0 do
       shell_info("Error: #{inspect(msg, pretty: true)}")
     else
diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex
index aea9c8ac5..53cac0b94 100644
--- a/lib/mix/tasks/pleroma/digest.ex
+++ b/lib/mix/tasks/pleroma/digest.ex
@@ -30,7 +30,7 @@ defmodule Mix.Tasks.Pleroma.Digest do
       shell_info("Digest email have been sent to #{nickname} (#{user.email})")
     else
       _ ->
-        shell_info("Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}")
+        shell_info("Couldn't find any mentions for #{nickname} since #{last_digest_emailed_at}")
     end
   end
 end
diff --git a/lib/mix/tasks/pleroma/ecto/rollback.ex b/lib/mix/tasks/pleroma/ecto/rollback.ex
index 3d78eaec4..121890f39 100644
--- a/lib/mix/tasks/pleroma/ecto/rollback.ex
+++ b/lib/mix/tasks/pleroma/ecto/rollback.ex
@@ -61,7 +61,7 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do
     Logger.configure(level: :info)
 
     if opts[:env] == "test" do
-      Logger.info("Rollback succesfully")
+      Logger.info("Rollback successfully")
     else
       {:ok, _, _} =
         Ecto.Migrator.with_repo(Pleroma.Repo, &Ecto.Migrator.run(&1, path, :down, opts))
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index 537f0715e..8b9c921c8 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -111,7 +111,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
 
         {:ok, _} =
           :zip.unzip(binary_archive,
-            cwd: pack_path,
+            cwd: String.to_charlist(pack_path),
             file_list: files_to_unzip
           )
 
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 5c93f19ff..0dc30549c 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -266,12 +266,20 @@ defmodule Mix.Tasks.Pleroma.Instance do
       config_dir = Path.dirname(config_path)
       psql_dir = Path.dirname(psql_path)
 
+      # Note: Distros requiring group read (0o750) on those directories should
+      # pre-create the directories.
       [config_dir, psql_dir, static_dir, uploads_dir]
       |> Enum.reject(&File.exists?/1)
-      |> Enum.map(&File.mkdir_p!/1)
+      |> Enum.each(fn dir ->
+        File.mkdir_p!(dir)
+        File.chmod!(dir, 0o700)
+      end)
 
       shell_info("Writing config to #{config_path}.")
 
+      # Sadly no fchmod(2) equivalent in Elixir…
+      File.touch!(config_path)
+      File.chmod!(config_path, 0o640)
       File.write(config_path, result_config)
       shell_info("Writing the postgres script to #{psql_path}.")
       File.write(psql_path, result_psql)
@@ -284,14 +292,13 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
       if db_configurable? do
         shell_info(
-          " Please transfer your config to the database after running database migrations. Refer to \"Transfering the config to/from the database\" section of the docs for more information."
+          " Please transfer your config to the database after running database migrations. Refer to \"Transferring the config to/from the database\" section of the docs for more information."
         )
       end
     else
       shell_error(
         "The task would have overwritten the following files:\n" <>
-          (Enum.map(will_overwrite, &"- #{&1}\n") |> Enum.join("")) <>
-          "Rerun with `--force` to overwrite them."
+          Enum.map_join(will_overwrite, &"- #{&1}\n") <> "Rerun with `--force` to overwrite them."
       )
     end
   end
@@ -345,6 +352,4 @@ defmodule Mix.Tasks.Pleroma.Instance do
 
     enabled_filters
   end
-
-  defp upload_filters(_), do: []
 end
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
new file mode 100644
index 000000000..8379a0c25
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -0,0 +1,145 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
+  require Pleroma.Constants
+
+  import Mix.Pleroma
+  import Ecto.Query
+
+  import Pleroma.Search.Meilisearch,
+    only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1]
+
+  def run(["index"]) do
+    start_pleroma()
+    Pleroma.HTML.compile_scrubbers()
+
+    meili_version =
+      (
+        {:ok, result} = meili_get("/version")
+
+        result["pkgVersion"]
+      )
+
+    # The ranking rule syntax was changed but nothing about that is mentioned in the changelog
+    if not Version.match?(meili_version, ">= 0.25.0") do
+      raise "Meilisearch <0.24.0 not supported"
+    end
+
+    {:ok, _} =
+      meili_post(
+        "/indexes/objects/settings/ranking-rules",
+        [
+          "published:desc",
+          "words",
+          "exactness",
+          "proximity",
+          "typo",
+          "attribute",
+          "sort"
+        ]
+      )
+
+    {:ok, _} =
+      meili_post(
+        "/indexes/objects/settings/searchable-attributes",
+        [
+          "content"
+        ]
+      )
+
+    IO.puts("Created indices. Starting to insert posts.")
+
+    chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size])
+
+    Pleroma.Repo.transaction(
+      fn ->
+        query =
+          from(Pleroma.Object,
+            # Only index public and unlisted posts which are notes and have some text
+            where:
+              fragment("data->>'type' = 'Note'") and
+                (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or
+                   fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())),
+            order_by: [desc: fragment("data->'published'")]
+          )
+
+        count = query |> Pleroma.Repo.aggregate(:count, :data)
+        IO.puts("Entries to index: #{count}")
+
+        Pleroma.Repo.stream(
+          query,
+          timeout: :infinity
+        )
+        |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1)
+        |> Stream.filter(fn o -> not is_nil(o) end)
+        |> Stream.chunk_every(chunk_size)
+        |> Stream.transform(0, fn objects, acc ->
+          new_acc = acc + Enum.count(objects)
+
+          # Reset to the beginning of the line and rewrite it
+          IO.write("\r")
+          IO.write("Indexed #{new_acc} entries")
+
+          {[objects], new_acc}
+        end)
+        |> Stream.each(fn objects ->
+          result =
+            meili_put(
+              "/indexes/objects/documents",
+              objects
+            )
+
+          with {:ok, res} <- result do
+            if not Map.has_key?(res, "uid") do
+              IO.puts("\nFailed to index: #{inspect(result)}")
+            end
+          else
+            e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}")
+          end
+        end)
+        |> Stream.run()
+      end,
+      timeout: :infinity
+    )
+
+    IO.write("\n")
+  end
+
+  def run(["clear"]) do
+    start_pleroma()
+
+    meili_delete("/indexes/objects/documents")
+  end
+
+  def run(["show-keys", master_key]) do
+    start_pleroma()
+
+    endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+    {:ok, result} =
+      Pleroma.HTTP.get(
+        Path.join(endpoint, "/keys"),
+        [{"Authorization", "Bearer #{master_key}"}]
+      )
+
+    decoded = Jason.decode!(result.body)
+
+    if decoded["results"] do
+      Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} ->
+        IO.puts("#{desc}: #{key}")
+      end)
+    else
+      IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}")
+    end
+  end
+
+  def run(["stats"]) do
+    start_pleroma()
+
+    {:ok, result} = meili_get("/indexes/objects/stats")
+    IO.puts("Number of entries: #{result["numberOfDocuments"]}")
+    IO.puts("Indexing? #{result["isIndexing"]}")
+  end
+end