summaryrefslogtreecommitdiff
path: root/lib/mix/tasks
diff options
context:
space:
mode:
Diffstat (limited to 'lib/mix/tasks')
-rw-r--r--lib/mix/tasks/pleroma/benchmark.ex113
-rw-r--r--lib/mix/tasks/pleroma/config.ex29
-rw-r--r--lib/mix/tasks/pleroma/database.ex169
-rw-r--r--lib/mix/tasks/pleroma/digest.ex2
-rw-r--r--lib/mix/tasks/pleroma/ecto/rollback.ex2
-rw-r--r--lib/mix/tasks/pleroma/emoji.ex2
-rw-r--r--lib/mix/tasks/pleroma/instance.ex4
-rw-r--r--lib/mix/tasks/pleroma/search/indexer.ex83
-rw-r--r--lib/mix/tasks/pleroma/search/meilisearch.ex145
-rw-r--r--lib/mix/tasks/pleroma/test_runner.ex25
10 files changed, 433 insertions, 141 deletions
diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex
deleted file mode 100644
index f32492169..000000000
--- a/lib/mix/tasks/pleroma/benchmark.ex
+++ /dev/null
@@ -1,113 +0,0 @@
-# Pleroma: A lightweight social networking server
-# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/>
-# SPDX-License-Identifier: AGPL-3.0-only
-
-defmodule Mix.Tasks.Pleroma.Benchmark do
- import Mix.Pleroma
- use Mix.Task
-
- def run(["search"]) do
- start_pleroma()
-
- Benchee.run(%{
- "search" => fn ->
- Pleroma.Activity.search(nil, "cofe")
- end
- })
- end
-
- def run(["tag"]) do
- start_pleroma()
-
- Benchee.run(%{
- "tag" => fn ->
- %{"type" => "Create", "tag" => "cofe"}
- |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
- end
- })
- end
-
- def run(["render_timeline", nickname | _] = args) do
- start_pleroma()
- user = Pleroma.User.get_by_nickname(nickname)
-
- activities =
- %{}
- |> Map.put("type", ["Create", "Announce"])
- |> Map.put("blocking_user", user)
- |> Map.put("muting_user", user)
- |> Map.put("user", user)
- |> Map.put("limit", 4096)
- |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities()
- |> Enum.reverse()
-
- inputs = %{
- "1 activity" => Enum.take_random(activities, 1),
- "10 activities" => Enum.take_random(activities, 10),
- "20 activities" => Enum.take_random(activities, 20),
- "40 activities" => Enum.take_random(activities, 40),
- "80 activities" => Enum.take_random(activities, 80)
- }
-
- inputs =
- if Enum.at(args, 2) == "extended" do
- Map.merge(inputs, %{
- "200 activities" => Enum.take_random(activities, 200),
- "500 activities" => Enum.take_random(activities, 500),
- "2000 activities" => Enum.take_random(activities, 2000),
- "4096 activities" => Enum.take_random(activities, 4096)
- })
- else
- inputs
- end
-
- Benchee.run(
- %{
- "Standart rendering" => fn activities ->
- Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{
- activities: activities,
- for: user,
- as: :activity
- })
- end
- },
- inputs: inputs
- )
- end
-
- def run(["adapters"]) do
- start_pleroma()
-
- :ok =
- Pleroma.Gun.Conn.open(
- "https://httpbin.org/stream-bytes/1500",
- :gun_connections
- )
-
- Process.sleep(1_500)
-
- Benchee.run(
- %{
- "Without conn and without pool" => fn ->
- {:ok, %Tesla.Env{}} =
- Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [],
- pool: :no_pool,
- receive_conn: false
- )
- end,
- "Without conn and with pool" => fn ->
- {:ok, %Tesla.Env{}} =
- Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false)
- end,
- "With reused conn and without pool" => fn ->
- {:ok, %Tesla.Env{}} =
- Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool)
- end,
- "With reused conn and with pool" => fn ->
- {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500")
- end
- },
- parallel: 10
- )
- end
-end
diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex
index 3a2ea44f8..8b3b2f18b 100644
--- a/lib/mix/tasks/pleroma/config.ex
+++ b/lib/mix/tasks/pleroma/config.ex
@@ -205,6 +205,35 @@ defmodule Mix.Tasks.Pleroma.Config do
end
end
+ # Removes any policies that are not a real module
+ # as they will prevent the server from starting
+ def run(["fix_mrf_policies"]) do
+ check_configdb(fn ->
+ start_pleroma()
+
+ group = :pleroma
+ key = :mrf
+
+ %{value: value} =
+ group
+ |> ConfigDB.get_by_group_and_key(key)
+
+ policies =
+ Keyword.get(value, :policies, [])
+ |> Enum.filter(&is_atom(&1))
+ |> Enum.filter(fn mrf ->
+ case Code.ensure_compiled(mrf) do
+ {:module, _} -> true
+ {:error, _} -> false
+ end
+ end)
+
+ value = Keyword.put(value, :policies, policies)
+
+ ConfigDB.update_or_create(%{group: group, key: key, value: value})
+ end)
+ end
+
@spec migrate_to_db(Path.t() | nil) :: any()
def migrate_to_db(file_path \\ nil) do
with :ok <- Pleroma.Config.DeprecationWarnings.warn() do
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index ed560c177..b82d1f079 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -67,43 +67,168 @@ defmodule Mix.Tasks.Pleroma.Database do
OptionParser.parse(
args,
strict: [
- vacuum: :boolean
+ vacuum: :boolean,
+ keep_threads: :boolean,
+ keep_non_public: :boolean,
+ prune_orphaned_activities: :boolean
]
)
start_pleroma()
deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
+ time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400))
- Logger.info("Pruning objects older than #{deadline} days")
+ log_message = "Pruning objects older than #{deadline} days"
- time_deadline =
- NaiveDateTime.utc_now()
- |> NaiveDateTime.add(-(deadline * 86_400))
+ log_message =
+ if Keyword.get(options, :keep_non_public) do
+ log_message <> ", keeping non public posts"
+ else
+ log_message
+ end
- from(o in Object,
- where:
- fragment(
- "?->'to' \\? ? OR ?->'cc' \\? ?",
- o.data,
- ^Pleroma.Constants.as_public(),
- o.data,
- ^Pleroma.Constants.as_public()
- ),
- where: o.inserted_at < ^time_deadline,
- where:
+ log_message =
+ if Keyword.get(options, :keep_threads) do
+ log_message <> ", keeping threads intact"
+ else
+ log_message
+ end
+
+ log_message =
+ if Keyword.get(options, :prune_orphaned_activities) do
+ log_message <> ", pruning orphaned activities"
+ else
+ log_message
+ end
+
+ log_message =
+ if Keyword.get(options, :vacuum) do
+ log_message <>
+ ", doing a full vacuum (you shouldn't do this as a recurring maintanance task)"
+ else
+ log_message
+ end
+
+ Logger.info(log_message)
+
+ if Keyword.get(options, :keep_threads) do
+ # We want to delete objects from threads where
+ # 1. the newest post is still old
+ # 2. none of the activities is local
+ # 3. none of the activities is bookmarked
+ # 4. optionally none of the posts is non-public
+ deletable_context =
+ if Keyword.get(options, :keep_non_public) do
+ Pleroma.Activity
+ |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
+ |> group_by([a], fragment("? ->> 'context'::text", a.data))
+ |> having(
+ [a],
+ not fragment(
+ # Posts (checked on Create Activity) is non-public
+ "bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')",
+ a.data,
+ ^Pleroma.Constants.as_public(),
+ a.data,
+ ^Pleroma.Constants.as_public(),
+ a.data
+ )
+ )
+ else
+ Pleroma.Activity
+ |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id)
+ |> group_by([a], fragment("? ->> 'context'::text", a.data))
+ end
+ |> having([a], max(a.updated_at) < ^time_deadline)
+ |> having([a], not fragment("bool_or(?)", a.local))
+ |> having([_, b], fragment("max(?::text) is null", b.id))
+ |> select([a], fragment("? ->> 'context'::text", a.data))
+
+ Pleroma.Object
+ |> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context))
+ else
+ if Keyword.get(options, :keep_non_public) do
+ Pleroma.Object
+ |> where(
+ [o],
+ fragment(
+ "?->'to' \\? ? OR ?->'cc' \\? ?",
+ o.data,
+ ^Pleroma.Constants.as_public(),
+ o.data,
+ ^Pleroma.Constants.as_public()
+ )
+ )
+ else
+ Pleroma.Object
+ end
+ |> where([o], o.updated_at < ^time_deadline)
+ |> where(
+ [o],
fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
- )
+ )
+ end
|> Repo.delete_all(timeout: :infinity)
- prune_hashtags_query = """
+ if !Keyword.get(options, :keep_threads) do
+ # Without the --keep-threads option, it's possible that bookmarked
+ # objects have been deleted. We remove the corresponding bookmarks.
+ """
+ delete from public.bookmarks
+ where id in (
+ select b.id from public.bookmarks b
+ left join public.activities a on b.activity_id = a.id
+ left join public.objects o on a."data" ->> 'object' = o.data ->> 'id'
+ where o.id is null
+ )
+ """
+ |> Repo.query([], timeout: :infinity)
+ end
+
+ if Keyword.get(options, :prune_orphaned_activities) do
+ # Prune activities who link to a single object
+ """
+ delete from public.activities
+ where id in (
+ select a.id from public.activities a
+ left join public.objects o on a.data ->> 'object' = o.data ->> 'id'
+ left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id'
+ left join public.users u on a.data ->> 'object' = u.ap_id
+ where not a.local
+ and jsonb_typeof(a."data" -> 'object') = 'string'
+ and o.id is null
+ and a2.id is null
+ and u.id is null
+ )
+ """
+ |> Repo.query([], timeout: :infinity)
+
+ # Prune activities who link to an array of objects
+ """
+ delete from public.activities
+ where id in (
+ select a.id from public.activities a
+ join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array'
+ left join public.objects o on j.value = o.data ->> 'id'
+ left join public.activities a2 on j.value = a2.data ->> 'id'
+ left join public.users u on j.value = u.ap_id
+ group by a.id
+ having max(o.data ->> 'id') is null
+ and max(a2.data ->> 'id') is null
+ and max(u.ap_id) is null
+ )
+ """
+ |> Repo.query([], timeout: :infinity)
+ end
+
+ """
DELETE FROM hashtags AS ht
WHERE NOT EXISTS (
SELECT 1 FROM hashtags_objects hto
WHERE ht.id = hto.hashtag_id)
"""
-
- Repo.query(prune_hashtags_query)
+ |> Repo.query()
if Keyword.get(options, :vacuum) do
Maintenance.vacuum("full")
@@ -193,7 +318,7 @@ defmodule Mix.Tasks.Pleroma.Database do
"ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';"
)
- # non-exist config will not raise excpetion but only give >0 messages
+ # non-exist config will not raise exception but only give >0 messages
if length(msg) > 0 do
shell_info("Error: #{inspect(msg, pretty: true)}")
else
@@ -226,7 +351,7 @@ defmodule Mix.Tasks.Pleroma.Database do
)
end
- shell_info('Done.')
+ shell_info(~c"Done.")
end
end
diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex
index aea9c8ac5..53cac0b94 100644
--- a/lib/mix/tasks/pleroma/digest.ex
+++ b/lib/mix/tasks/pleroma/digest.ex
@@ -30,7 +30,7 @@ defmodule Mix.Tasks.Pleroma.Digest do
shell_info("Digest email have been sent to #{nickname} (#{user.email})")
else
_ ->
- shell_info("Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}")
+ shell_info("Couldn't find any mentions for #{nickname} since #{last_digest_emailed_at}")
end
end
end
diff --git a/lib/mix/tasks/pleroma/ecto/rollback.ex b/lib/mix/tasks/pleroma/ecto/rollback.ex
index 3d78eaec4..121890f39 100644
--- a/lib/mix/tasks/pleroma/ecto/rollback.ex
+++ b/lib/mix/tasks/pleroma/ecto/rollback.ex
@@ -61,7 +61,7 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do
Logger.configure(level: :info)
if opts[:env] == "test" do
- Logger.info("Rollback succesfully")
+ Logger.info("Rollback successfully")
else
{:ok, _, _} =
Ecto.Migrator.with_repo(Pleroma.Repo, &Ecto.Migrator.run(&1, path, :down, opts))
diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex
index 537f0715e..8b9c921c8 100644
--- a/lib/mix/tasks/pleroma/emoji.ex
+++ b/lib/mix/tasks/pleroma/emoji.ex
@@ -111,7 +111,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do
{:ok, _} =
:zip.unzip(binary_archive,
- cwd: pack_path,
+ cwd: String.to_charlist(pack_path),
file_list: files_to_unzip
)
diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex
index 5d8b254a2..0dc30549c 100644
--- a/lib/mix/tasks/pleroma/instance.ex
+++ b/lib/mix/tasks/pleroma/instance.ex
@@ -292,7 +292,7 @@ defmodule Mix.Tasks.Pleroma.Instance do
if db_configurable? do
shell_info(
- " Please transfer your config to the database after running database migrations. Refer to \"Transfering the config to/from the database\" section of the docs for more information."
+ " Please transfer your config to the database after running database migrations. Refer to \"Transferring the config to/from the database\" section of the docs for more information."
)
end
else
@@ -352,6 +352,4 @@ defmodule Mix.Tasks.Pleroma.Instance do
enabled_filters
end
-
- defp upload_filters(_), do: []
end
diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
new file mode 100644
index 000000000..2a52472f9
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -0,0 +1,83 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Indexer do
+ import Mix.Pleroma
+ import Ecto.Query
+
+ alias Pleroma.Workers.SearchIndexingWorker
+
+ def run(["create_index"]) do
+ start_pleroma()
+
+ with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
+ IO.puts("Index created")
+ else
+ e -> IO.puts("Could not create index: #{inspect(e)}")
+ end
+ end
+
+ def run(["drop_index"]) do
+ start_pleroma()
+
+ with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
+ IO.puts("Index dropped")
+ else
+ e -> IO.puts("Could not drop index: #{inspect(e)}")
+ end
+ end
+
+ def run(["index" | options]) do
+ {options, [], []} =
+ OptionParser.parse(
+ options,
+ strict: [
+ chunk: :integer,
+ limit: :integer,
+ step: :integer
+ ]
+ )
+
+ start_pleroma()
+
+ chunk_size = Keyword.get(options, :chunk, 100)
+ limit = Keyword.get(options, :limit, 100_000)
+ per_step = Keyword.get(options, :step, 1000)
+
+ chunks = max(div(limit, per_step), 1)
+
+ 1..chunks
+ |> Enum.each(fn step ->
+ q =
+ from(a in Pleroma.Activity,
+ limit: ^per_step,
+ offset: ^per_step * (^step - 1),
+ select: [:id],
+ order_by: [desc: :id]
+ )
+
+ {:ok, ids} =
+ Pleroma.Repo.transaction(fn ->
+ Pleroma.Repo.stream(q, timeout: :infinity)
+ |> Enum.map(fn a ->
+ a.id
+ end)
+ end)
+
+ IO.puts("Got #{length(ids)} activities, adding to indexer")
+
+ ids
+ |> Enum.chunk_every(chunk_size)
+ |> Enum.each(fn chunk ->
+ IO.puts("Adding #{length(chunk)} activities to indexing queue")
+
+ chunk
+ |> Enum.map(fn id ->
+ SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
+ end)
+ |> Oban.insert_all()
+ end)
+ end)
+ end
+end
diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex
new file mode 100644
index 000000000..8379a0c25
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search/meilisearch.ex
@@ -0,0 +1,145 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Meilisearch do
+ require Pleroma.Constants
+
+ import Mix.Pleroma
+ import Ecto.Query
+
+ import Pleroma.Search.Meilisearch,
+ only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1]
+
+ def run(["index"]) do
+ start_pleroma()
+ Pleroma.HTML.compile_scrubbers()
+
+ meili_version =
+ (
+ {:ok, result} = meili_get("/version")
+
+ result["pkgVersion"]
+ )
+
+ # The ranking rule syntax was changed but nothing about that is mentioned in the changelog
+ if not Version.match?(meili_version, ">= 0.25.0") do
+ raise "Meilisearch <0.24.0 not supported"
+ end
+
+ {:ok, _} =
+ meili_post(
+ "/indexes/objects/settings/ranking-rules",
+ [
+ "published:desc",
+ "words",
+ "exactness",
+ "proximity",
+ "typo",
+ "attribute",
+ "sort"
+ ]
+ )
+
+ {:ok, _} =
+ meili_post(
+ "/indexes/objects/settings/searchable-attributes",
+ [
+ "content"
+ ]
+ )
+
+ IO.puts("Created indices. Starting to insert posts.")
+
+ chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size])
+
+ Pleroma.Repo.transaction(
+ fn ->
+ query =
+ from(Pleroma.Object,
+ # Only index public and unlisted posts which are notes and have some text
+ where:
+ fragment("data->>'type' = 'Note'") and
+ (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or
+ fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())),
+ order_by: [desc: fragment("data->'published'")]
+ )
+
+ count = query |> Pleroma.Repo.aggregate(:count, :data)
+ IO.puts("Entries to index: #{count}")
+
+ Pleroma.Repo.stream(
+ query,
+ timeout: :infinity
+ )
+ |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1)
+ |> Stream.filter(fn o -> not is_nil(o) end)
+ |> Stream.chunk_every(chunk_size)
+ |> Stream.transform(0, fn objects, acc ->
+ new_acc = acc + Enum.count(objects)
+
+ # Reset to the beginning of the line and rewrite it
+ IO.write("\r")
+ IO.write("Indexed #{new_acc} entries")
+
+ {[objects], new_acc}
+ end)
+ |> Stream.each(fn objects ->
+ result =
+ meili_put(
+ "/indexes/objects/documents",
+ objects
+ )
+
+ with {:ok, res} <- result do
+ if not Map.has_key?(res, "uid") do
+ IO.puts("\nFailed to index: #{inspect(result)}")
+ end
+ else
+ e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}")
+ end
+ end)
+ |> Stream.run()
+ end,
+ timeout: :infinity
+ )
+
+ IO.write("\n")
+ end
+
+ def run(["clear"]) do
+ start_pleroma()
+
+ meili_delete("/indexes/objects/documents")
+ end
+
+ def run(["show-keys", master_key]) do
+ start_pleroma()
+
+ endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url])
+
+ {:ok, result} =
+ Pleroma.HTTP.get(
+ Path.join(endpoint, "/keys"),
+ [{"Authorization", "Bearer #{master_key}"}]
+ )
+
+ decoded = Jason.decode!(result.body)
+
+ if decoded["results"] do
+ Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} ->
+ IO.puts("#{desc}: #{key}")
+ end)
+ else
+ IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}")
+ end
+ end
+
+ def run(["stats"]) do
+ start_pleroma()
+
+ {:ok, result} = meili_get("/indexes/objects/stats")
+ IO.puts("Number of entries: #{result["numberOfDocuments"]}")
+ IO.puts("Indexing? #{result["isIndexing"]}")
+ end
+end
diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex
new file mode 100644
index 000000000..69fefb001
--- /dev/null
+++ b/lib/mix/tasks/pleroma/test_runner.ex
@@ -0,0 +1,25 @@
+defmodule Mix.Tasks.Pleroma.TestRunner do
+ @shortdoc "Retries tests once if they fail"
+
+ use Mix.Task
+
+ def run(args \\ []) do
+ case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do
+ {_, 0} ->
+ :ok
+
+ _ ->
+ retry(args)
+ end
+ end
+
+ def retry(args) do
+ case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do
+ {_, 0} ->
+ :ok
+
+ _ ->
+ exit(1)
+ end
+ end
+end
generated by cgit v1.2.3 (git 2.25.1) at 2025-05-29 12:14:26 +0000