From 1bf3ae07b6719a762310615811a317035175ad2e Mon Sep 17 00:00:00 2001 From: faried nawaz Date: Mon, 18 Sep 2023 02:13:01 +0500 Subject: add options to mix pleroma.database prune_objects to delete more activities --- lib/mix/tasks/pleroma/database.ex | 165 +++++++++++++++++++++++++++++++++----- 1 file changed, 145 insertions(+), 20 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 93ee57dc3..13ac6536c 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -67,43 +67,168 @@ defmodule Mix.Tasks.Pleroma.Database do OptionParser.parse( args, strict: [ - vacuum: :boolean + vacuum: :boolean, + keep_threads: :boolean, + keep_non_public: :boolean, + prune_orphaned_activities: :boolean ] ) start_pleroma() deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + time_deadline = NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400)) - Logger.info("Pruning objects older than #{deadline} days") + log_message = "Pruning objects older than #{deadline} days" - time_deadline = - NaiveDateTime.utc_now() - |> NaiveDateTime.add(-(deadline * 86_400)) + log_message = + if Keyword.get(options, :keep_non_public) do + log_message <> ", keeping non public posts" + else + log_message + end - from(o in Object, - where: - fragment( - "?->'to' \\? ? OR ?->'cc' \\? ?", - o.data, - ^Pleroma.Constants.as_public(), - o.data, - ^Pleroma.Constants.as_public() - ), - where: o.inserted_at < ^time_deadline, - where: + log_message = + if Keyword.get(options, :keep_threads) do + log_message <> ", keeping threads intact" + else + log_message + end + + log_message = + if Keyword.get(options, :prune_orphaned_activities) do + log_message <> ", pruning orphaned activities" + else + log_message + end + + log_message = + if Keyword.get(options, :vacuum) do + log_message <> + ", doing a full vacuum (you shouldn't do this as a recurring maintanance task)" + else + log_message + end + + Logger.info(log_message) + + if Keyword.get(options, :keep_threads) do + # We want to delete objects from threads where + # 1. the newest post is still old + # 2. none of the activities is local + # 3. none of the activities is bookmarked + # 4. optionally none of the posts is non-public + deletable_context = + if Keyword.get(options, :keep_non_public) do + Pleroma.Activity + |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id) + |> group_by([a], fragment("? ->> 'context'::text", a.data)) + |> having( + [a], + not fragment( + # Posts (checked on Create Activity) is non-public + "bool_or((not(?->'to' \\? ? OR ?->'cc' \\? ?)) and ? ->> 'type' = 'Create')", + a.data, + ^Pleroma.Constants.as_public(), + a.data, + ^Pleroma.Constants.as_public(), + a.data + ) + ) + else + Pleroma.Activity + |> join(:left, [a], b in Pleroma.Bookmark, on: a.id == b.activity_id) + |> group_by([a], fragment("? ->> 'context'::text", a.data)) + end + |> having([a], max(a.updated_at) < ^time_deadline) + |> having([a], not fragment("bool_or(?)", a.local)) + |> having([_, b], fragment("max(?::text) is null", b.id)) + |> select([a], fragment("? ->> 'context'::text", a.data)) + + Pleroma.Object + |> where([o], fragment("? ->> 'context'::text", o.data) in subquery(deletable_context)) + else + if Keyword.get(options, :keep_non_public) do + Pleroma.Object + |> where( + [o], + fragment( + "?->'to' \\? ? OR ?->'cc' \\? ?", + o.data, + ^Pleroma.Constants.as_public(), + o.data, + ^Pleroma.Constants.as_public() + ) + ) + else + Pleroma.Object + end + |> where([o], o.updated_at < ^time_deadline) + |> where( + [o], fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) - ) + ) + end |> Repo.delete_all(timeout: :infinity) - prune_hashtags_query = """ + if !Keyword.get(options, :keep_threads) do + # Without the --keep-threads option, it's possible that bookmarked + # objects have been deleted. We remove the corresponding bookmarks. + """ + delete from public.bookmarks + where id in ( + select b.id from public.bookmarks b + left join public.activities a on b.activity_id = a.id + left join public.objects o on a."data" ->> 'object' = o.data ->> 'id' + where o.id is null + ) + """ + |> Repo.query([], timeout: :infinity) + end + + if Keyword.get(options, :prune_orphaned_activities) do + # Prune activities who link to a single object + """ + delete from public.activities + where id in ( + select a.id from public.activities a + left join public.objects o on a.data ->> 'object' = o.data ->> 'id' + left join public.activities a2 on a.data ->> 'object' = a2.data ->> 'id' + left join public.users u on a.data ->> 'object' = u.ap_id + where not a.local + and jsonb_typeof(a."data" -> 'object') = 'string' + and o.id is null + and a2.id is null + and u.id is null + ) + """ + |> Repo.query([], timeout: :infinity) + + # Prune activities who link to an array of objects + """ + delete from public.activities + where id in ( + select a.id from public.activities a + join json_array_elements_text((a."data" -> 'object')::json) as j on jsonb_typeof(a."data" -> 'object') = 'array' + left join public.objects o on j.value = o.data ->> 'id' + left join public.activities a2 on j.value = a2.data ->> 'id' + left join public.users u on j.value = u.ap_id + group by a.id + having max(o.data ->> 'id') is null + and max(a2.data ->> 'id') is null + and max(u.ap_id) is null + ) + """ + |> Repo.query([], timeout: :infinity) + end + + """ DELETE FROM hashtags AS ht WHERE NOT EXISTS ( SELECT 1 FROM hashtags_objects hto WHERE ht.id = hto.hashtag_id) """ - - Repo.query(prune_hashtags_query) + |> Repo.query() if Keyword.get(options, :vacuum) do Maintenance.vacuum("full") -- cgit v1.2.3 From cd7e2138d11901fc7a0c8c2f22b7a5d57383a555 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 14:13:37 +0400 Subject: Search: Basic Qdrant/Ollama search --- lib/mix/tasks/pleroma/search/indexer.ex | 60 +++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 lib/mix/tasks/pleroma/search/indexer.ex (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex new file mode 100644 index 000000000..ffa2f3c94 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -0,0 +1,60 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Indexer do + import Mix.Pleroma + import Ecto.Query + + alias Pleroma.Workers.SearchIndexingWorker + + def run(["index" | options]) do + {options, [], []} = + OptionParser.parse( + options, + strict: [ + limit: :integer + ] + ) + + start_pleroma() + + limit = Keyword.get(options, :limit, 100_000) + + per_step = 1000 + chunks = max(div(limit, per_step), 1) + + 1..chunks + |> Enum.each(fn step -> + q = + from(a in Pleroma.Activity, + limit: ^per_step, + offset: ^per_step * (^step - 1), + select: [:id], + order_by: [desc: :id] + ) + + {:ok, ids} = + Pleroma.Repo.transaction(fn -> + Pleroma.Repo.stream(q, timeout: :infinity) + |> Enum.map(fn a -> + a.id + end) + end) + + IO.puts("Got #{length(ids)} activities, adding to indexer") + + ids + |> Enum.chunk_every(100) + |> Enum.each(fn chunk -> + IO.puts("Adding #{length(chunk)} activities to indexing queue") + + chunk + |> Enum.map(fn id -> + SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id}) + end) + |> Oban.insert_all() + end) + end) + end +end -- cgit v1.2.3 From 1261c43a7af48ed6e6753461944659391c4c58cc Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Tue, 14 May 2024 17:19:36 +0400 Subject: SearchBackend: Add create_index --- lib/mix/tasks/pleroma/search/indexer.ex | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index ffa2f3c94..326646b69 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -8,6 +8,12 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do alias Pleroma.Workers.SearchIndexingWorker + def run(["create_index"]) do + Application.ensure_all_started(:pleroma) + + Pleroma.Config.get([Pleroma.Search, :module]).create_index() + end + def run(["index" | options]) do {options, [], []} = OptionParser.parse( -- cgit v1.2.3 From a9be4907c0d7b34e5564584d2d040632c32f2aa3 Mon Sep 17 00:00:00 2001 From: Lain Soykaf Date: Thu, 16 May 2024 10:47:24 +0400 Subject: SearchBackend: Add drop_index --- lib/mix/tasks/pleroma/search/indexer.ex | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index 326646b69..81a9fced6 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -9,9 +9,23 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do alias Pleroma.Workers.SearchIndexingWorker def run(["create_index"]) do - Application.ensure_all_started(:pleroma) + start_pleroma() + + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do + IO.puts("Index created") + else + e -> IO.puts("Could not create index: #{inspect(e)}") + end + end + + def run(["drop_index"]) do + start_pleroma() - Pleroma.Config.get([Pleroma.Search, :module]).create_index() + with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do + IO.puts("Index dropped") + else + e -> IO.puts("Could not drop index: #{inspect(e)}") + end end def run(["index" | options]) do -- cgit v1.2.3 From 14b4bd69a83846b3c117624851b96b9fc30528bf Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 29 May 2024 10:44:34 -0400 Subject: Add additional flags to the Pleroma.Search.Indexer Mix task --- lib/mix/tasks/pleroma/search/indexer.ex | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex index 81a9fced6..2a52472f9 100644 --- a/lib/mix/tasks/pleroma/search/indexer.ex +++ b/lib/mix/tasks/pleroma/search/indexer.ex @@ -33,15 +33,18 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do OptionParser.parse( options, strict: [ - limit: :integer + chunk: :integer, + limit: :integer, + step: :integer ] ) start_pleroma() + chunk_size = Keyword.get(options, :chunk, 100) limit = Keyword.get(options, :limit, 100_000) + per_step = Keyword.get(options, :step, 1000) - per_step = 1000 chunks = max(div(limit, per_step), 1) 1..chunks @@ -65,7 +68,7 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do IO.puts("Got #{length(ids)} activities, adding to indexer") ids - |> Enum.chunk_every(100) + |> Enum.chunk_every(chunk_size) |> Enum.each(fn chunk -> IO.puts("Adding #{length(chunk)} activities to indexing queue") -- cgit v1.2.3 From cb91dab75f70ade96028c32e270a27352c41714e Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Fri, 31 May 2024 07:19:48 +0200 Subject: Switch formatting checks to Elixir 1.15 --- lib/mix/tasks/pleroma/database.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 13ac6536c..b82d1f079 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -351,7 +351,7 @@ defmodule Mix.Tasks.Pleroma.Database do ) end - shell_info('Done.') + shell_info(~c"Done.") end end -- cgit v1.2.3 From ed2976b237b53a524247d564691e0a12d3231d68 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 19 Jun 2024 19:04:09 -0400 Subject: Custom mix task to retry failed tests once in CI pipeline This will be temporary* as we hunt down the cause of the random test failures * gonna regret this --- lib/mix/tasks/pleroma/test_runner.ex | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 lib/mix/tasks/pleroma/test_runner.ex (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/test_runner.ex b/lib/mix/tasks/pleroma/test_runner.ex new file mode 100644 index 000000000..69fefb001 --- /dev/null +++ b/lib/mix/tasks/pleroma/test_runner.ex @@ -0,0 +1,25 @@ +defmodule Mix.Tasks.Pleroma.TestRunner do + @shortdoc "Retries tests once if they fail" + + use Mix.Task + + def run(args \\ []) do + case System.cmd("mix", ["test"] ++ args, into: IO.stream(:stdio, :line)) do + {_, 0} -> + :ok + + _ -> + retry(args) + end + end + + def retry(args) do + case System.cmd("mix", ["test", "--failed"] ++ args, into: IO.stream(:stdio, :line)) do + {_, 0} -> + :ok + + _ -> + exit(1) + end + end +end -- cgit v1.2.3 From c9203f125c3ce44bc7b7273fedca089aadfb6b86 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 15 Jul 2024 15:21:16 -0400 Subject: Added a Mix task "pleroma.config fix_mrf_policies" which will remove erroneous MRF policies from ConfigDB --- lib/mix/tasks/pleroma/config.ex | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 3a2ea44f8..8b3b2f18b 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -205,6 +205,35 @@ defmodule Mix.Tasks.Pleroma.Config do end end + # Removes any policies that are not a real module + # as they will prevent the server from starting + def run(["fix_mrf_policies"]) do + check_configdb(fn -> + start_pleroma() + + group = :pleroma + key = :mrf + + %{value: value} = + group + |> ConfigDB.get_by_group_and_key(key) + + policies = + Keyword.get(value, :policies, []) + |> Enum.filter(&is_atom(&1)) + |> Enum.filter(fn mrf -> + case Code.ensure_compiled(mrf) do + {:module, _} -> true + {:error, _} -> false + end + end) + + value = Keyword.put(value, :policies, policies) + + ConfigDB.update_or_create(%{group: group, key: key, value: value}) + end) + end + @spec migrate_to_db(Path.t() | nil) :: any() def migrate_to_db(file_path \\ nil) do with :ok <- Pleroma.Config.DeprecationWarnings.warn() do -- cgit v1.2.3