From cd7e2138d11901fc7a0c8c2f22b7a5d57383a555 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 14 May 2024 14:13:37 +0400
Subject: Search: Basic Qdrant/Ollama search

---
 lib/mix/tasks/pleroma/search/indexer.ex | 60 +++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100644 lib/mix/tasks/pleroma/search/indexer.ex

(limited to 'lib/mix')

diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
new file mode 100644
index 000000000..ffa2f3c94
--- /dev/null
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -0,0 +1,60 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Search.Indexer do
+  import Mix.Pleroma
+  import Ecto.Query
+
+  alias Pleroma.Workers.SearchIndexingWorker
+
+  def run(["index" | options]) do
+    {options, [], []} =
+      OptionParser.parse(
+        options,
+        strict: [
+          limit: :integer
+        ]
+      )
+
+    start_pleroma()
+
+    limit = Keyword.get(options, :limit, 100_000)
+
+    per_step = 1000
+    chunks = max(div(limit, per_step), 1)
+
+    1..chunks
+    |> Enum.each(fn step ->
+      q =
+        from(a in Pleroma.Activity,
+          limit: ^per_step,
+          offset: ^per_step * (^step - 1),
+          select: [:id],
+          order_by: [desc: :id]
+        )
+
+      {:ok, ids} =
+        Pleroma.Repo.transaction(fn ->
+          Pleroma.Repo.stream(q, timeout: :infinity)
+          |> Enum.map(fn a ->
+            a.id
+          end)
+        end)
+
+      IO.puts("Got #{length(ids)} activities, adding to indexer")
+
+      ids
+      |> Enum.chunk_every(100)
+      |> Enum.each(fn chunk ->
+        IO.puts("Adding #{length(chunk)} activities to indexing queue")
+
+        chunk
+        |> Enum.map(fn id ->
+          SearchIndexingWorker.new(%{"op" => "add_to_index", "activity" => id})
+        end)
+        |> Oban.insert_all()
+      end)
+    end)
+  end
+end
-- 
cgit v1.2.3


From 1261c43a7af48ed6e6753461944659391c4c58cc Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Tue, 14 May 2024 17:19:36 +0400
Subject: SearchBackend: Add create_index

---
 lib/mix/tasks/pleroma/search/indexer.ex | 6 ++++++
 1 file changed, 6 insertions(+)

(limited to 'lib/mix')

diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
index ffa2f3c94..326646b69 100644
--- a/lib/mix/tasks/pleroma/search/indexer.ex
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -8,6 +8,12 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
 
   alias Pleroma.Workers.SearchIndexingWorker
 
+  def run(["create_index"]) do
+    Application.ensure_all_started(:pleroma)
+
+    Pleroma.Config.get([Pleroma.Search, :module]).create_index()
+  end
+
   def run(["index" | options]) do
     {options, [], []} =
       OptionParser.parse(
-- 
cgit v1.2.3


From a9be4907c0d7b34e5564584d2d040632c32f2aa3 Mon Sep 17 00:00:00 2001
From: Lain Soykaf <lain@lain.com>
Date: Thu, 16 May 2024 10:47:24 +0400
Subject: SearchBackend: Add drop_index

---
 lib/mix/tasks/pleroma/search/indexer.ex | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

(limited to 'lib/mix')

diff --git a/lib/mix/tasks/pleroma/search/indexer.ex b/lib/mix/tasks/pleroma/search/indexer.ex
index 326646b69..81a9fced6 100644
--- a/lib/mix/tasks/pleroma/search/indexer.ex
+++ b/lib/mix/tasks/pleroma/search/indexer.ex
@@ -9,9 +9,23 @@ defmodule Mix.Tasks.Pleroma.Search.Indexer do
   alias Pleroma.Workers.SearchIndexingWorker
 
   def run(["create_index"]) do
-    Application.ensure_all_started(:pleroma)
+    start_pleroma()
+
+    with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).create_index() do
+      IO.puts("Index created")
+    else
+      e -> IO.puts("Could not create index: #{inspect(e)}")
+    end
+  end
+
+  def run(["drop_index"]) do
+    start_pleroma()
 
-    Pleroma.Config.get([Pleroma.Search, :module]).create_index()
+    with :ok <- Pleroma.Config.get([Pleroma.Search, :module]).drop_index() do
+      IO.puts("Index dropped")
+    else
+      e -> IO.puts("Could not drop index: #{inspect(e)}")
+    end
   end
 
   def run(["index" | options]) do
-- 
cgit v1.2.3