From e154ebbf7933123e91d5b5c6f5070e78eb3e383b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 15 Aug 2021 21:53:04 +0300 Subject: Initial meilisearch implementation, doesn't delete posts yet --- lib/mix/tasks/pleroma/search/meilisearch.ex | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 lib/mix/tasks/pleroma/search/meilisearch.ex (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..2af8e5853 --- /dev/null +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2021 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Search.Meilisearch do + import Mix.Pleroma + + import Ecto.Query + + def run(["index"]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + Pleroma.Repo.chunk_stream( + from(Pleroma.Object, + limit: 200, + where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") + ), + 100, + :batches + ) + |> Stream.map(fn objects -> + Enum.map(objects, fn object -> + data = object.data + %{id: object.id, source: data["source"], ap: data["id"]} + end) + end) + |> Stream.each(fn activities -> + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!(activities) + ) + end) + |> Stream.run() + end +end -- cgit v1.2.3 From 0318e9a59945d7a5625111157867f0f9ebaffd91 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 10:18:01 +0300 Subject: Add logging to milisiearch index and make it use desc(id) --- lib/mix/tasks/pleroma/search/meilisearch.ex | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2af8e5853..1fece96e5 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -3,8 +3,9 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - import Mix.Pleroma + require Logger + import Mix.Pleroma import Ecto.Query def run(["index"]) do @@ -12,12 +13,25 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/settings/ranking-rules", + Jason.encode!([ + "desc(id)", + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ]) + ) + Pleroma.Repo.chunk_stream( from(Pleroma.Object, - limit: 200, where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") ), - 100, + 200, :batches ) |> Stream.map(fn objects -> @@ -26,12 +40,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do %{id: object.id, source: data["source"], ap: data["id"]} end) end) - |> Stream.each(fn activities -> + |> Stream.each(fn objects -> {:ok, _} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", - Jason.encode!(activities) + Jason.encode!(objects) ) + + IO.puts("Indexed #{Enum.count(objects)} entries") end) |> Stream.run() end -- cgit v1.2.3 From 365024abec905e427babb5403f0fccbde65f4bcd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 22:24:31 +0300 Subject: Ensure only indexing public posts and implement clearing and delete --- lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 1fece96e5..0b86fdece 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do require Logger + require Pleroma.Constants import Mix.Pleroma import Ecto.Query @@ -29,7 +30,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.Repo.chunk_stream( from(Pleroma.Object, - where: fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") + # Only index public posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + fragment("LENGTH(data->>'source') > 0") and + fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) ), 200, :batches @@ -51,4 +56,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end) |> Stream.run() end + + def run(["clear"]) do + start_pleroma() + + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) + + {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + end end -- cgit v1.2.3 From ea6a6a128712e81c4f298b2bb2cedfadf2295cff Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 16 Aug 2021 22:30:56 +0300 Subject: Make the indexing batch differently and more, show number indexed --- lib/mix/tasks/pleroma/search/meilisearch.ex | 63 +++++++++++++++++------------ 1 file changed, 38 insertions(+), 25 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 0b86fdece..2a6438528 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,33 +28,46 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) - Pleroma.Repo.chunk_stream( - from(Pleroma.Object, - # Only index public posts which are notes and have some text - where: - fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'source') > 0") and - fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) - ), - 200, - :batches - ) - |> Stream.map(fn objects -> - Enum.map(objects, fn object -> - data = object.data - %{id: object.id, source: data["source"], ap: data["id"]} - end) - end) - |> Stream.each(fn objects -> - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!(objects) + chunk_size = 100_000 + + Pleroma.Repo.transaction( + fn -> + Pleroma.Repo.stream( + from(Pleroma.Object, + # Only index public posts which are notes and have some text + where: + fragment("data->>'type' = 'Note'") and + fragment("LENGTH(data->>'source') > 0") and + fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), + order_by: fragment("data->'published' DESC") + ), + timeout: :infinity ) + |> Stream.chunk_every(chunk_size) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) - IO.puts("Indexed #{Enum.count(objects)} entries") - end) - |> Stream.run() + IO.puts("Indexed #{new_acc} entries") + + {[objects], new_acc} + end) + |> Stream.map(fn objects -> + Enum.map(objects, fn object -> + data = object.data + %{id: object.id, source: data["source"], ap: data["id"]} + end) + end) + |> Stream.each(fn objects -> + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/documents", + Jason.encode!(objects) + ) + end) + |> Stream.run() + end, + timeout: :infinity + ) end def run(["clear"]) do -- cgit v1.2.3 From 38996f551a4ec014e9f4cb4a691d31beecab43ba Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:06:32 +0300 Subject: Make meilisearch sort on publish date converted to unix time --- lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2a6438528..2dd9c0a62 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -18,7 +18,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.HTTP.post( "#{endpoint}/indexes/objects/settings/ranking-rules", Jason.encode!([ - "desc(id)", + "desc(published)", "typo", "words", "proximity", @@ -54,7 +54,15 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do |> Stream.map(fn objects -> Enum.map(objects, fn object -> data = object.data - %{id: object.id, source: data["source"], ap: data["id"]} + + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + source: data["source"], + ap: data["id"], + published: published |> DateTime.to_unix() + } end) end) |> Stream.each(fn objects -> -- cgit v1.2.3 From 9beaebd97e1746df010aecfcc01d9e2e9a4c60ac Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:30:14 +0300 Subject: Tweak search ordering to hopefully return newer results --- lib/mix/tasks/pleroma/search/meilisearch.ex | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2dd9c0a62..dcecbd7cf 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -39,7 +39,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fragment("data->>'type' = 'Note'") and fragment("LENGTH(data->>'source') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), - order_by: fragment("data->'published' DESC") + order_by: [desc: fragment("data->'published'")] ), timeout: :infinity ) @@ -66,11 +66,15 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end) end) |> Stream.each(fn objects -> - {:ok, _} = + {:ok, result} = Pleroma.HTTP.post( "#{endpoint}/indexes/objects/documents", Jason.encode!(objects) ) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + IO.puts("Failed to index: #{result}") + end end) |> Stream.run() end, @@ -83,6 +87,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + {:ok, result} = + Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) + + if not Map.has_key?(Jason.decode!(result.body), "updateId") do + IO.puts("Failed to clear: #{result}") + end end end -- cgit v1.2.3 From 00c48a33acf0bd59fa7e7b58a67b049e4f4adc31 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 00:57:53 +0300 Subject: Use content instead of source and scrub it --- lib/mix/tasks/pleroma/search/meilisearch.ex | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index dcecbd7cf..5270de255 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -37,7 +37,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do # Only index public posts which are notes and have some text where: fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'source') > 0") and + fragment("LENGTH(data->>'content') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), order_by: [desc: fragment("data->'published'")] ), @@ -56,10 +56,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do data = object.data {:ok, published, _} = DateTime.from_iso8601(data["published"]) + {:ok, content} = FastSanitize.strip_tags(data["content"]) %{ id: object.id, - source: data["source"], + content: content, ap: data["id"], published: published |> DateTime.to_unix() } @@ -87,11 +88,6 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, result} = - Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], []) - - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - IO.puts("Failed to clear: #{result}") - end + {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], []) end end -- cgit v1.2.3 From e35d87ea54f70a39206f6103ef0e7334e2a428cc Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 17 Aug 2021 01:37:43 +0300 Subject: Make the chunk size smaller --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 5270de255..44af25f3e 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,7 +28,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) - chunk_size = 100_000 + chunk_size = 10_000 Pleroma.Repo.transaction( fn -> -- cgit v1.2.3 From 9f16ca80e0fe60b8b0e3e8ddb9b06ca0bec31002 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 18:47:41 +0300 Subject: Mark only content as searchable for meilisearch --- lib/mix/tasks/pleroma/search/meilisearch.ex | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 44af25f3e..ebd3cc81f 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -28,6 +28,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ]) ) + {:ok, _} = + Pleroma.HTTP.post( + "#{endpoint}/indexes/objects/settings/searchable-attributes", + Jason.encode!([ + "content" + ]) + ) + chunk_size = 10_000 Pleroma.Repo.transaction( @@ -55,8 +63,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Enum.map(objects, fn object -> data = object.data + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + {:ok, published, _} = DateTime.from_iso8601(data["published"]) - {:ok, content} = FastSanitize.strip_tags(data["content"]) + {:ok, content} = FastSanitize.strip_tags(content_str) %{ id: object.id, -- cgit v1.2.3 From 3dedadf192a3acd0c1dfc2b11eba5a247ae7f61c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 19:38:03 +0300 Subject: Adjust content indexing to skip more unneeded stuff --- lib/mix/tasks/pleroma/search/meilisearch.ex | 47 +++++++++++++++++++---------- 1 file changed, 31 insertions(+), 16 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index ebd3cc81f..3704e0bdc 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -52,13 +52,6 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do timeout: :infinity ) |> Stream.chunk_every(chunk_size) - |> Stream.transform(0, fn objects, acc -> - new_acc = acc + Enum.count(objects) - - IO.puts("Indexed #{new_acc} entries") - - {[objects], new_acc} - end) |> Stream.map(fn objects -> Enum.map(objects, fn object -> data = object.data @@ -70,15 +63,34 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end {:ok, published, _} = DateTime.from_iso8601(data["published"]) - {:ok, content} = FastSanitize.strip_tags(content_str) - - %{ - id: object.id, - content: content, - ap: data["id"], - published: published |> DateTime.to_unix() - } + + content = + with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), + trimmed <- String.trim(scrubbed) do + trimmed + end + + # Only index if there is anything in the string. If there is a single symbol, + # it's probably a dot from mastodon posts with just the picture + if String.length(content) > 1 do + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + else + nil + end end) + |> Enum.filter(fn o -> not is_nil(o) end) + end) + |> Stream.transform(0, fn objects, acc -> + new_acc = acc + Enum.count(objects) + + IO.puts("Indexed #{new_acc} entries") + + {[objects], new_acc} end) |> Stream.each(fn objects -> {:ok, result} = @@ -102,6 +114,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects", "", [], []) + {:ok, _} = + Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], + timeout: :infinity + ) end end -- cgit v1.2.3 From 35e9192cedcbc56fb07c9933e2988bf900256b53 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 22:53:18 +0300 Subject: Rework task indexing to share code with the main module The code in the main module now scrubs new posts too --- lib/mix/tasks/pleroma/search/meilisearch.ex | 35 ++--------------------------- 1 file changed, 2 insertions(+), 33 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 3704e0bdc..b5a394e34 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -51,40 +51,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ), timeout: :infinity ) + |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) + |> Stream.filter(fn o -> not is_nil(o) end) |> Stream.chunk_every(chunk_size) - |> Stream.map(fn objects -> - Enum.map(objects, fn object -> - data = object.data - - content_str = - case data["content"] do - [nil | rest] -> to_string(rest) - str -> str - end - - {:ok, published, _} = DateTime.from_iso8601(data["published"]) - - content = - with {:ok, scrubbed} <- FastSanitize.strip_tags(content_str), - trimmed <- String.trim(scrubbed) do - trimmed - end - - # Only index if there is anything in the string. If there is a single symbol, - # it's probably a dot from mastodon posts with just the picture - if String.length(content) > 1 do - %{ - id: object.id, - content: content, - ap: data["id"], - published: published |> DateTime.to_unix() - } - else - nil - end - end) - |> Enum.filter(fn o -> not is_nil(o) end) - end) |> Stream.transform(0, fn objects, acc -> new_acc = acc + Enum.count(objects) -- cgit v1.2.3 From 410c8cb765bbec1014cb2bbdbcc44d3a25f834e1 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 22 Aug 2021 23:47:43 +0300 Subject: Make indexing logs rewrite themselves --- lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index b5a394e34..2485a441d 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -57,7 +57,9 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do |> Stream.transform(0, fn objects, acc -> new_acc = acc + Enum.count(objects) - IO.puts("Indexed #{new_acc} entries") + # Reset to the beginning of the line and rewrite it + IO.write("\r") + IO.write("Indexed #{new_acc} entries") {[objects], new_acc} end) @@ -76,6 +78,8 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do end, timeout: :infinity ) + + IO.write("\n") end def run(["clear"]) do -- cgit v1.2.3 From 2c7d973af7797ae860829c1764ade521a17e7263 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 19:35:21 +0300 Subject: Implement meilisearch auth --- lib/mix/tasks/pleroma/search/meilisearch.ex | 75 +++++++++++++++++------------ 1 file changed, 44 insertions(+), 31 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2485a441d..230be5aa1 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,32 +9,30 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Mix.Pleroma import Ecto.Query + import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1] + def run(["index"]) do start_pleroma() - endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/settings/ranking-rules", - Jason.encode!([ - "desc(published)", - "typo", - "words", - "proximity", - "attribute", - "wordsPosition", - "exactness" - ]) - ) + meili_post!( + "/indexes/objects/settings/ranking-rules", + [ + "desc(published)", + "typo", + "words", + "proximity", + "attribute", + "wordsPosition", + "exactness" + ] + ) - {:ok, _} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/settings/searchable-attributes", - Jason.encode!([ - "content" - ]) - ) + meili_post!( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) chunk_size = 10_000 @@ -64,14 +62,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> - {:ok, result} = - Pleroma.HTTP.post( - "#{endpoint}/indexes/objects/documents", - Jason.encode!(objects) + result = + meili_post!( + "/indexes/objects/documents", + objects ) - if not Map.has_key?(Jason.decode!(result.body), "updateId") do - IO.puts("Failed to index: #{result}") + if not Map.has_key?(result, "updateId") do + IO.puts("Failed to index: #{inspect(result)}") end end) |> Stream.run() @@ -85,11 +83,26 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["clear"]) do start_pleroma() + meili_delete!("/indexes/objects/documents") + end + + def run(["show-private-key", master_key]) do + start_pleroma() + endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) - {:ok, _} = - Pleroma.HTTP.request(:delete, "#{endpoint}/indexes/objects/documents", "", [], - timeout: :infinity + {:ok, result} = + Pleroma.HTTP.get( + Path.join(endpoint, "/keys"), + [{"X-Meili-API-Key", master_key}] ) + + decoded = Jason.decode!(result.body) + + if decoded["private"] do + IO.puts(decoded["private"]) + else + IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") + end end end -- cgit v1.2.3 From a67f9da5cc46b4e184aa1afe3dd1bd1df31de15b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:02:34 +0300 Subject: Add a message with a count of posts to index --- lib/mix/tasks/pleroma/search/meilisearch.ex | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 230be5aa1..557b06182 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -38,7 +38,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do Pleroma.Repo.transaction( fn -> - Pleroma.Repo.stream( + query = from(Pleroma.Object, # Only index public posts which are notes and have some text where: @@ -46,7 +46,13 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fragment("LENGTH(data->>'content') > 0") and fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), order_by: [desc: fragment("data->'published'")] - ), + ) + + count = query |> Pleroma.Repo.aggregate(:count, :data) + IO.puts("Entries to index: #{count}") + + Pleroma.Repo.stream( + query, timeout: :infinity ) |> Stream.map(&Pleroma.Search.Meilisearch.object_to_search_data/1) -- cgit v1.2.3 From 09a1ae1b6eca4efbb935aa1c0da950009d110fb2 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 23 Aug 2021 20:21:46 +0300 Subject: Add the meilisearch.stats command --- lib/mix/tasks/pleroma/search/meilisearch.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 557b06182..f2d9fe312 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Mix.Pleroma import Ecto.Query - import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1] + import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] def run(["index"]) do start_pleroma() @@ -111,4 +111,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") end end + + def run(["stats"]) do + start_pleroma() + + result = meili_get!("/indexes/objects/stats") + IO.puts("Number of entries: #{result["numberOfDocuments"]}") + IO.puts("Indexing? #{result["isIndexing"]}") + end end -- cgit v1.2.3 From 40280cc273ad7f2b355846e2f41b9873a8d5ff2c Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 28 Aug 2021 15:59:13 +0300 Subject: Reorder ranking rules for (maybe) better results --- lib/mix/tasks/pleroma/search/meilisearch.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index f2d9fe312..cdf9ab0bd 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -18,12 +18,12 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do "/indexes/objects/settings/ranking-rules", [ "desc(published)", - "typo", "words", + "exactness", "proximity", - "attribute", "wordsPosition", - "exactness" + "typo", + "attribute" ] ) -- cgit v1.2.3 From e928e307f34542b0a0af8b615c986aeac478b637 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Wed, 10 Nov 2021 21:25:12 +0300 Subject: Add a reindex option Signed-off-by: Ekaterina Vaartis --- lib/mix/tasks/pleroma/search/meilisearch.ex | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index cdf9ab0bd..2a3c3a8b9 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -11,9 +11,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] - def run(["index"]) do + def run(["index" | args]) do start_pleroma() + is_reindex = "--reindex" in args + meili_post!( "/indexes/objects/settings/ranking-rules", [ @@ -68,6 +70,19 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> + objects = + objects + |> Enum.filter(fn o -> + if is_reindex do + result = meili_get!("/indexes/objects/documents/#{o.id}") + + # Filter out the already indexed documents. This is true when the document does not exist + result["errorCode"] == "document_not_found" + else + true + end + end) + result = meili_post!( "/indexes/objects/documents", -- cgit v1.2.3 From 9c1a9307079c8d007ae7cbf3e089d2bc5ea6b733 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 13 Nov 2021 15:07:51 +0300 Subject: Support reindexing meilisearch >=0.24.0 It has has a different error code key --- lib/mix/tasks/pleroma/search/meilisearch.ex | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 2a3c3a8b9..3b134ad3f 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -76,8 +76,14 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do if is_reindex do result = meili_get!("/indexes/objects/documents/#{o.id}") + # With >= 0.24.0 the name for "errorCode" is just "code" + error_code_key = + if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"), + do: "code", + else: "errorCode" + # Filter out the already indexed documents. This is true when the document does not exist - result["errorCode"] == "document_not_found" + result[error_code_key] == "document_not_found" else true end -- cgit v1.2.3 From 39e596a5b51c0c86b6d6bd5f23177a1e6a64cf0b Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sun, 14 Nov 2021 21:42:18 +0300 Subject: Style fixes --- lib/mix/tasks/pleroma/search/meilisearch.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 3b134ad3f..62ace7e39 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -82,7 +82,8 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do do: "code", else: "errorCode" - # Filter out the already indexed documents. This is true when the document does not exist + # Filter out the already indexed documents. + # This is true when the document does not exist result[error_code_key] == "document_not_found" else true -- cgit v1.2.3 From a12f63bc81481e3f852934e8cc1269e16a57cf0a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 22 Nov 2021 21:39:54 +0300 Subject: Implement suggestions from the Meilisearch MR - Index unlisted posts - Move version check outside of the streaming and only do it once - Use a PUT request instead of checking manually if there is need to insert - Add error handling, sort of --- lib/mix/tasks/pleroma/search/meilisearch.ex | 86 ++++++++++++----------------- 1 file changed, 36 insertions(+), 50 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 62ace7e39..6730a99a9 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -3,38 +3,40 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Mix.Tasks.Pleroma.Search.Meilisearch do - require Logger require Pleroma.Constants import Mix.Pleroma import Ecto.Query - import Pleroma.Search.Meilisearch, only: [meili_post!: 2, meili_delete!: 1, meili_get!: 1] + import Pleroma.Search.Meilisearch, + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] - def run(["index" | args]) do + def run(["index"]) do start_pleroma() - is_reindex = "--reindex" in args - - meili_post!( - "/indexes/objects/settings/ranking-rules", - [ - "desc(published)", - "words", - "exactness", - "proximity", - "wordsPosition", - "typo", - "attribute" - ] - ) + {:ok, _} = + meili_post( + "/indexes/objects/settings/ranking-rules", + [ + "desc(published)", + "words", + "exactness", + "proximity", + "wordsPosition", + "typo", + "attribute" + ] + ) - meili_post!( - "/indexes/objects/settings/searchable-attributes", - [ - "content" - ] - ) + {:ok, _} = + meili_post( + "/indexes/objects/settings/searchable-attributes", + [ + "content" + ] + ) + + IO.puts("Created indices. Starting to insert posts.") chunk_size = 10_000 @@ -42,11 +44,11 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do fn -> query = from(Pleroma.Object, - # Only index public posts which are notes and have some text + # Only index public and unlisted posts which are notes and have some text where: fragment("data->>'type' = 'Note'") and - fragment("LENGTH(data->>'content') > 0") and - fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()), + (fragment("data->'to' \\? ?", ^Pleroma.Constants.as_public()) or + fragment("data->'cc' \\? ?", ^Pleroma.Constants.as_public())), order_by: [desc: fragment("data->'published'")] ) @@ -70,34 +72,18 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {[objects], new_acc} end) |> Stream.each(fn objects -> - objects = - objects - |> Enum.filter(fn o -> - if is_reindex do - result = meili_get!("/indexes/objects/documents/#{o.id}") - - # With >= 0.24.0 the name for "errorCode" is just "code" - error_code_key = - if meili_get!("/version")["pkgVersion"] |> Version.match?(">= 0.24.0"), - do: "code", - else: "errorCode" - - # Filter out the already indexed documents. - # This is true when the document does not exist - result[error_code_key] == "document_not_found" - else - true - end - end) - result = - meili_post!( + meili_put( "/indexes/objects/documents", objects ) - if not Map.has_key?(result, "updateId") do - IO.puts("Failed to index: #{inspect(result)}") + with {:ok, res} <- result do + if not Map.has_key?(res, "updateId") do + IO.puts("\nFailed to index: #{inspect(result)}") + end + else + e -> IO.puts("\nFailed to index due to network error: #{inspect(e)}") end end) |> Stream.run() @@ -137,7 +123,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["stats"]) do start_pleroma() - result = meili_get!("/indexes/objects/stats") + {:ok, result} = meili_get("/indexes/objects/stats") IO.puts("Number of entries: #{result["numberOfDocuments"]}") IO.puts("Indexing? #{result["isIndexing"]}") end -- cgit v1.2.3 From 3179ed0921197a8a8f32a519c7d41dc09011024d Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 18:48:52 +0300 Subject: Make chunk size configurable --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 6730a99a9..021552f7b 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -38,7 +38,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do IO.puts("Created indices. Starting to insert posts.") - chunk_size = 10_000 + chunk_size = Pleroma.Config.get([Pleroma.Search.Meilisearch, :initial_indexing_chunk_size]) Pleroma.Repo.transaction( fn -> -- cgit v1.2.3 From 571533ae2618478f26db312e52265e143356debd Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 20 Dec 2021 19:05:59 +0300 Subject: Don't support meilisearch < 0.24.0, since it breaks things --- lib/mix/tasks/pleroma/search/meilisearch.ex | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 021552f7b..5098668ad 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -14,17 +14,29 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["index"]) do start_pleroma() + meili_version = + ( + {:ok, result} = meili_get("/version") + + result["pkgVersion"] + ) + + # The ranking rule syntax was changed but nothing about that is mentioned in the changelog + if not Version.match?(meili_version, ">= 0.24.0") do + raise "Meilisearch <0.24.0 not supported" + end + {:ok, _} = meili_post( "/indexes/objects/settings/ranking-rules", [ - "desc(published)", + "published:desc", "words", "exactness", "proximity", - "wordsPosition", "typo", - "attribute" + "attribute", + "sort" ] ) -- cgit v1.2.3 From 1e23f527e3e22108b402552a0766e488048ed3f4 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 22 Mar 2022 20:29:17 +0300 Subject: Change the meilisearch key auth to conform to 0.25.0 --- lib/mix/tasks/pleroma/search/meilisearch.ex | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 5098668ad..db56876fa 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -22,7 +22,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ) # The ranking rule syntax was changed but nothing about that is mentioned in the changelog - if not Version.match?(meili_version, ">= 0.24.0") do + if not Version.match?(meili_version, ">= 0.25.0") do raise "Meilisearch <0.24.0 not supported" end @@ -112,7 +112,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do meili_delete!("/indexes/objects/documents") end - def run(["show-private-key", master_key]) do + def run(["show-keys", master_key]) do start_pleroma() endpoint = Pleroma.Config.get([Pleroma.Search.Meilisearch, :url]) @@ -120,15 +120,17 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do {:ok, result} = Pleroma.HTTP.get( Path.join(endpoint, "/keys"), - [{"X-Meili-API-Key", master_key}] + [{"Authorization", "Bearer #{master_key}"}] ) decoded = Jason.decode!(result.body) - if decoded["private"] do - IO.puts(decoded["private"]) + if decoded["results"] do + Enum.each(decoded["results"], fn %{"description" => desc, "key" => key} -> + IO.puts("#{desc}: #{key}") + end) else - IO.puts("Error fetching the key, check the master key is correct: #{inspect(decoded)}") + IO.puts("Error fetching the keys, check the master key is correct: #{inspect(decoded)}") end end -- cgit v1.2.3 From 84608be87e2c5961a4deb9030307c978bf1168e5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Tue, 22 Mar 2022 20:45:49 +0300 Subject: Change updateId to uid because apparently that's the new name --- lib/mix/tasks/pleroma/search/meilisearch.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index db56876fa..d4a83c3cd 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -91,7 +91,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do ) with {:ok, res} <- result do - if not Map.has_key?(res, "updateId") do + if not Map.has_key?(res, "uid") do IO.puts("\nFailed to index: #{inspect(result)}") end else -- cgit v1.2.3 From 5ac67632384bfb284ac51f2a450d41cf3913378a Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 00:31:36 +0300 Subject: Make add_to_index and remove_from_index report errors --- lib/mix/tasks/pleroma/search/meilisearch.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index d4a83c3cd..72a558228 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -9,7 +9,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do import Ecto.Query import Pleroma.Search.Meilisearch, - only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete!: 1] + only: [meili_post: 2, meili_put: 2, meili_get: 1, meili_delete: 1] def run(["index"]) do start_pleroma() @@ -109,7 +109,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["clear"]) do start_pleroma() - meili_delete!("/indexes/objects/documents") + meili_delete("/indexes/objects/documents") end def run(["show-keys", master_key]) do -- cgit v1.2.3 From 5a39866388c411f2bcee9848352f8c420513f34f Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Sat, 27 Aug 2022 01:43:59 +0300 Subject: Specifically strip mentions for search indexing --- lib/mix/tasks/pleroma/search/meilisearch.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/search/meilisearch.ex b/lib/mix/tasks/pleroma/search/meilisearch.ex index 72a558228..8379a0c25 100644 --- a/lib/mix/tasks/pleroma/search/meilisearch.ex +++ b/lib/mix/tasks/pleroma/search/meilisearch.ex @@ -13,6 +13,7 @@ defmodule Mix.Tasks.Pleroma.Search.Meilisearch do def run(["index"]) do start_pleroma() + Pleroma.HTML.compile_scrubbers() meili_version = ( -- cgit v1.2.3 From e23672d82f0a6e61bf4ada65cfca6cfbc647710d Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 8 Nov 2023 12:21:34 -0500 Subject: Ensure benchee doesn't run unless we are executing benchmarks --- lib/mix/tasks/pleroma/benchmark.ex | 113 ------------------------------------- 1 file changed, 113 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/benchmark.ex (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/benchmark.ex b/lib/mix/tasks/pleroma/benchmark.ex deleted file mode 100644 index f32492169..000000000 --- a/lib/mix/tasks/pleroma/benchmark.ex +++ /dev/null @@ -1,113 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Benchmark do - import Mix.Pleroma - use Mix.Task - - def run(["search"]) do - start_pleroma() - - Benchee.run(%{ - "search" => fn -> - Pleroma.Activity.search(nil, "cofe") - end - }) - end - - def run(["tag"]) do - start_pleroma() - - Benchee.run(%{ - "tag" => fn -> - %{"type" => "Create", "tag" => "cofe"} - |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() - end - }) - end - - def run(["render_timeline", nickname | _] = args) do - start_pleroma() - user = Pleroma.User.get_by_nickname(nickname) - - activities = - %{} - |> Map.put("type", ["Create", "Announce"]) - |> Map.put("blocking_user", user) - |> Map.put("muting_user", user) - |> Map.put("user", user) - |> Map.put("limit", 4096) - |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() - |> Enum.reverse() - - inputs = %{ - "1 activity" => Enum.take_random(activities, 1), - "10 activities" => Enum.take_random(activities, 10), - "20 activities" => Enum.take_random(activities, 20), - "40 activities" => Enum.take_random(activities, 40), - "80 activities" => Enum.take_random(activities, 80) - } - - inputs = - if Enum.at(args, 2) == "extended" do - Map.merge(inputs, %{ - "200 activities" => Enum.take_random(activities, 200), - "500 activities" => Enum.take_random(activities, 500), - "2000 activities" => Enum.take_random(activities, 2000), - "4096 activities" => Enum.take_random(activities, 4096) - }) - else - inputs - end - - Benchee.run( - %{ - "Standart rendering" => fn activities -> - Pleroma.Web.MastodonAPI.StatusView.render("index.json", %{ - activities: activities, - for: user, - as: :activity - }) - end - }, - inputs: inputs - ) - end - - def run(["adapters"]) do - start_pleroma() - - :ok = - Pleroma.Gun.Conn.open( - "https://httpbin.org/stream-bytes/1500", - :gun_connections - ) - - Process.sleep(1_500) - - Benchee.run( - %{ - "Without conn and without pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], - pool: :no_pool, - receive_conn: false - ) - end, - "Without conn and with pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], receive_conn: false) - end, - "With reused conn and without pool" => fn -> - {:ok, %Tesla.Env{}} = - Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500", [], pool: :no_pool) - end, - "With reused conn and with pool" => fn -> - {:ok, %Tesla.Env{}} = Pleroma.HTTP.get("https://httpbin.org/stream-bytes/1500") - end - }, - parallel: 10 - ) - end -end -- cgit v1.2.3 From 017e35fbf128d47c033275a70b76b72f24d7c754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Thu, 28 Dec 2023 00:15:32 +0100 Subject: Fix some more typos MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- lib/mix/tasks/pleroma/database.ex | 2 +- lib/mix/tasks/pleroma/digest.ex | 2 +- lib/mix/tasks/pleroma/ecto/rollback.ex | 2 +- lib/mix/tasks/pleroma/instance.ex | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ed560c177..93ee57dc3 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -193,7 +193,7 @@ defmodule Mix.Tasks.Pleroma.Database do "ALTER DATABASE #{db} SET default_text_search_config = '#{tsconfig}';" ) - # non-exist config will not raise excpetion but only give >0 messages + # non-exist config will not raise exception but only give >0 messages if length(msg) > 0 do shell_info("Error: #{inspect(msg, pretty: true)}") else diff --git a/lib/mix/tasks/pleroma/digest.ex b/lib/mix/tasks/pleroma/digest.ex index aea9c8ac5..53cac0b94 100644 --- a/lib/mix/tasks/pleroma/digest.ex +++ b/lib/mix/tasks/pleroma/digest.ex @@ -30,7 +30,7 @@ defmodule Mix.Tasks.Pleroma.Digest do shell_info("Digest email have been sent to #{nickname} (#{user.email})") else _ -> - shell_info("Cound't find any mentions for #{nickname} since #{last_digest_emailed_at}") + shell_info("Couldn't find any mentions for #{nickname} since #{last_digest_emailed_at}") end end end diff --git a/lib/mix/tasks/pleroma/ecto/rollback.ex b/lib/mix/tasks/pleroma/ecto/rollback.ex index 3d78eaec4..121890f39 100644 --- a/lib/mix/tasks/pleroma/ecto/rollback.ex +++ b/lib/mix/tasks/pleroma/ecto/rollback.ex @@ -61,7 +61,7 @@ defmodule Mix.Tasks.Pleroma.Ecto.Rollback do Logger.configure(level: :info) if opts[:env] == "test" do - Logger.info("Rollback succesfully") + Logger.info("Rollback successfully") else {:ok, _, _} = Ecto.Migrator.with_repo(Pleroma.Repo, &Ecto.Migrator.run(&1, path, :down, opts)) diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index 5d8b254a2..fd1809a42 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -292,7 +292,7 @@ defmodule Mix.Tasks.Pleroma.Instance do if db_configurable? do shell_info( - " Please transfer your config to the database after running database migrations. Refer to \"Transfering the config to/from the database\" section of the docs for more information." + " Please transfer your config to the database after running database migrations. Refer to \"Transferring the config to/from the database\" section of the docs for more information." ) end else -- cgit v1.2.3 From 2a28377be06d286a13c69d6735734d5cfd503309 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Sat, 20 Jan 2024 15:55:33 -0500 Subject: Fix mix task pleroma.instance dialyzer error lib/mix/tasks/pleroma/instance.ex:356:pattern_match_cov The pattern :variable_ can never match, because previous clauses completely cover the type %{ :anonymize => boolean(), :dedupe => boolean(), :read_description => boolean(), :strip_location => boolean() }. --- lib/mix/tasks/pleroma/instance.ex | 2 -- 1 file changed, 2 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/instance.ex b/lib/mix/tasks/pleroma/instance.ex index fd1809a42..0dc30549c 100644 --- a/lib/mix/tasks/pleroma/instance.ex +++ b/lib/mix/tasks/pleroma/instance.ex @@ -352,6 +352,4 @@ defmodule Mix.Tasks.Pleroma.Instance do enabled_filters end - - defp upload_filters(_), do: [] end -- cgit v1.2.3