diff options
Diffstat (limited to 'lib/pleroma')
20 files changed, 321 insertions, 32 deletions
diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 3556aaf9e..8a512dc57 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -368,7 +368,7 @@ defmodule Pleroma.Activity do ) end - defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search + defdelegate search(user, query, options \\ []), to: Pleroma.Search.DatabaseSearch def direct_conversation_id(activity, for_user) do alias Pleroma.Conversation.Participation diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index e68a3c57e..7bbc132f1 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -322,7 +322,11 @@ defmodule Pleroma.Application do def limiters_setup do config = Config.get(ConcurrentLimiter, []) - [Pleroma.Web.RichMedia.Helpers, Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy] + [ + Pleroma.Web.RichMedia.Helpers, + Pleroma.Web.ActivityPub.MRF.MediaProxyWarmingPolicy, + Pleroma.Search + ] |> Enum.each(fn module -> mod_config = Keyword.get(config, module, []) diff --git a/lib/pleroma/config/getting.ex b/lib/pleroma/config/getting.ex index f9b66bba6..ec93fd02a 100644 --- a/lib/pleroma/config/getting.ex +++ b/lib/pleroma/config/getting.ex @@ -5,4 +5,11 @@ defmodule Pleroma.Config.Getting do @callback get(any()) :: any() @callback get(any(), any()) :: any() + + def get(key), do: get(key, nil) + def get(key, default), do: impl().get(key, default) + + def impl do + Application.get_env(:pleroma, :config_impl, Pleroma.Config) + end end diff --git a/lib/pleroma/search.ex b/lib/pleroma/search.ex new file mode 100644 index 000000000..3b266e59b --- /dev/null +++ b/lib/pleroma/search.ex @@ -0,0 +1,17 @@ +defmodule Pleroma.Search do + alias Pleroma.Workers.SearchIndexingWorker + + def add_to_index(%Pleroma.Activity{id: activity_id}) do + SearchIndexingWorker.enqueue("add_to_index", %{"activity" => activity_id}) + end + + def remove_from_index(%Pleroma.Object{id: object_id}) do + SearchIndexingWorker.enqueue("remove_from_index", %{"object" => object_id}) + end + + def search(query, options) do + search_module = Pleroma.Config.get([Pleroma.Search, :module], Pleroma.Activity) + + search_module.search(options[:for_user], query, options) + end +end diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/search/database_search.ex index 0b9b24aa4..c6311e0c7 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/search/database_search.ex @@ -1,9 +1,10 @@ # Pleroma: A lightweight social networking server -# Copyright © 2017-2022 Pleroma Authors <https://pleroma.social/> +# Copyright © 2017-2021 Pleroma Authors <https://pleroma.social/> # SPDX-License-Identifier: AGPL-3.0-only -defmodule Pleroma.Activity.Search do +defmodule Pleroma.Search.DatabaseSearch do alias Pleroma.Activity + alias Pleroma.Config alias Pleroma.Object.Fetcher alias Pleroma.Pagination alias Pleroma.User @@ -13,8 +14,11 @@ defmodule Pleroma.Activity.Search do import Ecto.Query + @behaviour Pleroma.Search.SearchBackend + + @impl true def search(user, search_query, options \\ []) do - index_type = if Pleroma.Config.get([:database, :rum_enabled]), do: :rum, else: :gin + index_type = if Config.get([:database, :rum_enabled]), do: :rum, else: :gin limit = Enum.min([Keyword.get(options, :limit), 40]) offset = Keyword.get(options, :offset, 0) author = Keyword.get(options, :author) @@ -45,6 +49,12 @@ defmodule Pleroma.Activity.Search do end end + @impl true + def add_to_index(_activity), do: :ok + + @impl true + def remove_from_index(_object), do: :ok + def maybe_restrict_author(query, %User{} = author) do Activity.Queries.by_author(query, author) end @@ -136,8 +146,8 @@ defmodule Pleroma.Activity.Search do ) end - defp maybe_restrict_local(q, user) do - limit = Pleroma.Config.get([:instance, :limit_to_local_content], :unauthenticated) + def maybe_restrict_local(q, user) do + limit = Config.get([:instance, :limit_to_local_content], :unauthenticated) case {limit, user} do {:all, _} -> restrict_local(q) @@ -149,7 +159,7 @@ defmodule Pleroma.Activity.Search do defp restrict_local(q), do: where(q, local: true) - defp maybe_fetch(activities, user, search_query) do + def maybe_fetch(activities, user, search_query) do with true <- Regex.match?(~r/https?:/, search_query), {:ok, object} <- Fetcher.fetch_object_from_id(search_query), %Activity{} = activity <- Activity.get_create_by_object_ap_id(object.data["id"]), diff --git a/lib/pleroma/search/meilisearch.ex b/lib/pleroma/search/meilisearch.ex new file mode 100644 index 000000000..2bff663e8 --- /dev/null +++ b/lib/pleroma/search/meilisearch.ex @@ -0,0 +1,181 @@ +defmodule Pleroma.Search.Meilisearch do + require Logger + require Pleroma.Constants + + alias Pleroma.Activity + alias Pleroma.Config.Getting, as: Config + + import Pleroma.Search.DatabaseSearch + import Ecto.Query + + @behaviour Pleroma.Search.SearchBackend + + defp meili_headers do + private_key = Config.get([Pleroma.Search.Meilisearch, :private_key]) + + [{"Content-Type", "application/json"}] ++ + if is_nil(private_key), do: [], else: [{"Authorization", "Bearer #{private_key}"}] + end + + def meili_get(path) do + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.get( + Path.join(endpoint, path), + meili_headers() + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_post(path, params) do + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.post( + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers() + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_put(path, params) do + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + result = + Pleroma.HTTP.request( + :put, + Path.join(endpoint, path), + Jason.encode!(params), + meili_headers(), + [] + ) + + with {:ok, res} <- result do + {:ok, Jason.decode!(res.body)} + end + end + + def meili_delete(path) do + endpoint = Config.get([Pleroma.Search.Meilisearch, :url]) + + with {:ok, _} <- + Pleroma.HTTP.request( + :delete, + Path.join(endpoint, path), + "", + meili_headers(), + [] + ) do + :ok + else + _ -> {:error, "Could not remove from index"} + end + end + + @impl true + def search(user, query, options \\ []) do + limit = Enum.min([Keyword.get(options, :limit), 40]) + offset = Keyword.get(options, :offset, 0) + author = Keyword.get(options, :author) + + res = + meili_post( + "/indexes/objects/search", + %{q: query, offset: offset, limit: limit} + ) + + with {:ok, result} <- res do + hits = result["hits"] |> Enum.map(& &1["ap"]) + + try do + hits + |> Activity.create_by_object_ap_id() + |> Activity.with_preloaded_object() + |> Activity.restrict_deactivated_users() + |> maybe_restrict_local(user) + |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) + |> maybe_fetch(user, query) + |> order_by([object: obj], desc: obj.data["published"]) + |> Pleroma.Repo.all() + rescue + _ -> maybe_fetch([], user, query) + end + end + end + + def object_to_search_data(object) do + # Only index public or unlisted Notes + if not is_nil(object) and object.data["type"] == "Note" and + not is_nil(object.data["content"]) and + (Pleroma.Constants.as_public() in object.data["to"] or + Pleroma.Constants.as_public() in object.data["cc"]) and + object.data["content"] not in ["", "."] do + data = object.data + + content_str = + case data["content"] do + [nil | rest] -> to_string(rest) + str -> str + end + + content = + with {:ok, scrubbed} <- + FastSanitize.Sanitizer.scrub(content_str, Pleroma.HTML.Scrubber.SearchIndexing), + trimmed <- String.trim(scrubbed) do + trimmed + end + + # Make sure we have a non-empty string + if content != "" do + {:ok, published, _} = DateTime.from_iso8601(data["published"]) + + %{ + id: object.id, + content: content, + ap: data["id"], + published: published |> DateTime.to_unix() + } + end + end + end + + @impl true + def add_to_index(activity) do + maybe_search_data = object_to_search_data(activity.object) + + if activity.data["type"] == "Create" and maybe_search_data do + result = + meili_put( + "/indexes/objects/documents", + [maybe_search_data] + ) + + with {:ok, %{"status" => "enqueued"}} <- result do + # Added successfully + :ok + else + _ -> + # There was an error, report it + Logger.error("Failed to add activity #{activity.id} to index: #{inspect(result)}") + {:error, result} + end + else + # The post isn't something we can search, that's ok + :ok + end + end + + @impl true + def remove_from_index(object) do + meili_delete("/indexes/objects/documents/#{object.id}") + end +end diff --git a/lib/pleroma/search/search_backend.ex b/lib/pleroma/search/search_backend.ex new file mode 100644 index 000000000..a42e2f5f6 --- /dev/null +++ b/lib/pleroma/search/search_backend.ex @@ -0,0 +1,24 @@ +defmodule Pleroma.Search.SearchBackend do + @doc """ + Search statuses with a query, restricting to only those the user should have access to. + """ + @callback search(user :: Pleroma.User.t(), query :: String.t(), options :: [any()]) :: [ + Pleroma.Activity.t() + ] + + @doc """ + Add the object associated with the activity to the search index. + + The whole activity is passed, to allow filtering on things such as scope. + """ + @callback add_to_index(activity :: Pleroma.Activity.t()) :: :ok | {:error, any()} + + @doc """ + Remove the object from the index. + + Just the object, as opposed to the whole activity, is passed, since the object + is what contains the actual content and there is no need for fitlering when removing + from index. + """ + @callback remove_from_index(object :: Pleroma.Object.t()) :: {:ok, any()} | {:error, any()} +end diff --git a/lib/pleroma/web.ex b/lib/pleroma/web.ex index aee41b0fe..7a8b176cd 100644 --- a/lib/pleroma/web.ex +++ b/lib/pleroma/web.ex @@ -136,7 +136,7 @@ defmodule Pleroma.Web do namespace: Pleroma.Web # Import convenience functions from controllers - import Phoenix.Controller, only: [get_csrf_token: 0, get_flash: 2, view_module: 1] + import Phoenix.Controller, only: [get_csrf_token: 0, view_module: 1] import Pleroma.Web.ErrorHelpers import Pleroma.Web.Gettext diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index a81d33fb6..32d1a1037 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -151,6 +151,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + # Add local posts to search index + if local, do: Pleroma.Search.add_to_index(activity) + {:ok, activity} else %Activity{} = activity -> diff --git a/lib/pleroma/web/activity_pub/side_effects.ex b/lib/pleroma/web/activity_pub/side_effects.ex index 6a7ac2806..10f268f05 100644 --- a/lib/pleroma/web/activity_pub/side_effects.ex +++ b/lib/pleroma/web/activity_pub/side_effects.ex @@ -197,6 +197,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Increase replies count # - Set up ActivityExpiration # - Set up notifications + # - Index incoming posts for search (if needed) @impl true def handle(%{data: %{"type" => "Create"}} = activity, meta) do with {:ok, object, meta} <- handle_object_creation(meta[:object_data], activity, meta), @@ -230,6 +231,8 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) end) + Pleroma.Search.add_to_index(Map.put(activity, :object, object)) + meta = meta |> add_notifications(notifications) @@ -289,6 +292,7 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do # - Reduce the user note count # - Reduce the reply count # - Stream out the activity + # - Removes posts from search index (if needed) @impl true def handle(%{data: %{"type" => "Delete", "object" => deleted_object}} = object, meta) do deleted_object = @@ -331,6 +335,11 @@ defmodule Pleroma.Web.ActivityPub.SideEffects do end if result == :ok do + # Only remove from index when deleting actual objects, not users or anything else + with %Pleroma.Object{} <- deleted_object do + Pleroma.Search.remove_from_index(deleted_object) + end + {:ok, object, meta} else {:error, result} diff --git a/lib/pleroma/web/endpoint.ex b/lib/pleroma/web/endpoint.ex index 574f3ab63..65dd72c49 100644 --- a/lib/pleroma/web/endpoint.ex +++ b/lib/pleroma/web/endpoint.ex @@ -9,7 +9,20 @@ defmodule Pleroma.Web.Endpoint do alias Pleroma.Config - socket("/socket", Pleroma.Web.UserSocket) + socket("/socket", Pleroma.Web.UserSocket, + websocket: [ + path: "/websocket", + serializer: [ + {Phoenix.Socket.V1.JSONSerializer, "~> 1.0.0"}, + {Phoenix.Socket.V2.JSONSerializer, "~> 2.0.0"} + ], + timeout: 60_000, + transport_log: false, + compress: false + ], + longpoll: false + ) + socket("/live", Phoenix.LiveView.Socket) plug(Plug.Telemetry, event_prefix: [:phoenix, :endpoint]) diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 5e6e04734..e4acba226 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do use Pleroma.Web, :controller - alias Pleroma.Activity alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.ControllerHelper @@ -100,7 +99,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do end defp resource_search(_, "statuses", query, options) do - statuses = with_fallback(fn -> Activity.search(options[:for_user], query, options) end) + statuses = with_fallback(fn -> Pleroma.Search.search(query, options) end) StatusView.render("index.json", activities: statuses, diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index d40af499e..9abad65b0 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -1005,9 +1005,8 @@ defmodule Pleroma.Web.Router do options("/*path", RedirectController, :empty) end - # TODO: Change to Phoenix.Router.routes/1 for Phoenix 1.6.0+ def get_api_routes do - __MODULE__.__routes__() + Phoenix.Router.routes(__MODULE__) |> Enum.reject(fn r -> r.plug == Pleroma.Web.Fallback.RedirectController end) |> Enum.map(fn r -> r.path diff --git a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex index e45d13bdf..e3639aae7 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/recovery.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> -<p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p> +<%= if Phoenix.Flash.get(@flash, :info) do %> +<p class="alert alert-info" role="alert"><%= Phoenix.Flash.get(@flash, :info) %></p> <% end %> -<%= if get_flash(@conn, :error) do %> -<p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p> +<%= if Phoenix.Flash.get(@flash, :error) do %> +<p class="alert alert-danger" role="alert"><%= Phoenix.Flash.get(@flash, :error) %></p> <% end %> <h2><%= Gettext.dpgettext("static_pages", "mfa recover page title", "Two-factor recovery") %></h2> diff --git a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex index 50e6c04b6..f995b8805 100644 --- a/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex +++ b/lib/pleroma/web/templates/o_auth/mfa/totp.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> -<p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p> +<%= if Phoenix.Flash.get(@flash, :info) do %> +<p class="alert alert-info" role="alert"><%= Phoenix.Flash.get(@flash, :info) %></p> <% end %> -<%= if get_flash(@conn, :error) do %> -<p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p> +<%= if Phoenix.Flash.get(@flash, :error) do %> +<p class="alert alert-danger" role="alert"><%= Phoenix.Flash.get(@flash, :error) %></p> <% end %> <h2><%= Gettext.dpgettext("static_pages", "mfa auth page title", "Two-factor authentication") %></h2> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex index 1f661efb2..e7f65266f 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/register.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> - <p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p> +<%= if Phoenix.Flash.get(@flash, :info) do %> + <p class="alert alert-info" role="alert"><%= Phoenix.Flash.get(@flash, :info) %></p> <% end %> -<%= if get_flash(@conn, :error) do %> - <p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p> +<%= if Phoenix.Flash.get(@flash, :error) do %> + <p class="alert alert-danger" role="alert"><%= Phoenix.Flash.get(@flash, :error) %></p> <% end %> <h2><%= Gettext.dpgettext("static_pages", "oauth register page title", "Registration Details") %></h2> diff --git a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex index b3654f3eb..5b38f7142 100644 --- a/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex +++ b/lib/pleroma/web/templates/o_auth/o_auth/show.html.eex @@ -1,8 +1,8 @@ -<%= if get_flash(@conn, :info) do %> -<p class="alert alert-info" role="alert"><%= get_flash(@conn, :info) %></p> +<%= if Phoenix.Flash.get(@flash, :info) do %> +<p class="alert alert-info" role="alert"><%= Phoenix.Flash.get(@flash, :info) %></p> <% end %> -<%= if get_flash(@conn, :error) do %> -<p class="alert alert-danger" role="alert"><%= get_flash(@conn, :error) %></p> +<%= if Phoenix.Flash.get(@flash, :error) do %> +<p class="alert alert-danger" role="alert"><%= Phoenix.Flash.get(@flash, :error) %></p> <% end %> <%= form_for @conn, Routes.o_auth_path(@conn, :authorize), [as: "authorization"], fn f -> %> diff --git a/lib/pleroma/workers/cron/digest_emails_worker.ex b/lib/pleroma/workers/cron/digest_emails_worker.ex index 1540c1605..0292bbb3b 100644 --- a/lib/pleroma/workers/cron/digest_emails_worker.ex +++ b/lib/pleroma/workers/cron/digest_emails_worker.ex @@ -7,7 +7,7 @@ defmodule Pleroma.Workers.Cron.DigestEmailsWorker do The worker to send digest emails. """ - use Oban.Worker, queue: "digest_emails" + use Oban.Worker, queue: "mailer" alias Pleroma.Config alias Pleroma.Emails diff --git a/lib/pleroma/workers/cron/new_users_digest_worker.ex b/lib/pleroma/workers/cron/new_users_digest_worker.ex index 267fe2837..1c3e445aa 100644 --- a/lib/pleroma/workers/cron/new_users_digest_worker.ex +++ b/lib/pleroma/workers/cron/new_users_digest_worker.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Workers.Cron.NewUsersDigestWorker do import Ecto.Query - use Pleroma.Workers.WorkerHelper, queue: "new_users_digest" + use Pleroma.Workers.WorkerHelper, queue: "mailer" @impl Oban.Worker def perform(_job) do diff --git a/lib/pleroma/workers/search_indexing_worker.ex b/lib/pleroma/workers/search_indexing_worker.ex new file mode 100644 index 000000000..8476a2be5 --- /dev/null +++ b/lib/pleroma/workers/search_indexing_worker.ex @@ -0,0 +1,23 @@ +defmodule Pleroma.Workers.SearchIndexingWorker do + use Pleroma.Workers.WorkerHelper, queue: "search_indexing" + + @impl Oban.Worker + + alias Pleroma.Config.Getting, as: Config + + def perform(%Job{args: %{"op" => "add_to_index", "activity" => activity_id}}) do + activity = Pleroma.Activity.get_by_id_with_object(activity_id) + + search_module = Config.get([Pleroma.Search, :module]) + + search_module.add_to_index(activity) + end + + def perform(%Job{args: %{"op" => "remove_from_index", "object" => object_id}}) do + object = Pleroma.Object.get_by_id(object_id) + + search_module = Config.get([Pleroma.Search, :module]) + + search_module.remove_from_index(object) + end +end |