From 0709757e47a546225163482474a46e8bc3d09837 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Tue, 14 Jan 2020 19:53:28 +0300 Subject: Check if object->data is array when looking up attachment objects to delete --- lib/pleroma/object.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 2452a7389..6c56a9c62 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -203,7 +203,8 @@ defmodule Pleroma.Object do from(o in Object, where: fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href'))::jsonb \\?| (?)", + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, o.data, ^hrefs ) -- cgit v1.2.3 From 81133702d4246e5e5c17ea4d4119747dc880edd6 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Fri, 17 Jan 2020 20:20:37 +0300 Subject: Delete attachments after object and cache cleanup --- lib/pleroma/object.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 6c56a9c62..499339982 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -180,10 +180,10 @@ defmodule Pleroma.Object do def delete(%Object{data: %{"id" => id}} = object) do with {:ok, _obj} = swap_object_with_tombstone(object), - :ok <- delete_attachments(object), deleted_activity = Activity.delete_all_by_object_ap_id(id), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), - {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path) do + {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path), + :ok <- delete_attachments(object) do {:ok, object, deleted_activity} end end -- cgit v1.2.3 From d6a532bf0f280cc191a9f2c1f53af31c451481d9 Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 19 Jan 2020 19:45:20 +0300 Subject: Delete attachments asynchronously --- lib/pleroma/object.ex | 79 ++------------------ lib/pleroma/workers/attachments_cleanup_worker.ex | 87 +++++++++++++++++++++++ 2 files changed, 93 insertions(+), 73 deletions(-) create mode 100644 lib/pleroma/workers/attachments_cleanup_worker.ex (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 499339982..38e372f6d 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -19,6 +19,8 @@ defmodule Pleroma.Object do @type t() :: %__MODULE__{} + @derive {Jason.Encoder, only: [:data]} + schema "objects" do field(:data, :map) @@ -183,83 +185,14 @@ defmodule Pleroma.Object do deleted_activity = Activity.delete_all_by_object_ap_id(id), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), {:ok, _} <- Cachex.del(:web_resp_cache, URI.parse(id).path), - :ok <- delete_attachments(object) do + {:ok, _} <- + Pleroma.Workers.AttachmentsCleanupWorker.enqueue("cleanup_attachments", %{ + "object" => object + }) do {:ok, object, deleted_activity} end end - defp delete_attachments(%{data: %{"attachment" => [_ | _] = attachments, "actor" => actor}}) do - hrefs = - Enum.flat_map(attachments, fn attachment -> - Enum.map(attachment["url"], & &1["href"]) - end) - - names = Enum.map(attachments, & &1["name"]) - - uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) - - # find all objects for copies of the attachments, name and actor doesn't matter here - delete_ids = - from(o in Object, - where: - fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", - o.data, - o.data, - ^hrefs - ) - ) - |> Repo.all() - # we should delete 1 object for any given attachment, but don't delete files if - # there are more than 1 object for it - |> Enum.reduce(%{}, fn %{ - id: id, - data: %{ - "url" => [%{"href" => href}], - "actor" => obj_actor, - "name" => name - } - }, - acc -> - Map.update(acc, href, %{id: id, count: 1}, fn val -> - case obj_actor == actor and name in names do - true -> - # set id of the actor's object that will be deleted - %{val | id: id, count: val.count + 1} - - false -> - # another actor's object, just increase count to not delete file - %{val | count: val.count + 1} - end - end) - end) - |> Enum.map(fn {href, %{id: id, count: count}} -> - # only delete files that have single instance - with 1 <- count do - prefix = - case Pleroma.Config.get([Pleroma.Upload, :base_url]) do - nil -> "media" - _ -> "" - end - - base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url()) - - file_path = String.trim_leading(href, "#{base_url}/#{prefix}") - - uploader.delete_file(file_path) - end - - id - end) - - from(o in Object, where: o.id in ^delete_ids) - |> Repo.delete_all() - - :ok - end - - defp delete_attachments(%{data: _data}), do: :ok - def prune(%Object{data: %{"id" => id}} = object) do with {:ok, object} <- Repo.delete(object), {:ok, true} <- Cachex.del(:object_cache, "object:#{id}"), diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex new file mode 100644 index 000000000..f8239ece6 --- /dev/null +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -0,0 +1,87 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Workers.AttachmentsCleanupWorker do + import Ecto.Query + + alias Pleroma.Object + alias Pleroma.Repo + + use Pleroma.Workers.WorkerHelper, queue: "attachments_cleanup" + + @impl Oban.Worker + def perform( + %{"object" => %{"data" => %{"attachment" => [_ | _] = attachments, "actor" => actor}}}, + _job + ) do + hrefs = + Enum.flat_map(attachments, fn attachment -> + Enum.map(attachment["url"], & &1["href"]) + end) + + names = Enum.map(attachments, & &1["name"]) + + uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) + + # find all objects for copies of the attachments, name and actor doesn't matter here + delete_ids = + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, + o.data, + ^hrefs + ) + ) + # The query above can be time consumptive on large instances until we refactor how uploads are stored + |> Repo.all(timout: :infinity) + # we should delete 1 object for any given attachment, but don't delete files if + # there are more than 1 object for it + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) + |> Enum.map(fn {href, %{id: id, count: count}} -> + # only delete files that have single instance + with 1 <- count do + prefix = + case Pleroma.Config.get([Pleroma.Upload, :base_url]) do + nil -> "media" + _ -> "" + end + + base_url = Pleroma.Config.get([__MODULE__, :base_url], Pleroma.Web.base_url()) + + file_path = String.trim_leading(href, "#{base_url}/#{prefix}") + + uploader.delete_file(file_path) + end + + id + end) + + from(o in Object, where: o.id in ^delete_ids) + |> Repo.delete_all() + end + + def perform(%{"object" => _object}, _job), do: :ok +end -- cgit v1.2.3 From cde828ff7df64740f36b2fc9dfdbfc8d76a8a78d Mon Sep 17 00:00:00 2001 From: Roman Chvanikov Date: Sun, 19 Jan 2020 22:09:47 +0300 Subject: Fix credo warning --- lib/pleroma/workers/attachments_cleanup_worker.ex | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index f8239ece6..3f421db40 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -35,10 +35,11 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do ^hrefs ) ) - # The query above can be time consumptive on large instances until we refactor how uploads are stored + # The query above can be time consumptive on large instances until we + # refactor how uploads are stored |> Repo.all(timout: :infinity) - # we should delete 1 object for any given attachment, but don't delete files if - # there are more than 1 object for it + # we should delete 1 object for any given attachment, but don't delete + # files if there are more than 1 object for it |> Enum.reduce(%{}, fn %{ id: id, data: %{ -- cgit v1.2.3 From 4c5b5f14dcec51cba8e86bcfcf2943ee9b49b0e4 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 20 Jan 2020 16:24:20 +0100 Subject: StatusView: Add `emoji_reactions` --- lib/pleroma/web/mastodon_api/views/status_view.ex | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index e9590224b..b59ac39bc 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -253,6 +253,16 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do nil end + emoji_reactions = + with %{data: %{"reactions" => emoji_reactions}} <- object do + Enum.map(emoji_reactions, fn {emoji, users} -> + {emoji, length(users)} + end) + |> Enum.into(%{}) + else + _ -> %{} + end + %{ id: to_string(activity.id), uri: object.data["id"], @@ -293,7 +303,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do spoiler_text: %{"text/plain" => summary_plaintext}, expires_at: expires_at, direct_conversation_id: direct_conversation_id, - thread_muted: thread_muted? + thread_muted: thread_muted?, + emoji_reactions: emoji_reactions } } end -- cgit v1.2.3 From 6e88a7e5917f558fe3e9d639297086a9e8b98939 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 20 Jan 2020 21:47:44 +0300 Subject: exclude blocked user posts from search results --- lib/pleroma/activity.ex | 4 +--- lib/pleroma/activity/queries.ex | 10 ++++++++++ lib/pleroma/activity/search.ex | 11 ++++++++--- lib/pleroma/web/mastodon_api/controllers/search_controller.ex | 2 +- 4 files changed, 20 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 510d3273c..896cbb3c5 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -312,9 +312,7 @@ defmodule Pleroma.Activity do from(u in User.Query.build(deactivated: true), select: u.ap_id) |> Repo.all() - from(activity in query, - where: activity.actor not in ^deactivated_users - ) + Activity.Queries.exclude_authors(query, deactivated_users) end defdelegate search(user, query, options \\ []), to: Pleroma.Activity.Search diff --git a/lib/pleroma/activity/queries.ex b/lib/pleroma/activity/queries.ex index 26bc1099d..79f305201 100644 --- a/lib/pleroma/activity/queries.ex +++ b/lib/pleroma/activity/queries.ex @@ -12,6 +12,7 @@ defmodule Pleroma.Activity.Queries do @type query :: Ecto.Queryable.t() | Activity.t() alias Pleroma.Activity + alias Pleroma.User @spec by_ap_id(query, String.t()) :: query def by_ap_id(query \\ Activity, ap_id) do @@ -29,6 +30,11 @@ defmodule Pleroma.Activity.Queries do ) end + @spec by_author(query, String.t()) :: query + def by_author(query \\ Activity, %User{ap_id: ap_id}) do + from(a in query, where: a.actor == ^ap_id) + end + @spec by_object_id(query, String.t() | [String.t()]) :: query def by_object_id(query \\ Activity, object_id) @@ -72,4 +78,8 @@ defmodule Pleroma.Activity.Queries do where: fragment("(?)->>'type' != ?", activity.data, ^activity_type) ) end + + def exclude_authors(query \\ Activity, actors) do + from(activity in query, where: activity.actor not in ^actors) + end end diff --git a/lib/pleroma/activity/search.ex b/lib/pleroma/activity/search.ex index d30a5a6a5..f96e208da 100644 --- a/lib/pleroma/activity/search.ex +++ b/lib/pleroma/activity/search.ex @@ -26,18 +26,23 @@ defmodule Pleroma.Activity.Search do |> query_with(index_type, search_query) |> maybe_restrict_local(user) |> maybe_restrict_author(author) + |> maybe_restrict_blocked(user) |> Pagination.fetch_paginated(%{"offset" => offset, "limit" => limit}, :offset) |> maybe_fetch(user, search_query) end def maybe_restrict_author(query, %User{} = author) do - from([a, o] in query, - where: a.actor == ^author.ap_id - ) + Activity.Queries.by_author(query, author) end def maybe_restrict_author(query, _), do: query + def maybe_restrict_blocked(query, %User{} = user) do + Activity.Queries.exclude_authors(query, User.blocked_users_ap_ids(user)) + end + + def maybe_restrict_blocked(query, _), do: query + defp restrict_public(q) do from([a, o] in q, where: fragment("?->>'type' = 'Create'", a.data), diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 0a929f55b..5a5db8e00 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -43,7 +43,7 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do result = default_values |> Enum.map(fn {resource, default_value} -> - if params["type"] == nil or params["type"] == resource do + if params["type"] in [nil, resource] do {resource, fn -> resource_search(version, resource, query, options) end} else {resource, fn -> default_value end} -- cgit v1.2.3