From 93291c3d7a4dc580a2c76653521b7f6653cf4113 Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Sun, 17 Mar 2019 13:23:08 +0300 Subject: Order users by nickname --- lib/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 692ae836c..6de766fc1 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -773,7 +773,7 @@ defmodule Pleroma.User do }) :: {:ok, [Pleroma.User.t()], number()} def search_for_admin(%{query: nil, local: local, page: page, page_size: page_size}) do query = - from(u in User, order_by: u.id) + from(u in User, order_by: u.nickname) |> maybe_local_user_query(local) paginated_query = -- cgit v1.2.3 From 4ed2618f6c732ba1009510f5698a5d981a151925 Mon Sep 17 00:00:00 2001 From: Fong-Wan Chau Date: Sun, 17 Mar 2019 09:46:46 -0400 Subject: Allow 'rel' attribute on `` link with specific values (for hashtag recognition). --- lib/pleroma/html.ex | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/html.ex b/lib/pleroma/html.ex index 05253157e..5b152d926 100644 --- a/lib/pleroma/html.ex +++ b/lib/pleroma/html.ex @@ -95,6 +95,13 @@ defmodule Pleroma.HTML.Scrubber.TwitterText do Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes) Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"]) + Meta.allow_tag_with_this_attribute_values("a", "rel", [ + "tag", + "nofollow", + "noopener", + "noreferrer" + ]) + # paragraphs and linebreaks Meta.allow_tag_with_these_attributes("br", []) Meta.allow_tag_with_these_attributes("p", []) @@ -137,6 +144,13 @@ defmodule Pleroma.HTML.Scrubber.Default do Meta.allow_tag_with_uri_attributes("a", ["href", "data-user", "data-tag"], @valid_schemes) Meta.allow_tag_with_these_attributes("a", ["name", "title", "class"]) + Meta.allow_tag_with_this_attribute_values("a", "rel", [ + "tag", + "nofollow", + "noopener", + "noreferrer" + ]) + Meta.allow_tag_with_these_attributes("abbr", ["title"]) Meta.allow_tag_with_these_attributes("b", []) -- cgit v1.2.3 From 1588688a11ab82bf794d736fd6d18eebf1269dfe Mon Sep 17 00:00:00 2001 From: eugenijm Date: Mon, 18 Mar 2019 04:32:23 +0300 Subject: Added support for exclude_types, limit, and min_id in Mastodon notifications. Unify Mastodon-compatible pagination logic. --- lib/pleroma/activity.ex | 8 +++ lib/pleroma/notification.ex | 36 +++------- lib/pleroma/pagination.ex | 78 ++++++++++++++++++++++ lib/pleroma/web/mastodon_api/mastodon_api.ex | 48 +++++-------- .../web/mastodon_api/mastodon_api_controller.ex | 2 +- 5 files changed, 114 insertions(+), 58 deletions(-) create mode 100644 lib/pleroma/pagination.ex (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 79dc26b01..de0e66681 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -22,6 +22,10 @@ defmodule Pleroma.Activity do "Like" => "favourite" } + @mastodon_to_ap_notification_types for {k, v} <- @mastodon_notification_types, + into: %{}, + do: {v, k} + schema "activities" do field(:data, :map) field(:local, :boolean, default: true) @@ -126,6 +130,10 @@ defmodule Pleroma.Activity do def mastodon_notification_type(%Activity{}), do: nil + def from_mastodon_notification_type(type) do + Map.get(@mastodon_to_ap_notification_types, type) + end + def all_by_actor_and_id(actor, status_ids \\ []) def all_by_actor_and_id(_actor, []), do: [] diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index 765191275..a98649b63 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Notification do alias Pleroma.Activity alias Pleroma.Notification + alias Pleroma.Pagination alias Pleroma.Repo alias Pleroma.User alias Pleroma.Web.CommonAPI @@ -28,36 +29,17 @@ defmodule Pleroma.Notification do |> cast(attrs, [:seen]) end - # TODO: Make generic and unify (see activity_pub.ex) - defp restrict_max(query, %{"max_id" => max_id}) do - from(activity in query, where: activity.id < ^max_id) + def for_user_query(user) do + Notification + |> where(user_id: ^user.id) + |> join(:inner, [n], activity in assoc(n, :activity)) + |> preload(:activity) end - defp restrict_max(query, _), do: query - - defp restrict_since(query, %{"since_id" => since_id}) do - from(activity in query, where: activity.id > ^since_id) - end - - defp restrict_since(query, _), do: query - def for_user(user, opts \\ %{}) do - query = - from( - n in Notification, - where: n.user_id == ^user.id, - order_by: [desc: n.id], - join: activity in assoc(n, :activity), - preload: [activity: activity], - limit: 20 - ) - - query = - query - |> restrict_since(opts) - |> restrict_max(opts) - - Repo.all(query) + user + |> for_user_query() + |> Pagination.fetch_paginated(opts) end def set_read_up_to(%{id: user_id} = _user, id) do diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex new file mode 100644 index 000000000..7c864deef --- /dev/null +++ b/lib/pleroma/pagination.ex @@ -0,0 +1,78 @@ +defmodule Pleroma.Pagination do + @moduledoc """ + Implements Mastodon-compatible pagination. + """ + + import Ecto.Query + import Ecto.Changeset + + alias Pleroma.Repo + + @default_limit 20 + + def fetch_paginated(query, params) do + options = cast_params(params) + + query + |> paginate(options) + |> Repo.all() + |> enforce_order(options) + end + + def paginate(query, options) do + query + |> restrict(:min_id, options) + |> restrict(:since_id, options) + |> restrict(:max_id, options) + |> restrict(:order, options) + |> restrict(:limit, options) + end + + defp cast_params(params) do + param_types = %{ + min_id: :string, + since_id: :string, + max_id: :string, + limit: :integer + } + + changeset = cast({%{}, param_types}, params, Map.keys(param_types)) + changeset.changes + end + + defp restrict(query, :min_id, %{min_id: min_id}) do + where(query, [q], q.id > ^min_id) + end + + defp restrict(query, :since_id, %{since_id: since_id}) do + where(query, [q], q.id > ^since_id) + end + + defp restrict(query, :max_id, %{max_id: max_id}) do + where(query, [q], q.id < ^max_id) + end + + defp restrict(query, :order, %{min_id: _}) do + order_by(query, [u], fragment("? asc nulls last", u.id)) + end + + defp restrict(query, :order, _options) do + order_by(query, [u], fragment("? desc nulls last", u.id)) + end + + defp restrict(query, :limit, options) do + limit = Map.get(options, :limit, @default_limit) + + query + |> limit(^limit) + end + + defp restrict(query, _, _), do: query + + defp enforce_order(result, %{min_id: _}) do + result + |> Enum.reverse() + end + + defp enforce_order(result, _), do: result +end diff --git a/lib/pleroma/web/mastodon_api/mastodon_api.ex b/lib/pleroma/web/mastodon_api/mastodon_api.ex index 54cb6c97a..08ea5f967 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api.ex @@ -2,61 +2,49 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPI do import Ecto.Query import Ecto.Changeset - alias Pleroma.Repo + alias Pleroma.Activity + alias Pleroma.Notification + alias Pleroma.Pagination alias Pleroma.User - @default_limit 20 - def get_followers(user, params \\ %{}) do user |> User.get_followers_query() - |> paginate(params) - |> Repo.all() + |> Pagination.fetch_paginated(params) end def get_friends(user, params \\ %{}) do user |> User.get_friends_query() - |> paginate(params) - |> Repo.all() + |> Pagination.fetch_paginated(params) end - def paginate(query, params \\ %{}) do + def get_notifications(user, params \\ %{}) do options = cast_params(params) - query - |> restrict(:max_id, options) - |> restrict(:since_id, options) - |> restrict(:limit, options) - |> order_by([u], fragment("? desc nulls last", u.id)) + user + |> Notification.for_user_query() + |> restrict(:exclude_types, options) + |> Pagination.fetch_paginated(params) end - def cast_params(params) do + defp cast_params(params) do param_types = %{ - max_id: :string, - since_id: :string, - limit: :integer + exclude_types: {:array, :string} } changeset = cast({%{}, param_types}, params, Map.keys(param_types)) changeset.changes end - defp restrict(query, :max_id, %{max_id: max_id}) do - query - |> where([q], q.id < ^max_id) - end - - defp restrict(query, :since_id, %{since_id: since_id}) do - query - |> where([q], q.id > ^since_id) - end - - defp restrict(query, :limit, options) do - limit = Map.get(options, :limit, @default_limit) + defp restrict(query, :exclude_types, %{exclude_types: mastodon_types = [_ | _]}) do + ap_types = + mastodon_types + |> Enum.map(&Activity.from_mastodon_notification_type/1) + |> Enum.filter(& &1) query - |> limit(^limit) + |> where([q, a], not fragment("? @> ARRAY[?->>'type']::varchar[]", ^ap_types, a.data)) end defp restrict(query, _, _), do: query diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 952aa2453..2eb1da561 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -502,7 +502,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end def notifications(%{assigns: %{user: user}} = conn, params) do - notifications = Notification.for_user(user, params) + notifications = MastodonAPI.get_notifications(user, params) conn |> add_link_headers(:notifications, notifications) -- cgit v1.2.3 From a1aacc08a43b287e0f9b7337da6230cff9e0e5c5 Mon Sep 17 00:00:00 2001 From: Ekaterina Vaartis Date: Mon, 18 Mar 2019 20:14:49 +0300 Subject: Check if the user has indeed not been federated with Just updating the user triggered post fetching too, now it shouldn't. It only happened in the AP user fetching since that's what's used to update users --- lib/pleroma/user.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 692ae836c..f72bc4bce 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1159,9 +1159,12 @@ defmodule Pleroma.User do if !is_nil(user) and !User.needs_update?(user) do user else + # Whether to fetch initial posts for the user (if it's a new user & the fetching is enabled) + should_fetch_initial = is_nil(user) and Pleroma.Config.get([:fetch_initial_posts, :enabled]) + user = fetch_by_ap_id(ap_id) - if Pleroma.Config.get([:fetch_initial_posts, :enabled]) do + if should_fetch_initial do with %User{} = user do {:ok, _} = Task.start(__MODULE__, :fetch_initial_posts, [user]) end -- cgit v1.2.3 From 798da28812b7af2e79e2c59896418192efcab543 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 19 Mar 2019 17:27:42 +0000 Subject: activitypub: transmogrifier: ensure as:Public activities are delivered to followers --- lib/pleroma/web/activity_pub/transmogrifier.ex | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 8e4bf7b47..7f3d8fd4b 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -128,13 +128,42 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do |> fix_explicit_addressing(explicit_mentions) end + # if as:Public is addressed, then make sure the followers collection is also addressed + # so that the activities will be delivered to local users. + def fix_implicit_addressing(%{"to" => to, "cc" => cc} = object, followers_collection) do + recipients = to ++ cc + + if followers_collection not in recipients do + cond do + "https://www.w3.org/ns/activitystreams#Public" in cc -> + to = to ++ [followers_collection] + Map.put(object, "to", to) + + "https://www.w3.org/ns/activitystreams#Public" in to -> + cc = cc ++ [followers_collection] + Map.put(object, "cc", cc) + + true -> + object + end + else + object + end + end + + def fix_implicit_addressing(object, _), do: object + def fix_addressing(object) do + %User{} = user = User.get_cached_by_ap_id(object["actor"]) + followers_collection = User.ap_followers(user) + object |> fix_addressing_list("to") |> fix_addressing_list("cc") |> fix_addressing_list("bto") |> fix_addressing_list("bcc") |> fix_explicit_addressing + |> fix_implicit_addressing(followers_collection) end def fix_actor(%{"attributedTo" => actor} = object) do -- cgit v1.2.3 From d487b753c3116e7a4261404b2357f337acc2d64d Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 19 Mar 2019 17:30:25 +0000 Subject: activitypub: transmogrifier: do not allow missing lists to be interpreted as nil --- lib/pleroma/web/activity_pub/transmogrifier.ex | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 7f3d8fd4b..9d536f7f5 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -86,11 +86,15 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do end def fix_addressing_list(map, field) do - if is_binary(map[field]) do - map - |> Map.put(field, [map[field]]) - else - map + cond do + is_binary(map[field]) -> + Map.put(map, field, [map[field]]) + + is_nil(map[field]) -> + Map.put(map, field, []) + + true -> + map end end -- cgit v1.2.3 From cd055983c3d066c9f08aa26b62d3ec6d4419cb7c Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 19 Mar 2019 18:04:57 +0000 Subject: transmogrifier: when determining followers collection URI, we may need to fetch the actor --- lib/pleroma/web/activity_pub/transmogrifier.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 9d536f7f5..4f663d01e 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -158,7 +158,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do def fix_implicit_addressing(object, _), do: object def fix_addressing(object) do - %User{} = user = User.get_cached_by_ap_id(object["actor"]) + %User{} = user = User.get_or_fetch_by_ap_id(object["actor"]) followers_collection = User.ap_followers(user) object -- cgit v1.2.3 From 67ff8d9311d453220af1b84385281a077a3cb20e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 19 Mar 2019 18:23:06 +0000 Subject: user: properly cope with actors which do not declare a followers collection --- lib/pleroma/user.ex | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index fb3bd121d..8df276ae0 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -104,9 +104,8 @@ defmodule Pleroma.User do "#{Web.base_url()}/users/#{nickname}" end - def ap_followers(%User{} = user) do - "#{ap_id(user)}/followers" - end + def ap_followers(%User{follower_address: fa}) when is_binary(fa), do: fa + def ap_followers(%User{} = user), do: "#{ap_id(user)}/followers" def user_info(%User{} = user) do oneself = if user.local, do: 1, else: 0 -- cgit v1.2.3 From 1685e4258f452343c86d9dd3914076f101c8ed73 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 19 Mar 2019 18:39:33 +0000 Subject: transmogrifier: upgrade: when upgrading OStatus users to AP, ensure we always use the fake collection --- lib/pleroma/web/activity_pub/transmogrifier.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 4f663d01e..f733ae7e1 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -955,7 +955,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do defp strip_internal_tags(object), do: object defp user_upgrade_task(user) do - old_follower_address = User.ap_followers(user) + # we pass a fake user so that the followers collection is stripped away + old_follower_address = User.ap_followers(%User{nickname: user.nickname}) q = from( -- cgit v1.2.3 From ed8a2935f55b5271e65955ddb7446e322439599b Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 20 Mar 2019 01:37:40 +0300 Subject: Use ILIKE to search users --- lib/pleroma/user.ex | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 8df276ae0..bf84eaf7f 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -788,34 +788,26 @@ defmodule Pleroma.User do @spec search_for_admin(%{ query: binary(), - admin: Pleroma.User.t(), local: boolean(), page: number(), page_size: number() }) :: {:ok, [Pleroma.User.t()], number()} def search_for_admin(%{ query: term, - admin: admin, local: local, page: page, page_size: page_size }) do - term = String.trim_leading(term, "@") + maybe_local_query = User |> maybe_local_user_query(local) - local_paginated_query = - User - |> maybe_local_user_query(local) + search_query = from(u in maybe_local_query, where: ilike(u.nickname, ^"%#{term}%")) + count = search_query |> Repo.aggregate(:count, :id) + results = + search_query |> paginate(page, page_size) + |> Repo.all() - search_query = fts_search_subquery(term, local_paginated_query) - - count = - term - |> fts_search_subquery() - |> maybe_local_user_query(local) - |> Repo.aggregate(:count, :id) - - {:ok, do_search(search_query, admin), count} + {:ok, results, count} end def search(query, resolve \\ false, for_user \\ nil) do -- cgit v1.2.3 From 7ac9fff437a2aee3403a769b6cdfb3533771b03e Mon Sep 17 00:00:00 2001 From: Maxim Filippov Date: Wed, 20 Mar 2019 01:40:50 +0300 Subject: Format --- lib/pleroma/user.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index bf84eaf7f..c1aebfc29 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -802,6 +802,7 @@ defmodule Pleroma.User do search_query = from(u in maybe_local_query, where: ilike(u.nickname, ^"%#{term}%")) count = search_query |> Repo.aggregate(:count, :id) + results = search_query |> paginate(page, page_size) -- cgit v1.2.3 From 9a7b817c9a785171368a9b6d61122c7b4b33547c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 20 Mar 2019 15:59:27 +0300 Subject: Ecto 3.0.5 migration kms --- lib/pleroma/activity.ex | 3 ++- lib/pleroma/instances/instance.ex | 2 +- lib/pleroma/repo.ex | 2 +- lib/pleroma/user.ex | 28 +++++++++++++--------- lib/pleroma/web/oauth/authorization.ex | 2 +- lib/pleroma/web/oauth/token.ex | 2 +- .../web/websub/websub_client_subscription.ex | 2 +- 7 files changed, 24 insertions(+), 17 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 79dc26b01..b8a3d3054 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -109,7 +109,8 @@ defmodule Pleroma.Activity do def delete_by_ap_id(id) when is_binary(id) do by_object_ap_id(id) - |> Repo.delete_all(returning: true) + |> select([u], u) + |> Repo.delete_all() |> elem(1) |> Enum.find(fn %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id diff --git a/lib/pleroma/instances/instance.ex b/lib/pleroma/instances/instance.ex index e92006151..420803a8f 100644 --- a/lib/pleroma/instances/instance.ex +++ b/lib/pleroma/instances/instance.ex @@ -12,7 +12,7 @@ defmodule Pleroma.Instances.Instance do schema "instances" do field(:host, :string) - field(:unreachable_since, :naive_datetime) + field(:unreachable_since, :naive_datetime_usec) timestamps() end diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index e6a51b19e..224d784e3 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Repo do - use Ecto.Repo, otp_app: :pleroma + use Ecto.Repo, otp_app: :pleroma, adapter: Ecto.Adapters.Postgres @doc """ Dynamically loads the repository url from the diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 692ae836c..7a6675208 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -52,7 +52,7 @@ defmodule Pleroma.User do field(:search_rank, :float, virtual: true) field(:tags, {:array, :string}, default: []) field(:bookmarks, {:array, :string}, default: []) - field(:last_refreshed_at, :naive_datetime) + field(:last_refreshed_at, :naive_datetime_usec) has_many(:notifications, Notification) embeds_one(:info, Pleroma.User.Info) @@ -335,10 +335,11 @@ defmodule Pleroma.User do ^followed_addresses ) ] - ] + ], + select: u ) - {1, [follower]} = Repo.update_all(q, [], returning: true) + {1, [follower]} = Repo.update_all(q, []) Enum.each(followeds, fn followed -> update_follower_count(followed) @@ -368,10 +369,11 @@ defmodule Pleroma.User do q = from(u in User, where: u.id == ^follower.id, - update: [push: [following: ^ap_followers]] + update: [push: [following: ^ap_followers]], + select: u ) - {1, [follower]} = Repo.update_all(q, [], returning: true) + {1, [follower]} = Repo.update_all(q, []) {:ok, _} = update_follower_count(followed) @@ -386,10 +388,11 @@ defmodule Pleroma.User do q = from(u in User, where: u.id == ^follower.id, - update: [pull: [following: ^ap_followers]] + update: [pull: [following: ^ap_followers]], + select: u ) - {1, [follower]} = Repo.update_all(q, [], returning: true) + {1, [follower]} = Repo.update_all(q, []) {:ok, followed} = update_follower_count(followed) @@ -637,7 +640,7 @@ defmodule Pleroma.User do users = user |> User.get_follow_requests_query() - |> join(:inner, [a], u in User, a.actor == u.ap_id) + |> join(:inner, [a], u in User, on: a.actor == u.ap_id) |> where([a, u], not fragment("? @> ?", u.following, ^[user.follower_address])) |> group_by([a, u], u.id) |> select([a, u], u) @@ -659,7 +662,8 @@ defmodule Pleroma.User do ) ] ) - |> Repo.update_all([], returning: true) + |> select([u], u) + |> Repo.update_all([]) |> case do {1, [user]} -> set_cache(user) _ -> {:error, user} @@ -679,7 +683,8 @@ defmodule Pleroma.User do ) ] ) - |> Repo.update_all([], returning: true) + |> select([u], u) + |> Repo.update_all([]) |> case do {1, [user]} -> set_cache(user) _ -> {:error, user} @@ -725,7 +730,8 @@ defmodule Pleroma.User do ) ] ) - |> Repo.update_all([], returning: true) + |> select([u], u) + |> Repo.update_all([]) |> case do {1, [user]} -> set_cache(user) _ -> {:error, user} diff --git a/lib/pleroma/web/oauth/authorization.ex b/lib/pleroma/web/oauth/authorization.ex index a80543adf..3461f9983 100644 --- a/lib/pleroma/web/oauth/authorization.ex +++ b/lib/pleroma/web/oauth/authorization.ex @@ -16,7 +16,7 @@ defmodule Pleroma.Web.OAuth.Authorization do schema "oauth_authorizations" do field(:token, :string) field(:scopes, {:array, :string}, default: []) - field(:valid_until, :naive_datetime) + field(:valid_until, :naive_datetime_usec) field(:used, :boolean, default: false) belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) belongs_to(:app, App) diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex index 2b074b470..a8b06db36 100644 --- a/lib/pleroma/web/oauth/token.ex +++ b/lib/pleroma/web/oauth/token.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.OAuth.Token do field(:token, :string) field(:refresh_token, :string) field(:scopes, {:array, :string}, default: []) - field(:valid_until, :naive_datetime) + field(:valid_until, :naive_datetime_usec) belongs_to(:user, Pleroma.User, type: Pleroma.FlakeId) belongs_to(:app, App) diff --git a/lib/pleroma/web/websub/websub_client_subscription.ex b/lib/pleroma/web/websub/websub_client_subscription.ex index 969ee0684..77703c496 100644 --- a/lib/pleroma/web/websub/websub_client_subscription.ex +++ b/lib/pleroma/web/websub/websub_client_subscription.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.Websub.WebsubClientSubscription do schema "websub_client_subscriptions" do field(:topic, :string) field(:secret, :string) - field(:valid_until, :naive_datetime) + field(:valid_until, :naive_datetime_usec) field(:state, :string) field(:subscribers, {:array, :string}, default: []) field(:hub, :string) -- cgit v1.2.3 From 19a19bdd8145bd91b38d9c6571d2e7c597754055 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 20 Mar 2019 16:04:59 +0300 Subject: Fix migration timestamp type --- lib/pleroma/repo.ex | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index 224d784e3..4af1bde56 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -3,7 +3,10 @@ # SPDX-License-Identifier: AGPL-3.0-only defmodule Pleroma.Repo do - use Ecto.Repo, otp_app: :pleroma, adapter: Ecto.Adapters.Postgres + use Ecto.Repo, + otp_app: :pleroma, + adapter: Ecto.Adapters.Postgres, + migration_timestamps: [type: :naive_datetime_usec] @doc """ Dynamically loads the repository url from the -- cgit v1.2.3 From 8468f3f6d48693d2a27a257e5555aa71decff3df Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 20 Mar 2019 21:09:36 +0100 Subject: Add safe dm mode option. --- lib/pleroma/formatter.ex | 20 +++++++++++++++++--- lib/pleroma/web/common_api/common_api.ex | 3 ++- lib/pleroma/web/common_api/utils.ex | 12 ++++++++++-- 3 files changed, 29 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 1e4ede3f2..e3625383b 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy + @safe_mention_regex ~r/^(\s*(?@.+?\s+)+)(?.*)/ @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ @link_regex ~r{((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+}ui # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength @@ -45,15 +46,28 @@ defmodule Pleroma.Formatter do @doc """ Parses a text and replace plain text links with HTML. Returns a tuple with a result text, mentions, and hashtags. + + If the 'safe_mention' option is given, only consecutive mentions at the start the post are actually mentioned. """ @spec linkify(String.t(), keyword()) :: {String.t(), [{String.t(), User.t()}], [{String.t(), String.t()}]} def linkify(text, options \\ []) do options = options ++ @auto_linker_config - acc = %{mentions: MapSet.new(), tags: MapSet.new()} - {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) - {text, MapSet.to_list(mentions), MapSet.to_list(tags)} + if options[:safe_mention] && Regex.named_captures(@safe_mention_regex, text) do + %{"mentions" => mentions, "rest" => rest} = Regex.named_captures(@safe_mention_regex, text) + acc = %{mentions: MapSet.new(), tags: MapSet.new()} + + {text_mentions, %{mentions: mentions}} = AutoLinker.link_map(mentions, acc, options) + {text_rest, %{tags: tags}} = AutoLinker.link_map(rest, acc, options) + + {text_mentions <> text_rest, MapSet.to_list(mentions), MapSet.to_list(tags)} + else + acc = %{mentions: MapSet.new(), tags: MapSet.new()} + {text, %{mentions: mentions, tags: tags}} = AutoLinker.link_map(text, acc, options) + + {text, MapSet.to_list(mentions), MapSet.to_list(tags)} + end end def emojify(text) do diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index b5f79c3bf..50d60aade 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -142,7 +142,8 @@ defmodule Pleroma.Web.CommonAPI do make_content_html( status, attachments, - data + data, + visibility ), {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b7513ef28..368945418 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -101,7 +101,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do def make_content_html( status, attachments, - data + data, + visibility ) do no_attachment_links = data @@ -110,8 +111,15 @@ defmodule Pleroma.Web.CommonAPI.Utils do content_type = get_content_type(data["content_type"]) + options = + if visibility == "direct" && Config.get([:instance, :safe_dm_mentions]) do + [safe_mention: true] + else + [] + end + status - |> format_input(content_type) + |> format_input(content_type, options) |> maybe_add_attachments(attachments, no_attachment_links) |> maybe_add_nsfw_tag(data) end -- cgit v1.2.3 From bf27190f7f0942a05de518f2085a299eb011614c Mon Sep 17 00:00:00 2001 From: lain Date: Thu, 21 Mar 2019 16:16:26 +0100 Subject: UtilController: Return state of safe dm mentions. --- lib/pleroma/web/twitter_api/controllers/util_controller.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/twitter_api/controllers/util_controller.ex b/lib/pleroma/web/twitter_api/controllers/util_controller.ex index 320ec778c..faa733fec 100644 --- a/lib/pleroma/web/twitter_api/controllers/util_controller.ex +++ b/lib/pleroma/web/twitter_api/controllers/util_controller.ex @@ -197,7 +197,9 @@ defmodule Pleroma.Web.TwitterAPI.UtilController do vapidPublicKey: vapid_public_key, accountActivationRequired: if(Keyword.get(instance, :account_activation_required, false), do: "1", else: "0"), - invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0") + invitesEnabled: if(Keyword.get(instance, :invites_enabled, false), do: "1", else: "0"), + safeDMMentionsEnabled: + if(Pleroma.Config.get([:instance, :safe_dm_mentions]), do: "1", else: "0") } pleroma_fe = -- cgit v1.2.3 From 80bc9ed2ba9aa86edf8c1756c9ee59ad2a9bf20b Mon Sep 17 00:00:00 2001 From: Quentin Rameau Date: Mon, 18 Mar 2019 15:47:58 +0100 Subject: Add a gopher url port config option This lets the user advertise a different port in the gopher urls, for example listening locally on port 7070 but telling clients to connect to the regular port 70. --- lib/pleroma/gopher/server.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex index 6baacc566..3b9629d77 100644 --- a/lib/pleroma/gopher/server.ex +++ b/lib/pleroma/gopher/server.ex @@ -66,7 +66,8 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do def link(name, selector, type \\ 1) do address = Pleroma.Web.Endpoint.host() port = Pleroma.Config.get([:gopher, :port], 1234) - "#{type}#{name}\t#{selector}\t#{address}\t#{port}\r\n" + dstport = Pleroma.Config.get([:gopher, :dstport], port) + "#{type}#{name}\t#{selector}\t#{address}\t#{dstport}\r\n" end def render_activities(activities) do -- cgit v1.2.3 From 88096c65a525a44c845e651fc082f6374fad6042 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Thu, 21 Mar 2019 23:16:32 +0300 Subject: Move gluing search results from application to database and get mutuals a higher score multiplier --- lib/pleroma/user.ex | 83 +++++++++++++++++++++++++---------------------------- 1 file changed, 39 insertions(+), 44 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index eb0933c85..e16141edc 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -823,31 +823,53 @@ defmodule Pleroma.User do if resolve, do: get_or_fetch(query) - fts_results = do_search(fts_search_subquery(query), for_user) - - {:ok, trigram_results} = + {:ok, results} = Repo.transaction(fn -> Ecto.Adapters.SQL.query(Repo, "select set_limit(0.25)", []) - do_search(trigram_search_subquery(query), for_user) + Repo.all(search_query(query, for_user)) end) - Enum.uniq_by(fts_results ++ trigram_results, & &1.id) + results end - defp do_search(subquery, for_user, options \\ []) do - q = - from( - s in subquery(subquery), - order_by: [desc: s.search_rank], - limit: ^(options[:limit] || 20) - ) + def search_query(query, for_user) do + fts_subquery = fts_search_subquery(query) + trigram_subquery = trigram_search_subquery(query) + union_query = from(s in trigram_subquery, union: ^fts_subquery) + distinct_query = from(s in subquery(union_query), distinct: s.id) - results = - q - |> Repo.all() - |> Enum.filter(&(&1.search_rank > 0)) + from(s in subquery(boost_search_rank_query(distinct_query, for_user)), + order_by: [desc: s.search_rank], + limit: 20 + ) + end - boost_search_results(results, for_user) + defp boost_search_rank_query(query, nil), do: query + + defp boost_search_rank_query(query, for_user) do + friends_ids = get_friends_ids(for_user) + followers_ids = get_followers_ids(for_user) + + from(u in subquery(query), + select_merge: %{ + search_rank: + fragment( + """ + CASE WHEN (?) THEN (?) * 1.3 + WHEN (?) THEN (?) * 1.2 + WHEN (?) THEN (?) * 1.1 + ELSE (?) END + """, + u.id in ^friends_ids and u.id in ^followers_ids, + u.search_rank, + u.id in ^friends_ids, + u.search_rank, + u.id in ^followers_ids, + u.search_rank, + u.search_rank + ) + } + ) end defp fts_search_subquery(term, query \\ User) do @@ -906,33 +928,6 @@ defmodule Pleroma.User do ) end - defp boost_search_results(results, nil), do: results - - defp boost_search_results(results, for_user) do - friends_ids = get_friends_ids(for_user) - followers_ids = get_followers_ids(for_user) - - Enum.map( - results, - fn u -> - search_rank_coef = - cond do - u.id in friends_ids -> - 1.2 - - u.id in followers_ids -> - 1.1 - - true -> - 1 - end - - Map.put(u, :search_rank, u.search_rank * search_rank_coef) - end - ) - |> Enum.sort_by(&(-&1.search_rank)) - end - def blocks_import(%User{} = blocker, blocked_identifiers) when is_list(blocked_identifiers) do Enum.map( blocked_identifiers, -- cgit v1.2.3 From fea36967999fed5399ab3533e806e4cbc990ad05 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 21 Mar 2019 23:17:53 +0000 Subject: common api: move context functions from twitterapi --- lib/pleroma/web/common_api/utils.ex | 29 +++++++++++++++++++++ lib/pleroma/web/twitter_api/twitter_api.ex | 30 ---------------------- .../web/twitter_api/twitter_api_controller.ex | 3 ++- lib/pleroma/web/twitter_api/views/activity_view.ex | 3 +-- 4 files changed, 32 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index b7513ef28..fcdfea8e1 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -344,4 +344,33 @@ defmodule Pleroma.Web.CommonAPI.Utils do end def get_report_statuses(_, _), do: {:ok, nil} + + # DEPRECATED mostly, context objects are now created at insertion time. + def context_to_conversation_id(context) do + with %Object{id: id} <- Object.get_cached_by_ap_id(context) do + id + else + _e -> + changeset = Object.context_mapping(context) + + case Repo.insert(changeset) do + {:ok, %{id: id}} -> + id + + # This should be solved by an upsert, but it seems ecto + # has problems accessing the constraint inside the jsonb. + {:error, _} -> + Object.get_cached_by_ap_id(context).id + end + end + end + + def conversation_id_to_context(id) do + with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do + context + else + _e -> + {:error, "No such conversation"} + end + end end diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex index d57100491..9978c7f64 100644 --- a/lib/pleroma/web/twitter_api/twitter_api.ex +++ b/lib/pleroma/web/twitter_api/twitter_api.ex @@ -5,7 +5,6 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do alias Pleroma.Activity alias Pleroma.Mailer - alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User alias Pleroma.UserEmail @@ -282,35 +281,6 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do _activities = Repo.all(q) end - # DEPRECATED mostly, context objects are now created at insertion time. - def context_to_conversation_id(context) do - with %Object{id: id} <- Object.get_cached_by_ap_id(context) do - id - else - _e -> - changeset = Object.context_mapping(context) - - case Repo.insert(changeset) do - {:ok, %{id: id}} -> - id - - # This should be solved by an upsert, but it seems ecto - # has problems accessing the constraint inside the jsonb. - {:error, _} -> - Object.get_cached_by_ap_id(context).id - end - end - end - - def conversation_id_to_context(id) do - with %Object{data: %{"id" => context}} <- Repo.get(Object, id) do - context - else - _e -> - {:error, "No such conversation"} - end - end - def get_external_profile(for_user, uri) do with %User{} = user <- User.get_or_fetch(uri) do {:ok, UserView.render("show.json", %{user: user, for: for_user})} diff --git a/lib/pleroma/web/twitter_api/twitter_api_controller.ex b/lib/pleroma/web/twitter_api/twitter_api_controller.ex index 6ea0b110b..62cce18dc 100644 --- a/lib/pleroma/web/twitter_api/twitter_api_controller.ex +++ b/lib/pleroma/web/twitter_api/twitter_api_controller.ex @@ -16,6 +16,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Visibility alias Pleroma.Web.CommonAPI + alias Pleroma.Web.CommonAPI.Utils alias Pleroma.Web.OAuth.Token alias Pleroma.Web.TwitterAPI.ActivityView alias Pleroma.Web.TwitterAPI.NotificationView @@ -278,7 +279,7 @@ defmodule Pleroma.Web.TwitterAPI.Controller do end def fetch_conversation(%{assigns: %{user: user}} = conn, %{"id" => id}) do - with context when is_binary(context) <- TwitterAPI.conversation_id_to_context(id), + with context when is_binary(context) <- Utils.conversation_id_to_context(id), activities <- ActivityPub.fetch_activities_for_context(context, %{ "blocking_user" => user, diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 4926f007e..fe7d49975 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -15,7 +15,6 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do alias Pleroma.Web.MastodonAPI.StatusView alias Pleroma.Web.TwitterAPI.ActivityView alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter - alias Pleroma.Web.TwitterAPI.TwitterAPI alias Pleroma.Web.TwitterAPI.UserView import Ecto.Query @@ -78,7 +77,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do defp get_context_id(%{data: %{"context" => context}}, options) do cond do id = options[:context_ids][context] -> id - true -> TwitterAPI.context_to_conversation_id(context) + true -> Utils.context_to_conversation_id(context) end end -- cgit v1.2.3 From 3cc2554fa3d63ba22dc5f598229a02b928b9fd14 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Thu, 21 Mar 2019 23:25:41 +0000 Subject: mastodon api: add conversation_id extension (ref #674) --- lib/pleroma/web/mastodon_api/views/status_view.ex | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index 209119dd5..1ca8338cc 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -46,6 +46,14 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do end end + defp get_context_id(%{data: %{"context_id" => context_id}}) when not is_nil(context_id), + do: context_id + + defp get_context_id(%{data: %{"context" => context}}) when is_binary(context), + do: Utils.context_to_conversation_id(context) + + defp get_context_id(_), do: nil + def render("index.json", opts) do replied_to_activities = get_replied_to_activities(opts.activities) @@ -186,7 +194,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do language: nil, emojis: build_emojis(activity.data["object"]["emoji"]), pleroma: %{ - local: activity.local + local: activity.local, + conversation_id: get_context_id(activity) } } end -- cgit v1.2.3 From 27e03a21773445375da7b0888ef517f14e6ad219 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 22 Mar 2019 01:17:14 +0000 Subject: reports: fix up email generation for remote reports --- lib/pleroma/emails/admin_email.ex | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/emails/admin_email.ex b/lib/pleroma/emails/admin_email.ex index 9b20c7e08..afefccec5 100644 --- a/lib/pleroma/emails/admin_email.ex +++ b/lib/pleroma/emails/admin_email.ex @@ -29,9 +29,13 @@ defmodule Pleroma.AdminEmail do if length(statuses) > 0 do statuses_list_html = statuses - |> Enum.map(fn %{id: id} -> - status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id) - "
  • #{status_url}
    • " + |> Enum.map(fn + %{id: id} -> + status_url = Helpers.o_status_url(Pleroma.Web.Endpoint, :notice, id) + "
  • #{status_url}
    • " + + id when is_binary(id) -> + "
  • #{id}
    • " end) |> Enum.join("\n") -- cgit v1.2.3 From 3229c7a1d689802f21a6efe648fcc56b8def3aa0 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Fri, 22 Mar 2019 08:39:49 +0300 Subject: Ensure fts is prefered over trigram and use union_all instead of union in user search query --- lib/pleroma/user.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index e16141edc..f2ef0a838 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -50,6 +50,7 @@ defmodule Pleroma.User do field(:local, :boolean, default: true) field(:follower_address, :string) field(:search_rank, :float, virtual: true) + field(:search_type, :integer, virtual: true) field(:tags, {:array, :string}, default: []) field(:bookmarks, {:array, :string}, default: []) field(:last_refreshed_at, :naive_datetime_usec) @@ -835,8 +836,8 @@ defmodule Pleroma.User do def search_query(query, for_user) do fts_subquery = fts_search_subquery(query) trigram_subquery = trigram_search_subquery(query) - union_query = from(s in trigram_subquery, union: ^fts_subquery) - distinct_query = from(s in subquery(union_query), distinct: s.id) + union_query = from(s in trigram_subquery, union_all: ^fts_subquery) + distinct_query = from(s in subquery(union_query), order_by: s.search_type, distinct: s.id) from(s in subquery(boost_search_rank_query(distinct_query, for_user)), order_by: [desc: s.search_rank], @@ -884,6 +885,7 @@ defmodule Pleroma.User do from( u in query, select_merge: %{ + search_type: ^0, search_rank: fragment( """ @@ -916,6 +918,8 @@ defmodule Pleroma.User do from( u in User, select_merge: %{ + # ^1 gives 'Postgrex expected a binary, got 1' for some weird reason + search_type: fragment("?", 1), search_rank: fragment( "similarity(?, trim(? || ' ' || coalesce(?, '')))", -- cgit v1.2.3 From e2afce34b68d88ec8e1ff624fe0a245fd0726fee Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 22 Mar 2019 11:57:20 +0100 Subject: NodeInfo: Return safe_dm_mentions feature flag. --- lib/pleroma/web/nodeinfo/nodeinfo_controller.ex | 3 +++ 1 file changed, 3 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex index 8c775ce24..216a962bd 100644 --- a/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex +++ b/lib/pleroma/web/nodeinfo/nodeinfo_controller.ex @@ -124,6 +124,9 @@ defmodule Pleroma.Web.Nodeinfo.NodeinfoController do end, if Keyword.get(instance, :allow_relay) do "relay" + end, + if Keyword.get(instance, :safe_dm_mentions) do + "safe_dm_mentions" end ] |> Enum.filter(& &1) -- cgit v1.2.3 From 1b33986bfa3e924d49e0b37228979ed5f8f82d8c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Sat, 23 Mar 2019 00:10:50 +0300 Subject: Fix text being nullable in TwitterAPI --- lib/pleroma/web/twitter_api/views/activity_view.ex | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index fe7d49975..aa1d41fa2 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -266,6 +266,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do content |> String.replace(~r//, "\n") |> HTML.get_cached_stripped_html_for_object(activity, __MODULE__) + else + "" end reply_parent = Activity.get_in_reply_to_activity(activity) -- cgit v1.2.3 From 62bccddde02edbf825c1806805cc9d7d7c9a0f13 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Fri, 22 Mar 2019 23:34:47 +0000 Subject: object: add support for preloading objects when walking an activity graph in normal form --- lib/pleroma/activity.ex | 62 ++++++++++++++++++++++++++++++++++++++++++++++++- lib/pleroma/object.ex | 25 ++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 04c3bb673..2f91b3c77 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Activity do alias Pleroma.Activity alias Pleroma.Notification + alias Pleroma.Object alias Pleroma.Repo import Ecto.Query @@ -33,6 +34,18 @@ defmodule Pleroma.Activity do field(:recipients, {:array, :string}) has_many(:notifications, Notification, on_delete: :delete_all) + # Attention: this is a fake relation, don't try to preload it blindly and expect it to work! + # The foreign key is embedded in a jsonb field. + # + # To use it, you probably want to do an inner join and a preload: + # + # ``` + # |> join(:inner, [activity], o in Object, + # fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", o.data, activity.data)) + # |> preload([activity, object], [object: object]) + # ``` + has_one(:object, Object, on_delete: :nothing, foreign_key: :id) + timestamps() end @@ -49,6 +62,21 @@ defmodule Pleroma.Activity do Repo.get(Activity, id) end + def get_by_id_with_object(id) do + from(activity in Activity, + where: activity.id == ^id, + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ), + preload: [object: o] + ) + |> Repo.one() + end + def by_object_ap_id(ap_id) do from( activity in Activity, @@ -76,7 +104,7 @@ defmodule Pleroma.Activity do ) end - def create_by_object_ap_id(ap_id) do + def create_by_object_ap_id(ap_id) when is_binary(ap_id) do from( activity in Activity, where: @@ -90,6 +118,8 @@ defmodule Pleroma.Activity do ) end + def create_by_object_ap_id(_), do: nil + def get_all_create_by_object_ap_id(ap_id) do Repo.all(create_by_object_ap_id(ap_id)) end @@ -101,6 +131,36 @@ defmodule Pleroma.Activity do def get_create_by_object_ap_id(_), do: nil + def create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do + from( + activity in Activity, + where: + fragment( + "coalesce((?)->'object'->>'id', (?)->>'object') = ?", + activity.data, + activity.data, + ^to_string(ap_id) + ), + where: fragment("(?)->>'type' = 'Create'", activity.data), + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ), + preload: [object: o] + ) + end + + def create_by_object_ap_id_with_object(_), do: nil + + def get_create_by_object_ap_id_with_object(ap_id) do + ap_id + |> create_by_object_ap_id_with_object() + |> Repo.one() + end + def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"]) def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id) def normalize(_), do: nil diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 58e46ef1d..0f5c532ec 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -14,6 +14,8 @@ defmodule Pleroma.Object do import Ecto.Query import Ecto.Changeset + require Logger + schema "objects" do field(:data, :map) @@ -38,6 +40,29 @@ defmodule Pleroma.Object do Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id))) end + # If we pass an Activity to Object.normalize(), we can try to use the preloaded object. + # Use this whenever possible, especially when walking graphs in an O(N) loop! + def normalize(%Activity{object: %Object{} = object}), do: object + + # Catch and log Object.normalize() calls where the Activity's child object is not + # preloaded. + def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do + Logger.info( + "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" + ) + + normalize(ap_id) + end + + def normalize(%Activity{data: %{"object" => ap_id}}) do + Logger.info( + "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" + ) + + normalize(ap_id) + end + + # Old way, try fetching the object through cache. def normalize(%{"id" => ap_id}), do: normalize(ap_id) def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id) def normalize(_), do: nil -- cgit v1.2.3 From 092cedede507b3e89134deb6f5d09c5db339dfae Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:09:56 +0000 Subject: activity: add with_preloaded_object() convenience --- lib/pleroma/activity.ex | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 2f91b3c77..370721070 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -44,11 +44,29 @@ defmodule Pleroma.Activity do # fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", o.data, activity.data)) # |> preload([activity, object], [object: object]) # ``` + # + # As a convenience, Activity.with_preloaded_object() sets up an inner join and preload for the + # typical case. has_one(:object, Object, on_delete: :nothing, foreign_key: :id) timestamps() end + def with_preloaded_object(query) do + query + |> join( + :inner, + [activity], + o in Object, + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ) + ) + |> preload([activity, object], object: object) + end + def get_by_ap_id(ap_id) do Repo.one( from( -- cgit v1.2.3 From 9aea7cc224c09e37a9a46277a9fbfb0af383fc0e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:10:17 +0000 Subject: activitypub: preload child objects when fetching timelines --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 2470b4a71..7d38a46e5 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -716,6 +716,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do limit: 20, order_by: [fragment("? desc nulls last", activity.id)] ) + |> Activity.with_preloaded_object() base_query |> restrict_recipients(recipients, opts["user"]) -- cgit v1.2.3 From e75e43b949ab6f6fb5751eeff10644d04259c149 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:22:14 +0000 Subject: common api: use the optimized Object.normalize whenever possible --- lib/pleroma/web/common_api/common_api.ex | 13 ++++++------- lib/pleroma/web/common_api/utils.ex | 8 ++++---- 2 files changed, 10 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 50d60aade..8625f6621 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -6,7 +6,6 @@ defmodule Pleroma.Web.CommonAPI do alias Pleroma.Activity alias Pleroma.Formatter alias Pleroma.Object - alias Pleroma.Repo alias Pleroma.ThreadMute alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub @@ -64,8 +63,8 @@ defmodule Pleroma.Web.CommonAPI do end def delete(activity_id, user) do - with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id), - %Object{} = object <- Object.normalize(object_id), + with %Activity{data: %{"object" => _}} = activity <- Activity.get_by_id_with_object(activity_id), + %Object{} = object <- Object.normalize(activity), true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, _} <- unpin(activity_id, user), {:ok, delete} <- ActivityPub.delete(object) do @@ -75,7 +74,7 @@ defmodule Pleroma.Web.CommonAPI do def repeat(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]), + object <- Object.normalize(activity), nil <- Utils.get_existing_announce(user.ap_id, object) do ActivityPub.announce(user, object) else @@ -86,7 +85,7 @@ defmodule Pleroma.Web.CommonAPI do def unrepeat(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity) do ActivityPub.unannounce(user, object) else _ -> @@ -96,7 +95,7 @@ defmodule Pleroma.Web.CommonAPI do def favorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]), + object <- Object.normalize(activity), nil <- Utils.get_existing_like(user.ap_id, object) do ActivityPub.like(user, object) else @@ -107,7 +106,7 @@ defmodule Pleroma.Web.CommonAPI do def unfavorite(id_or_ap_id, user) do with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id), - object <- Object.normalize(activity.data["object"]["id"]) do + object <- Object.normalize(activity) do ActivityPub.unlike(user, object) else _ -> diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 3e807a5b7..6a94c2d26 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -17,13 +17,13 @@ defmodule Pleroma.Web.CommonAPI.Utils do # This is a hack for twidere. def get_by_id_or_ap_id(id) do - activity = Repo.get(Activity, id) || Activity.get_create_by_object_ap_id(id) + activity = Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id) activity && if activity.data["type"] == "Create" do activity else - Activity.get_create_by_object_ap_id(activity.data["object"]) + Activity.get_create_by_object_ap_id_with_object(activity.data["object"]) end end @@ -302,10 +302,10 @@ defmodule Pleroma.Web.CommonAPI.Utils do def maybe_notify_mentioned_recipients( recipients, - %Activity{data: %{"to" => _to, "type" => type} = data} = _activity + %Activity{data: %{"to" => _to, "type" => type} = data} = activity ) when type == "Create" do - object = Object.normalize(data["object"]) + object = Object.normalize(activity) object_data = cond do -- cgit v1.2.3 From b3bf523c0967e27616b78a42ba50a6b4c432749e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:22:57 +0000 Subject: rich media: use optimized Object.normalize() --- lib/pleroma/web/rich_media/helpers.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 92c61ff51..f97d1863c 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -23,7 +23,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do def fetch_data_for_activity(%Activity{} = activity) do with true <- Pleroma.Config.get([:rich_media, :enabled]), - %Object{} = object <- Object.normalize(activity.data["object"]), + %Object{} = object <- Object.normalize(activity), {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), :ok <- validate_page_url(page_url), {:ok, rich_media} <- Parser.parse(page_url) do -- cgit v1.2.3 From 59518cbcd8bcc0fe98f2d05ef88d3e2ff68a256f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:24:23 +0000 Subject: activity: fix credo nitpick --- lib/pleroma/activity.ex | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 370721070..f041d0e53 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -41,7 +41,8 @@ defmodule Pleroma.Activity do # # ``` # |> join(:inner, [activity], o in Object, - # fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", o.data, activity.data)) + # on: fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + # o.data, activity.data)) # |> preload([activity, object], [object: object]) # ``` # @@ -58,7 +59,7 @@ defmodule Pleroma.Activity do :inner, [activity], o in Object, - fragment( + on: fragment( "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", o.data, activity.data -- cgit v1.2.3 From a6973a668e40645f9c0940a4fb5aeee45003b66f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:28:16 +0000 Subject: formatting --- lib/pleroma/activity.ex | 11 ++++++----- lib/pleroma/web/common_api/common_api.ex | 3 ++- lib/pleroma/web/common_api/utils.ex | 3 ++- 3 files changed, 10 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index f041d0e53..a762d66ef 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -59,11 +59,12 @@ defmodule Pleroma.Activity do :inner, [activity], o in Object, - on: fragment( - "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", - o.data, - activity.data - ) + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ) ) |> preload([activity, object], object: object) end diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index 8625f6621..25b990677 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -63,7 +63,8 @@ defmodule Pleroma.Web.CommonAPI do end def delete(activity_id, user) do - with %Activity{data: %{"object" => _}} = activity <- Activity.get_by_id_with_object(activity_id), + with %Activity{data: %{"object" => _}} = activity <- + Activity.get_by_id_with_object(activity_id), %Object{} = object <- Object.normalize(activity), true <- User.superuser?(user) || user.ap_id == object.data["actor"], {:ok, _} <- unpin(activity_id, user), diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index 6a94c2d26..f596f703b 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -17,7 +17,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do # This is a hack for twidere. def get_by_id_or_ap_id(id) do - activity = Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id) + activity = + Activity.get_by_id_with_object(id) || Activity.get_create_by_object_ap_id_with_object(id) activity && if activity.data["type"] == "Create" do -- cgit v1.2.3 From e4307cadc86018914cbe1298bcd06299e81006e5 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:40:08 +0000 Subject: activitypub: splice in the child object if we have one --- lib/pleroma/web/activity_pub/activity_pub.ex | 10 +++++++++- lib/pleroma/web/activity_pub/utils.ex | 6 +++--- 2 files changed, 12 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 7d38a46e5..9cca7ec6f 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -95,7 +95,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do :ok <- check_actor_is_active(map["actor"]), {_, true} <- {:remote_limit_error, check_remote_limit(map)}, {:ok, map} <- MRF.filter(map), - :ok <- insert_full_object(map) do + {:ok, object} <- insert_full_object(map) do {recipients, _, _} = get_recipients(map) {:ok, activity} = @@ -106,6 +106,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do recipients: recipients }) + # Splice in the child object if we have one. + activity = + if !is_nil(object) do + Map.put(activity, :object, object) + else + activity + end + Task.start(fn -> Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) end) diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex index af317245f..2e9ffe41c 100644 --- a/lib/pleroma/web/activity_pub/utils.ex +++ b/lib/pleroma/web/activity_pub/utils.ex @@ -209,12 +209,12 @@ defmodule Pleroma.Web.ActivityPub.Utils do """ def insert_full_object(%{"object" => %{"type" => type} = object_data}) when is_map(object_data) and type in @supported_object_types do - with {:ok, _} <- Object.create(object_data) do - :ok + with {:ok, object} <- Object.create(object_data) do + {:ok, object} end end - def insert_full_object(_), do: :ok + def insert_full_object(_), do: {:ok, nil} def update_object_in_activities(%{data: %{"id" => id}} = object) do # TODO -- cgit v1.2.3 From ba7299fc875adc95102ddb1332bec2e6e89b6155 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 00:53:35 +0000 Subject: activitypub: add missing with_preloaded_object() --- lib/pleroma/web/activity_pub/activity_pub.ex | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 9cca7ec6f..95bbadec1 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -438,6 +438,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do ), order_by: [desc: :id] ) + |> Activity.with_preloaded_object() Repo.all(query) end -- cgit v1.2.3 From 73efe95368dfc910c965e4025f47b36b6eb37aaa Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 01:09:12 +0000 Subject: activitypub: allow skipping preload in some cases (like certain tests where the preload is obnoxious) --- lib/pleroma/web/activity_pub/activity_pub.ex | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 95bbadec1..0441376e3 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -718,6 +718,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do defp restrict_muted_reblogs(query, _), do: query + defp maybe_preload_objects(query, %{"skip_preload" => true}), do: query + + defp maybe_preload_objects(query, _) do + query + |> Activity.with_preloaded_object() + end + def fetch_activities_query(recipients, opts \\ %{}) do base_query = from( @@ -725,9 +732,9 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do limit: 20, order_by: [fragment("? desc nulls last", activity.id)] ) - |> Activity.with_preloaded_object() base_query + |> maybe_preload_objects(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) |> restrict_tag_reject(opts) -- cgit v1.2.3 From e430a71d373a15b105a52771551b0ec4ed436ba4 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 01:17:26 +0000 Subject: ostatus: fetch preloaded object in note handler for testsuite --- lib/pleroma/web/ostatus/handlers/note_handler.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex index 770a71a0a..db995ec77 100644 --- a/lib/pleroma/web/ostatus/handlers/note_handler.ex +++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex @@ -106,7 +106,7 @@ defmodule Pleroma.Web.OStatus.NoteHandler do # TODO: Clean this up a bit. def handle_note(entry, doc \\ nil) do with id <- XML.string_from_xpath("//id", entry), - activity when is_nil(activity) <- Activity.get_create_by_object_ap_id(id), + activity when is_nil(activity) <- Activity.get_create_by_object_ap_id_with_object(id), [author] <- :xmerl_xpath.string('//author[1]', doc), {:ok, actor} <- OStatus.find_make_or_update_user(author), content_html <- OStatus.get_content(entry), -- cgit v1.2.3 From 4cedf454236c73b9337e58f5bf0e20ae65c65313 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 01:33:41 +0000 Subject: relay: use preloaded object since we always have it --- lib/pleroma/web/activity_pub/relay.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex index 01fef71b9..a7a20ca37 100644 --- a/lib/pleroma/web/activity_pub/relay.ex +++ b/lib/pleroma/web/activity_pub/relay.ex @@ -41,7 +41,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do def publish(%Activity{data: %{"type" => "Create"}} = activity) do with %User{} = user <- get_actor(), - %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do + %Object{} = object <- Object.normalize(activity) do ActivityPub.announce(user, object, nil, true, false) else e -> Logger.error("error: #{inspect(e)}") -- cgit v1.2.3 From 9a06d9f6e87c948997b908456ff8838e6110757e Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:19:34 +0000 Subject: notification: preload child objects --- lib/pleroma/notification.ex | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index a98649b63..ef8e0b4c1 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Notification do alias Pleroma.Activity alias Pleroma.Notification + alias Pleroma.Object alias Pleroma.Pagination alias Pleroma.Repo alias Pleroma.User @@ -33,7 +34,10 @@ defmodule Pleroma.Notification do Notification |> where(user_id: ^user.id) |> join(:inner, [n], activity in assoc(n, :activity)) - |> preload(:activity) + |> join(:left, [n, a], object in Object, + on: fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", object.data, a.data) + ) + |> preload([n, a, o], activity: {a, object: o}) end def for_user(user, opts \\ %{}) do -- cgit v1.2.3 From c62220c50078c648c33d7aad7a615b9c7e1f27c6 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:26:49 +0000 Subject: rich media: helpers: only crawl Create activities --- lib/pleroma/web/rich_media/helpers.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index f97d1863c..f67aaf58b 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -21,7 +21,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do defp validate_page_url(%URI{}), do: :ok defp validate_page_url(_), do: :error - def fetch_data_for_activity(%Activity{} = activity) do + def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do with true <- Pleroma.Config.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity), {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), @@ -32,4 +32,6 @@ defmodule Pleroma.Web.RichMedia.Helpers do _ -> %{} end end + + def fetch_data_for_activity(_), do: %{} end -- cgit v1.2.3 From 07cdd9ed02838069b58c3b6cf9aec73b1d1852c3 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:27:52 +0000 Subject: streamer: use the preloaded object if possible --- lib/pleroma/web/streamer.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/streamer.ex b/lib/pleroma/web/streamer.ex index 7425bfb54..592749b42 100644 --- a/lib/pleroma/web/streamer.ex +++ b/lib/pleroma/web/streamer.ex @@ -202,7 +202,7 @@ defmodule Pleroma.Web.Streamer do mutes = user.info.mutes || [] reblog_mutes = user.info.muted_reblogs || [] - parent = Object.normalize(item.data["object"]) + parent = Object.normalize(item) unless is_nil(parent) or item.actor in blocks or item.actor in mutes or item.actor in reblog_mutes or not ActivityPub.contain_activity(item, user) or -- cgit v1.2.3 From aabcecb26935475b5985b6438774ee1d3cc14514 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:30:53 +0000 Subject: notification: formatting --- lib/pleroma/notification.ex | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/notification.ex b/lib/pleroma/notification.ex index ef8e0b4c1..cac10f24a 100644 --- a/lib/pleroma/notification.ex +++ b/lib/pleroma/notification.ex @@ -35,7 +35,12 @@ defmodule Pleroma.Notification do |> where(user_id: ^user.id) |> join(:inner, [n], activity in assoc(n, :activity)) |> join(:left, [n, a], object in Object, - on: fragment("(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", object.data, a.data) + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + object.data, + a.data + ) ) |> preload([n, a, o], activity: {a, object: o}) end -- cgit v1.2.3 From ce47eb8b29aaba8f59720597485d3c586fbb7831 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:38:59 +0000 Subject: activitypub: when fetching objects, use the preloaded object from the synthesized activity --- lib/pleroma/web/activity_pub/activity_pub.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 0441376e3..80c64ae04 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -957,7 +957,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do }, :ok <- Transmogrifier.contain_origin(id, params), {:ok, activity} <- Transmogrifier.handle_incoming(params) do - {:ok, Object.normalize(activity.data["object"])} + {:ok, Object.normalize(activity)} else {:error, {:reject, nil}} -> {:reject, nil} @@ -969,7 +969,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do Logger.info("Couldn't get object via AP, trying out OStatus fetching...") case OStatus.fetch_activity_from_url(id) do - {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])} + {:ok, [activity | _]} -> {:ok, Object.normalize(activity)} e -> e end end -- cgit v1.2.3 From f9d5c13b21905b6839f489d4c8d75bc1e95d3875 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:49:10 +0000 Subject: activity: add get_by_ap_id_with_object() --- lib/pleroma/activity.ex | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index a762d66ef..fcdac3a3f 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -78,6 +78,23 @@ defmodule Pleroma.Activity do ) end + def get_by_ap_id_with_object(ap_id) do + Repo.one( + from( + activity in Activity, + where: fragment("(?)->>'id' = ?", activity.data, ^to_string(ap_id)), + inner_join: o in Object, + on: + fragment( + "(?->>'id') = COALESCE((? -> 'object'::text) ->> 'id'::text)", + o.data, + activity.data + ), + preload: [object: o] + ) + ) + end + def get_by_id(id) do Repo.get(Activity, id) end @@ -181,8 +198,8 @@ defmodule Pleroma.Activity do |> Repo.one() end - def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"]) - def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id) + def normalize(obj) when is_map(obj), do: get_by_ap_id_with_object(obj["id"]) + def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id_with_object(ap_id) def normalize(_), do: nil def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do -- cgit v1.2.3 From 8c70156157fcb2e0091c21e566324bcb24886f6f Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 02:49:40 +0000 Subject: activitypub: object view: use preloaded object when possible --- lib/pleroma/web/activity_pub/views/object_view.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/views/object_view.ex b/lib/pleroma/web/activity_pub/views/object_view.ex index 84fa94e32..6028b773c 100644 --- a/lib/pleroma/web/activity_pub/views/object_view.ex +++ b/lib/pleroma/web/activity_pub/views/object_view.ex @@ -17,7 +17,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do def render("object.json", %{object: %Activity{data: %{"type" => "Create"}} = activity}) do base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header() - object = Object.normalize(activity.data["object"]) + object = Object.normalize(activity) additional = Transmogrifier.prepare_object(activity.data) @@ -28,7 +28,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectView do def render("object.json", %{object: %Activity{} = activity}) do base = Pleroma.Web.ActivityPub.Utils.make_json_ld_header() - object = Object.normalize(activity.data["object"]) + object = Object.normalize(activity) additional = Transmogrifier.prepare_object(activity.data) -- cgit v1.2.3 From 3c2350cbee95df3b6a31945d54bf7c46a8afff25 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 03:00:04 +0000 Subject: object: downgrade normalize warning to debug severity --- lib/pleroma/object.ex | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex index 0f5c532ec..193ae3fa8 100644 --- a/lib/pleroma/object.ex +++ b/lib/pleroma/object.ex @@ -47,18 +47,22 @@ defmodule Pleroma.Object do # Catch and log Object.normalize() calls where the Activity's child object is not # preloaded. def normalize(%Activity{data: %{"object" => %{"id" => ap_id}}}) do - Logger.info( + Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) + Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}") + normalize(ap_id) end def normalize(%Activity{data: %{"object" => ap_id}}) do - Logger.info( + Logger.debug( "Object.normalize() called without preloaded object (#{ap_id}). Consider preloading the object!" ) + Logger.debug("Backtrace: #{inspect(Process.info(:erlang.self(), :current_stacktrace))}") + normalize(ap_id) end -- cgit v1.2.3 From debf7f016d5f09b5878ddb24051a28ddb2cf1e4a Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Sat, 23 Mar 2019 03:03:06 +0000 Subject: ostatus: use preload objects with Object.normalize() when opportunistic --- lib/pleroma/web/ostatus/ostatus.ex | 10 +++++----- lib/pleroma/web/ostatus/ostatus_controller.ex | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/ostatus/ostatus.ex b/lib/pleroma/web/ostatus/ostatus.ex index 266f86bf4..9a34d7ad5 100644 --- a/lib/pleroma/web/ostatus/ostatus.ex +++ b/lib/pleroma/web/ostatus/ostatus.ex @@ -23,8 +23,8 @@ defmodule Pleroma.Web.OStatus do alias Pleroma.Web.WebFinger alias Pleroma.Web.Websub - def is_representable?(%Activity{data: data}) do - object = Object.normalize(data["object"]) + def is_representable?(%Activity{} = activity) do + object = Object.normalize(activity) cond do is_nil(object) -> @@ -119,7 +119,7 @@ defmodule Pleroma.Web.OStatus do def make_share(entry, doc, retweeted_activity) do with {:ok, actor} <- find_make_or_update_user(doc), - %Object{} = object <- Object.normalize(retweeted_activity.data["object"]), + %Object{} = object <- Object.normalize(retweeted_activity), id when not is_nil(id) <- string_from_xpath("/entry/id", entry), {:ok, activity, _object} = ActivityPub.announce(actor, object, id, false) do {:ok, activity} @@ -137,7 +137,7 @@ defmodule Pleroma.Web.OStatus do def make_favorite(entry, doc, favorited_activity) do with {:ok, actor} <- find_make_or_update_user(doc), - %Object{} = object <- Object.normalize(favorited_activity.data["object"]), + %Object{} = object <- Object.normalize(favorited_activity), id when not is_nil(id) <- string_from_xpath("/entry/id", entry), {:ok, activity, _object} = ActivityPub.like(actor, object, id, false) do {:ok, activity} @@ -159,7 +159,7 @@ defmodule Pleroma.Web.OStatus do Logger.debug("Trying to get entry from db") with id when not is_nil(id) <- string_from_xpath("//activity:object[1]/id", entry), - %Activity{} = activity <- Activity.get_create_by_object_ap_id(id) do + %Activity{} = activity <- Activity.get_create_by_object_ap_id_with_object(id) do {:ok, activity} else _ -> diff --git a/lib/pleroma/web/ostatus/ostatus_controller.ex b/lib/pleroma/web/ostatus/ostatus_controller.ex index 0579a5f3d..2fb6ce41b 100644 --- a/lib/pleroma/web/ostatus/ostatus_controller.ex +++ b/lib/pleroma/web/ostatus/ostatus_controller.ex @@ -102,7 +102,8 @@ defmodule Pleroma.Web.OStatus.OStatusController do ActivityPubController.call(conn, :object) else with id <- o_status_url(conn, :object, uuid), - {_, %Activity{} = activity} <- {:activity, Activity.get_create_by_object_ap_id(id)}, + {_, %Activity{} = activity} <- + {:activity, Activity.get_create_by_object_ap_id_with_object(id)}, {_, true} <- {:public?, Visibility.is_public?(activity)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case get_format(conn) do @@ -148,13 +149,13 @@ defmodule Pleroma.Web.OStatus.OStatusController do end def notice(conn, %{"id" => id}) do - with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id(id)}, + with {_, %Activity{} = activity} <- {:activity, Activity.get_by_id_with_object(id)}, {_, true} <- {:public?, Visibility.is_public?(activity)}, %User{} = user <- User.get_cached_by_ap_id(activity.data["actor"]) do case format = get_format(conn) do "html" -> if activity.data["type"] == "Create" do - %Object{} = object = Object.normalize(activity.data["object"]) + %Object{} = object = Object.normalize(activity) Fallback.RedirectController.redirector_with_meta(conn, %{ activity_id: activity.id, @@ -191,9 +192,9 @@ defmodule Pleroma.Web.OStatus.OStatusController do # Returns an HTML embedded