summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/mix/tasks/pleroma/database.ex64
-rw-r--r--lib/pleroma/activity/ir/topics.ex10
-rw-r--r--lib/pleroma/hashtag.ex58
-rw-r--r--lib/pleroma/object.ex54
-rw-r--r--lib/pleroma/web/activity_pub/activity_pub.ex167
-rw-r--r--lib/pleroma/web/activity_pub/mrf/simple_policy.ex8
-rw-r--r--lib/pleroma/web/activity_pub/transmogrifier.ex29
-rw-r--r--lib/pleroma/web/feed/feed_view.ex1
-rw-r--r--lib/pleroma/web/mastodon_api/views/status_view.ex6
-rw-r--r--lib/pleroma/web/templates/feed/feed/_activity.atom.eex2
-rw-r--r--lib/pleroma/web/templates/feed/feed/_activity.rss.eex2
-rw-r--r--lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex2
12 files changed, 332 insertions, 71 deletions
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index 22151ce08..093c7dd30 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -4,14 +4,18 @@
defmodule Mix.Tasks.Pleroma.Database do
alias Pleroma.Conversation
+ alias Pleroma.Hashtag
alias Pleroma.Maintenance
alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User
+
require Logger
require Pleroma.Constants
+
import Ecto.Query
import Mix.Pleroma
+
use Mix.Task
@shortdoc "A collection of database related tasks"
@@ -128,6 +132,66 @@ defmodule Mix.Tasks.Pleroma.Database do
|> Stream.run()
end
+ def run(["transfer_hashtags"]) do
+ import Ecto.Query
+
+ start_pleroma()
+
+ from(
+ object in Object,
+ left_join: hashtag in assoc(object, :hashtags),
+ where: is_nil(hashtag.id),
+ where: fragment("(?)->>'tag' != '[]'", object.data),
+ select: %{
+ id: object.id,
+ inserted_at: object.inserted_at,
+ tag: fragment("(?)->>'tag'", object.data)
+ },
+ order_by: [desc: object.id]
+ )
+ |> Pleroma.Repo.chunk_stream(100, :batches)
+ |> Stream.each(fn objects ->
+ chunk_start = List.first(objects)
+ chunk_end = List.last(objects)
+
+ Logger.info(
+ "transfer_hashtags: " <>
+ "#{chunk_start.id} (#{chunk_start.inserted_at}) -- " <>
+ "#{chunk_end.id} (#{chunk_end.inserted_at})"
+ )
+
+ Enum.map(
+ objects,
+ fn object ->
+ hashtags =
+ object.tag
+ |> Jason.decode!()
+ |> Enum.filter(&is_bitstring(&1))
+
+ with {:ok, hashtag_records} <- Hashtag.get_or_create_by_names(hashtags) do
+ Repo.transaction(fn ->
+ for hashtag_record <- hashtag_records do
+ with {:error, _} <-
+ Ecto.Adapters.SQL.query(
+ Repo,
+ "insert into hashtags_objects(hashtag_id, object_id) values " <>
+ "(#{hashtag_record.id}, #{object.id});"
+ ) do
+ Logger.warn(
+ "ERROR: could not link object #{object.id} and hashtag #{hashtag_record.id}"
+ )
+ end
+ end
+ end)
+ else
+ e -> Logger.warn("ERROR: could not process object #{object.id}: #{inspect(e)}")
+ end
+ end
+ )
+ end)
+ |> Stream.run()
+ end
+
def run(["vacuum", args]) do
start_pleroma()
diff --git a/lib/pleroma/activity/ir/topics.ex b/lib/pleroma/activity/ir/topics.ex
index fe2e8cb5c..2c74ac2bf 100644
--- a/lib/pleroma/activity/ir/topics.ex
+++ b/lib/pleroma/activity/ir/topics.ex
@@ -48,14 +48,12 @@ defmodule Pleroma.Activity.Ir.Topics do
tags
end
- defp hashtags_to_topics(%{data: %{"tag" => tags}}) do
- tags
- |> Enum.filter(&is_bitstring(&1))
- |> Enum.map(fn tag -> "hashtag:" <> tag end)
+ defp hashtags_to_topics(object) do
+ object
+ |> Object.hashtags()
+ |> Enum.map(fn hashtag -> "hashtag:" <> hashtag end)
end
- defp hashtags_to_topics(_), do: []
-
defp remote_topics(%{local: true}), do: []
defp remote_topics(%{actor: actor}) when is_binary(actor),
diff --git a/lib/pleroma/hashtag.ex b/lib/pleroma/hashtag.ex
new file mode 100644
index 000000000..b05927563
--- /dev/null
+++ b/lib/pleroma/hashtag.ex
@@ -0,0 +1,58 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2020 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Hashtag do
+ use Ecto.Schema
+
+ import Ecto.Changeset
+
+ alias Pleroma.Hashtag
+ alias Pleroma.Repo
+
+ @derive {Jason.Encoder, only: [:data]}
+
+ schema "hashtags" do
+ field(:name, :string)
+ field(:data, :map, default: %{})
+
+ many_to_many(:objects, Pleroma.Object, join_through: "hashtags_objects", on_replace: :delete)
+
+ timestamps()
+ end
+
+ def get_by_name(name) do
+ Repo.get_by(Hashtag, name: name)
+ end
+
+ def get_or_create_by_name(name) when is_bitstring(name) do
+ with %Hashtag{} = hashtag <- get_by_name(name) do
+ {:ok, hashtag}
+ else
+ _ ->
+ %Hashtag{}
+ |> changeset(%{name: name})
+ |> Repo.insert()
+ end
+ end
+
+ def get_or_create_by_names(names) when is_list(names) do
+ Enum.reduce_while(names, {:ok, []}, fn name, {:ok, list} ->
+ case get_or_create_by_name(name) do
+ {:ok, %Hashtag{} = hashtag} ->
+ {:cont, {:ok, list ++ [hashtag]}}
+
+ error ->
+ {:halt, error}
+ end
+ end)
+ end
+
+ def changeset(%Hashtag{} = struct, params) do
+ struct
+ |> cast(params, [:name, :data])
+ |> update_change(:name, &String.downcase/1)
+ |> validate_required([:name])
+ |> unique_constraint(:name)
+ end
+end
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index b4a994da9..1d756bcd1 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -10,6 +10,7 @@ defmodule Pleroma.Object do
alias Pleroma.Activity
alias Pleroma.Config
+ alias Pleroma.Hashtag
alias Pleroma.Object
alias Pleroma.Object.Fetcher
alias Pleroma.ObjectTombstone
@@ -28,6 +29,8 @@ defmodule Pleroma.Object do
schema "objects" do
field(:data, :map)
+ many_to_many(:hashtags, Hashtag, join_through: "hashtags_objects", on_replace: :delete)
+
timestamps()
end
@@ -49,7 +52,8 @@ defmodule Pleroma.Object do
end
def create(data) do
- Object.change(%Object{}, %{data: data})
+ %Object{}
+ |> Object.change(%{data: data})
|> Repo.insert()
end
@@ -58,8 +62,37 @@ defmodule Pleroma.Object do
|> cast(params, [:data])
|> validate_required([:data])
|> unique_constraint(:ap_id, name: :objects_unique_apid_index)
+ |> maybe_handle_hashtags_change(struct)
+ end
+
+ defp maybe_handle_hashtags_change(changeset, struct) do
+ with data_hashtags_change = get_change(changeset, :data),
+ true <- hashtags_changed?(struct, data_hashtags_change),
+ {:ok, hashtag_records} <-
+ data_hashtags_change
+ |> object_data_hashtags()
+ |> Hashtag.get_or_create_by_names() do
+ put_assoc(changeset, :hashtags, hashtag_records)
+ else
+ false ->
+ changeset
+
+ {:error, hashtag_changeset} ->
+ failed_hashtag = get_field(hashtag_changeset, :name)
+
+ validate_change(changeset, :data, fn _, _ ->
+ [data: "error referencing hashtag: #{failed_hashtag}"]
+ end)
+ end
+ end
+
+ defp hashtags_changed?(%Object{} = struct, %{"tag" => _} = data) do
+ Enum.sort(embedded_hashtags(struct)) !=
+ Enum.sort(object_data_hashtags(data))
end
+ defp hashtags_changed?(_, _), do: false
+
def get_by_id(nil), do: nil
def get_by_id(id), do: Repo.get(Object, id)
@@ -346,4 +379,23 @@ defmodule Pleroma.Object do
def self_replies(object, opts \\ []),
do: replies(object, Keyword.put(opts, :self_only, true))
+
+ def tags(%Object{data: %{"tag" => tags}}) when is_list(tags), do: tags
+
+ def tags(_), do: []
+
+ def hashtags(object), do: embedded_hashtags(object)
+
+ defp embedded_hashtags(%Object{data: data}) do
+ object_data_hashtags(data)
+ end
+
+ defp embedded_hashtags(_), do: []
+
+ defp object_data_hashtags(%{"tag" => tags}) when is_list(tags) do
+ # Note: AS2 map-type elements are ignored
+ Enum.filter(tags, &is_bitstring(&1))
+ end
+
+ defp object_data_hashtags(_), do: []
end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 5059bff03..54d1a2350 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -663,33 +663,41 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
defp restrict_since(query, _), do: query
defp restrict_tag_reject(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ raise_on_missing_preload()
end
- defp restrict_tag_reject(query, %{tag_reject: [_ | _] = tag_reject}) do
+ defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_list(tag_reject) do
from(
[_activity, object] in query,
where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
)
end
+ defp restrict_tag_reject(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+ restrict_tag_reject(query, %{tag_reject: [tag_reject]})
+ end
+
defp restrict_tag_reject(query, _), do: query
defp restrict_tag_all(_query, %{tag_all: _tag_all, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ raise_on_missing_preload()
end
- defp restrict_tag_all(query, %{tag_all: [_ | _] = tag_all}) do
+ defp restrict_tag_all(query, %{tag_all: tag_all}) when is_list(tag_all) do
from(
[_activity, object] in query,
where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
)
end
+ defp restrict_tag_all(query, %{tag_all: tag}) when is_binary(tag) do
+ restrict_tag(query, %{tag: tag})
+ end
+
defp restrict_tag_all(query, _), do: query
defp restrict_tag(_query, %{tag: _tag, skip_preload: true}) do
- raise "Can't use the child object without preloading!"
+ raise_on_missing_preload()
end
defp restrict_tag(query, %{tag: tag}) when is_list(tag) do
@@ -700,13 +708,79 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
end
defp restrict_tag(query, %{tag: tag}) when is_binary(tag) do
+ restrict_tag(query, %{tag: [tag]})
+ end
+
+ defp restrict_tag(query, _), do: query
+
+ defp restrict_hashtag_reject_any(_query, %{tag_reject: _tag_reject, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_hashtag_reject_any(query, %{tag_reject: tags_reject}) when is_list(tags_reject) do
+ if has_named_binding?(query, :thread_mute) do
+ from(
+ [activity, object, thread_mute] in query,
+ group_by: [activity.id, object.id, thread_mute.id]
+ )
+ else
+ from(
+ [activity, object] in query,
+ group_by: [activity.id, object.id]
+ )
+ end
+ |> join(:left, [_activity, object], hashtag in assoc(object, :hashtags), as: :hashtag)
+ |> having(
+ [hashtag: hashtag],
+ fragment("not(array_agg(?) && (?))", hashtag.name, ^tags_reject)
+ )
+ end
+
+ defp restrict_hashtag_reject_any(query, %{tag_reject: tag_reject}) when is_binary(tag_reject) do
+ restrict_hashtag_reject_any(query, %{tag_reject: [tag_reject]})
+ end
+
+ defp restrict_hashtag_reject_any(query, _), do: query
+
+ defp restrict_hashtag_all(_query, %{tag_all: _tag, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_hashtag_all(query, %{tag_all: tags}) when is_list(tags) do
+ Enum.reduce(
+ tags,
+ query,
+ fn tag, acc -> restrict_hashtag_any(acc, %{tag: tag}) end
+ )
+ end
+
+ defp restrict_hashtag_all(query, %{tag_all: tag}) when is_binary(tag) do
+ restrict_hashtag_any(query, %{tag: tag})
+ end
+
+ defp restrict_hashtag_all(query, _), do: query
+
+ defp restrict_hashtag_any(_query, %{tag: _tag, skip_preload: true}) do
+ raise_on_missing_preload()
+ end
+
+ defp restrict_hashtag_any(query, %{tag: tags}) when is_list(tags) do
from(
[_activity, object] in query,
- where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
+ join: hashtag in assoc(object, :hashtags),
+ where: hashtag.name in ^tags
)
end
- defp restrict_tag(query, _), do: query
+ defp restrict_hashtag_any(query, %{tag: tag}) when is_binary(tag) do
+ restrict_hashtag_any(query, %{tag: [tag]})
+ end
+
+ defp restrict_hashtag_any(query, _), do: query
+
+ defp raise_on_missing_preload do
+ raise "Can't use the child object without preloading!"
+ end
defp restrict_recipients(query, [], _user), do: query
@@ -1091,40 +1165,51 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
skip_thread_containment: Config.get([:instance, :skip_thread_containment])
}
- Activity
- |> maybe_preload_objects(opts)
- |> maybe_preload_bookmarks(opts)
- |> maybe_preload_report_notes(opts)
- |> maybe_set_thread_muted_field(opts)
- |> maybe_order(opts)
- |> restrict_recipients(recipients, opts[:user])
- |> restrict_replies(opts)
- |> restrict_tag(opts)
- |> restrict_tag_reject(opts)
- |> restrict_tag_all(opts)
- |> restrict_since(opts)
- |> restrict_local(opts)
- |> restrict_actor(opts)
- |> restrict_type(opts)
- |> restrict_state(opts)
- |> restrict_favorited_by(opts)
- |> restrict_blocked(restrict_blocked_opts)
- |> restrict_muted(restrict_muted_opts)
- |> restrict_filtered(opts)
- |> restrict_media(opts)
- |> restrict_visibility(opts)
- |> restrict_thread_visibility(opts, config)
- |> restrict_reblogs(opts)
- |> restrict_pinned(opts)
- |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
- |> restrict_instance(opts)
- |> restrict_announce_object_actor(opts)
- |> restrict_filtered(opts)
- |> Activity.restrict_deactivated_users()
- |> exclude_poll_votes(opts)
- |> exclude_chat_messages(opts)
- |> exclude_invisible_actors(opts)
- |> exclude_visibility(opts)
+ query =
+ Activity
+ |> distinct([a], true)
+ |> maybe_preload_objects(opts)
+ |> maybe_preload_bookmarks(opts)
+ |> maybe_preload_report_notes(opts)
+ |> maybe_set_thread_muted_field(opts)
+ |> maybe_order(opts)
+ |> restrict_recipients(recipients, opts[:user])
+ |> restrict_replies(opts)
+ |> restrict_since(opts)
+ |> restrict_local(opts)
+ |> restrict_actor(opts)
+ |> restrict_type(opts)
+ |> restrict_state(opts)
+ |> restrict_favorited_by(opts)
+ |> restrict_blocked(restrict_blocked_opts)
+ |> restrict_muted(restrict_muted_opts)
+ |> restrict_filtered(opts)
+ |> restrict_media(opts)
+ |> restrict_visibility(opts)
+ |> restrict_thread_visibility(opts, config)
+ |> restrict_reblogs(opts)
+ |> restrict_pinned(opts)
+ |> restrict_muted_reblogs(restrict_muted_reblogs_opts)
+ |> restrict_instance(opts)
+ |> restrict_announce_object_actor(opts)
+ |> restrict_filtered(opts)
+ |> Activity.restrict_deactivated_users()
+ |> exclude_poll_votes(opts)
+ |> exclude_chat_messages(opts)
+ |> exclude_invisible_actors(opts)
+ |> exclude_visibility(opts)
+
+ if Config.get([:instance, :improved_hashtag_timeline]) do
+ query
+ |> restrict_hashtag_any(opts)
+ |> restrict_hashtag_all(opts)
+ |> restrict_hashtag_reject_any(opts)
+ else
+ query
+ |> restrict_tag(opts)
+ |> restrict_tag_reject(opts)
+ |> restrict_tag_all(opts)
+ end
end
def fetch_activities(recipients, opts \\ %{}, pagination \\ :keyset) do
diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
index 6cd91826d..e92091d66 100644
--- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
+++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex
@@ -74,9 +74,11 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do
object =
if MRF.subdomain_match?(media_nsfw, actor_host) do
- tags = (child_object["tag"] || []) ++ ["nsfw"]
- child_object = Map.put(child_object, "tag", tags)
- child_object = Map.put(child_object, "sensitive", true)
+ child_object =
+ child_object
+ |> Map.put("tag", (child_object["tag"] || []) ++ ["nsfw"])
+ |> Map.put("sensitive", true)
+
Map.put(object, "object", child_object)
else
object
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 565d32433..fd17793d0 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -32,18 +32,18 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
"""
def fix_object(object, options \\ []) do
object
- |> strip_internal_fields
- |> fix_actor
- |> fix_url
- |> fix_attachments
- |> fix_context
+ |> strip_internal_fields()
+ |> fix_actor()
+ |> fix_url()
+ |> fix_attachments()
+ |> fix_context()
|> fix_in_reply_to(options)
- |> fix_emoji
- |> fix_tag
- |> set_sensitive
- |> fix_content_map
- |> fix_addressing
- |> fix_summary
+ |> fix_emoji()
+ |> fix_tag()
+ |> set_sensitive()
+ |> fix_content_map()
+ |> fix_addressing()
+ |> fix_summary()
|> fix_type(options)
end
@@ -315,10 +315,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
tags =
tag
|> Enum.filter(fn data -> data["type"] == "Hashtag" and data["name"] end)
- |> Enum.map(fn %{"name" => name} ->
- name
- |> String.slice(1..-1)
- |> String.downcase()
+ |> Enum.map(fn
+ %{"name" => "#" <> hashtag} -> String.downcase(hashtag)
+ %{"name" => hashtag} -> String.downcase(hashtag)
end)
Map.put(object, "tag", tag ++ tags)
diff --git a/lib/pleroma/web/feed/feed_view.ex b/lib/pleroma/web/feed/feed_view.ex
index 30e0a2a55..1155c6a39 100644
--- a/lib/pleroma/web/feed/feed_view.ex
+++ b/lib/pleroma/web/feed/feed_view.ex
@@ -32,6 +32,7 @@ defmodule Pleroma.Web.Feed.FeedView do
%{
activity: activity,
+ object: object,
data: Map.get(object, :data),
actor: actor
}
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 2301e21cf..bd08aa203 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -201,8 +201,10 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
like_count = object.data["like_count"] || 0
announcement_count = object.data["announcement_count"] || 0
- tags = object.data["tag"] || []
- sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
+ hashtags = Object.hashtags(object)
+ sensitive = object.data["sensitive"] || Enum.member?(hashtags, "nsfw")
+
+ tags = Object.tags(object)
tag_mentions =
tags
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
index 3fd150c4e..6688830ba 100644
--- a/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
+++ b/lib/pleroma/web/templates/feed/feed/_activity.atom.eex
@@ -22,7 +22,7 @@
<link type="text/html" href='<%= @data["external_url"] %>' rel="alternate"/>
<% end %>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
diff --git a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
index 42960de7d..fc6d74b42 100644
--- a/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
+++ b/lib/pleroma/web/templates/feed/feed/_activity.rss.eex
@@ -21,7 +21,7 @@
<link><%= @data["external_url"] %></link>
<% end %>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
diff --git a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
index cf5874a91..c2de28fe4 100644
--- a/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
+++ b/lib/pleroma/web/templates/feed/feed/_tag_activity.atom.eex
@@ -41,7 +41,7 @@
<% end %>
<% end %>
- <%= for tag <- @data["tag"] || [] do %>
+ <%= for tag <- Pleroma.Object.hashtags(@object) do %>
<category term="<%= tag %>"></category>
<% end %>
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-26 18:59:19 +0000