From 663007b42c185efce41db73854ff1376a5dae0e5 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 18:44:04 +0000
Subject: activity: clean up direct use of object data

---
 lib/pleroma/activity.ex | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index c065f3b6c..3c7e150ee 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -1,6 +1,6 @@
 defmodule Pleroma.Activity do
   use Ecto.Schema
-  alias Pleroma.{Repo, Activity, Notification}
+  alias Pleroma.{Repo, Activity, Notification, Object}
   import Ecto.Query
 
   schema "activities" do
@@ -83,9 +83,13 @@ defmodule Pleroma.Activity do
   def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id)
   def normalize(_), do: nil
 
-  def get_in_reply_to_activity(%Activity{data: %{"object" => %{"inReplyTo" => ap_id}}}) do
+  defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
     get_create_activity_by_object_ap_id(ap_id)
   end
 
-  def get_in_reply_to_activity(_), do: nil
+  defp get_in_reply_to_activity_from_object(_), do: nil
+
+  def get_in_reply_to_activity(%Activity{data: %{"object" => object}}) do
+    get_in_reply_to_activity_from_object(Object.normalize(object))
+  end
 end
-- 
cgit v1.2.3


From 0522b26883534fb93210ece32aed60fcfa4b7a5b Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 18:48:37 +0000
Subject: gopher: use Object.normalize()

---
 lib/pleroma/gopher/server.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/gopher/server.ex b/lib/pleroma/gopher/server.ex
index 3b0569a99..1ab15611c 100644
--- a/lib/pleroma/gopher/server.ex
+++ b/lib/pleroma/gopher/server.ex
@@ -37,6 +37,7 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
   alias Pleroma.Activity
   alias Pleroma.Repo
   alias Pleroma.HTML
+  alias Pleroma.Object
 
   def start_link(ref, socket, transport, opts) do
     pid = spawn_link(__MODULE__, :init, [ref, socket, transport, opts])
@@ -70,14 +71,14 @@ defmodule Pleroma.Gopher.Server.ProtocolHandler do
     |> Enum.map(fn activity ->
       user = User.get_cached_by_ap_id(activity.data["actor"])
 
-      object = activity.data["object"]
+      object = Object.normalize(activity.data["object"])
       like_count = object["like_count"] || 0
       announcement_count = object["announcement_count"] || 0
 
       link("Post ##{activity.id} by #{user.nickname}", "/notices/#{activity.id}") <>
         info("#{like_count} likes, #{announcement_count} repeats") <>
         "i\tfake\t(NULL)\t0\r\n" <>
-        info(HTML.strip_tags(String.replace(activity.data["object"]["content"], "<br>", "\r")))
+        info(HTML.strip_tags(String.replace(object["content"], "<br>", "\r")))
     end)
     |> Enum.join("i\tfake\t(NULL)\t0\r\n")
   end
-- 
cgit v1.2.3


From d6b266163b9161bf28df7919ffd1391c81b142e3 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 18:57:38 +0000
Subject: common api: fetch visibility from normalized object

---
 lib/pleroma/web/common_api/common_api.ex | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index e3385310f..e0c9dedd5 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -64,7 +64,10 @@ defmodule Pleroma.Web.CommonAPI do
         "public"
 
       inReplyTo ->
-        Pleroma.Web.MastodonAPI.StatusView.get_visibility(inReplyTo.data["object"])
+        # XXX: these heuristics should be moved out of MastodonAPI.
+        with %Object{} = object <- Object.normalize(inReplyTo.data["object"]) do
+          Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data)
+        end
     end
   end
 
-- 
cgit v1.2.3


From 67038ae15e1f4a0b136388416af23547d82cbd0d Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 19:00:05 +0000
Subject: common api: utils: access inReplyTo object ID correctly

---
 lib/pleroma/web/common_api/utils.ex | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 728f24c7e..ec66452c2 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -177,8 +177,10 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     }
 
     if inReplyTo do
+      inReplyToObject = Object.normalize(inReplyTo.data["object"])
+
       object
-      |> Map.put("inReplyTo", inReplyTo.data["object"]["id"])
+      |> Map.put("inReplyTo", inReplyToObject.data["id"])
       |> Map.put("inReplyToStatusId", inReplyTo.id)
     else
       object
-- 
cgit v1.2.3


From d3fde9b5f28b802b9150e1f8a763f4a67b42aadb Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 19:05:17 +0000
Subject: ostatus: note handler: appropriately use Object.normalize()

---
 lib/pleroma/web/ostatus/handlers/note_handler.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex
index 0d4080291..39004367a 100644
--- a/lib/pleroma/web/ostatus/handlers/note_handler.ex
+++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex
@@ -106,7 +106,8 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
          cw <- OStatus.get_cw(entry),
          inReplyTo <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
          inReplyToActivity <- fetch_replied_to_activity(entry, inReplyTo),
-         inReplyTo <- (inReplyToActivity && inReplyToActivity.data["object"]["id"]) || inReplyTo,
+         inReplyToObject <- (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil,
+         inReplyTo <- (inReplyToObject && inReplyToObject.data["id"]) || inReplyTo,
          attachments <- OStatus.get_attachments(entry),
          context <- get_context(entry, inReplyTo),
          tags <- OStatus.get_tags(entry),
-- 
cgit v1.2.3


From 4482ce7e2dbd23e3b7948ee8d084c29d84ae5a1e Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 20:09:28 +0000
Subject: activitypub: normalize objects when streaming them out

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 7e207c620..b11d3221f 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -81,6 +81,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
     if activity.data["type"] in ["Create", "Announce"] do
+      object = Object.normalize(activity.data["object"])
+
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
 
@@ -91,12 +93,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
           Pleroma.Web.Streamer.stream("public:local", activity)
         end
 
-        activity.data["object"]
+        object.data
         |> Map.get("tag", [])
         |> Enum.filter(fn tag -> is_bitstring(tag) end)
         |> Enum.map(fn tag -> Pleroma.Web.Streamer.stream("hashtag:" <> tag, activity) end)
 
-        if activity.data["object"]["attachment"] != [] do
+        if object.data["attachment"] != [] do
           Pleroma.Web.Streamer.stream("public:media", activity)
 
           if activity.local do
-- 
cgit v1.2.3


From 95a458f3925fec4ae31a59face1bc130abdcde28 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 20:14:00 +0000
Subject: twitterapi: more object normalization work

---
 .../representers/activity_representer.ex           | 34 ++++++++++++----------
 lib/pleroma/web/twitter_api/views/activity_view.ex |  6 ++--
 2 files changed, 21 insertions(+), 19 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
index fbd33f07e..4b4e202b3 100644
--- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
@@ -3,7 +3,7 @@
 defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
   use Pleroma.Web.TwitterAPI.Representers.BaseRepresenter
   alias Pleroma.Web.TwitterAPI.Representers.ObjectRepresenter
-  alias Pleroma.{Activity, User}
+  alias Pleroma.{Activity, User, Object}
   alias Pleroma.Web.TwitterAPI.{TwitterAPI, UserView, ActivityView}
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Formatter
@@ -144,11 +144,13 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
         %Activity{data: %{"object" => %{"content" => content} = object}} = activity,
         %{user: user} = opts
       ) do
-    created_at = object["published"] |> Utils.date_to_asctime()
-    like_count = object["like_count"] || 0
-    announcement_count = object["announcement_count"] || 0
-    favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
-    repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
+    object = Object.normalize(object)
+
+    created_at = object.data["published"] |> Utils.date_to_asctime()
+    like_count = object.data["like_count"] || 0
+    announcement_count = object.data["announcement_count"] || 0
+    favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
+    repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || [])
 
     mentions = opts[:mentioned] || []
 
@@ -160,8 +162,8 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
 
     conversation_id = conversation_id(activity)
 
-    tags = activity.data["object"]["tag"] || []
-    possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw")
+    tags = object.data["tag"] || []
+    possibly_sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
 
     tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags
 
@@ -169,16 +171,16 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
 
     html =
       HTML.filter_tags(content, User.html_filter_policy(opts[:for]))
-      |> Formatter.emojify(object["emoji"])
+      |> Formatter.emojify(object.data["emoji"])
 
     video =
       if object["type"] == "Video" do
-        vid = [object]
+        vid = [object.data]
       else
         []
       end
 
-    attachments = (object["attachment"] || []) ++ video
+    attachments = (object.data["attachment"] || []) ++ video
 
     reply_parent = Activity.get_in_reply_to_activity(activity)
 
@@ -186,14 +188,14 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
 
     %{
       "id" => activity.id,
-      "uri" => activity.data["object"]["id"],
+      "uri" => object.data["id"],
       "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
       "statusnet_html" => html,
       "text" => HTML.strip_tags(content),
       "is_local" => activity.local,
       "is_post_verb" => true,
       "created_at" => created_at,
-      "in_reply_to_status_id" => object["inReplyToStatusId"],
+      "in_reply_to_status_id" => object.data["inReplyToStatusId"],
       "in_reply_to_screen_name" => reply_user && reply_user.nickname,
       "in_reply_to_profileurl" => User.profile_url(reply_user),
       "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id,
@@ -205,12 +207,12 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
       "repeat_num" => announcement_count,
       "favorited" => to_boolean(favorited),
       "repeated" => to_boolean(repeated),
-      "external_url" => object["external_url"] || object["id"],
+      "external_url" => object.data["external_url"] || object.data["id"],
       "tags" => tags,
       "activity_type" => "post",
       "possibly_sensitive" => possibly_sensitive,
-      "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object),
-      "summary" => object["summary"]
+      "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data),
+      "summary" => object.data["summary"]
     }
   end
 
diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index 83e8fb765..7839fe878 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -225,8 +225,8 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
 
     conversation_id = get_context_id(activity, opts)
 
-    tags = activity.data["object"]["tag"] || []
-    possibly_sensitive = activity.data["object"]["sensitive"] || Enum.member?(tags, "nsfw")
+    tags = object.data["tag"] || []
+    possibly_sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
 
     tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags
 
@@ -242,7 +242,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
 
     %{
       "id" => activity.id,
-      "uri" => activity.data["object"]["id"],
+      "uri" => object.data["id"],
       "user" => UserView.render("show.json", %{user: user, for: opts[:for]}),
       "statusnet_html" => html,
       "text" => HTML.strip_tags(content),
-- 
cgit v1.2.3


From e9b718cea2ee21a38f6c2137f32e3da6f5893cc2 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 20:21:42 +0000
Subject: mastodon api: status view: use Object.normalize()

---
 lib/pleroma/web/mastodon_api/views/status_view.ex | 50 +++++++++++++----------
 1 file changed, 28 insertions(+), 22 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 2d9a915f0..3b4911d53 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -1,7 +1,7 @@
 defmodule Pleroma.Web.MastodonAPI.StatusView do
   use Pleroma.Web, :view
   alias Pleroma.Web.MastodonAPI.{AccountView, StatusView}
-  alias Pleroma.{User, Activity}
+  alias Pleroma.{User, Activity, Object}
   alias Pleroma.Web.CommonAPI.Utils
   alias Pleroma.Web.MediaProxy
   alias Pleroma.Repo
@@ -11,8 +11,9 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   defp get_replied_to_activities(activities) do
     activities
     |> Enum.map(fn
-      %{data: %{"type" => "Create", "object" => %{"inReplyTo" => inReplyTo}}} ->
-        inReplyTo != "" && inReplyTo
+      %{data: %{"type" => "Create", "object" => object}} ->
+        object = Object.normalize(object)
+        object.data["inReplyTo"] != "" && object.data["inReplyTo"]
 
       _ ->
         nil
@@ -21,7 +22,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     |> Activity.create_activity_by_object_id_query()
     |> Repo.all()
     |> Enum.reduce(%{}, fn activity, acc ->
-      Map.put(acc, activity.data["object"]["id"], activity)
+      object = Object.normalize(activity.data["object"])
+      Map.put(acc, object.data["id"], activity)
     end)
   end
 
@@ -85,13 +87,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   end
 
   def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do
+    object = Object.normalize(object)
+
     user = User.get_cached_by_ap_id(activity.data["actor"])
 
-    like_count = object["like_count"] || 0
-    announcement_count = object["announcement_count"] || 0
+    like_count = object.data["like_count"] || 0
+    announcement_count = object.data["announcement_count"] || 0
 
-    tags = object["tag"] || []
-    sensitive = object["sensitive"] || Enum.member?(tags, "nsfw")
+    tags = object.data["tag"] || []
+    sensitive = object.data["sensitive"] || Enum.member?(tags, "nsfw")
 
     mentions =
       activity.recipients
@@ -99,20 +103,20 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       |> Enum.filter(& &1)
       |> Enum.map(fn user -> AccountView.render("mention.json", %{user: user}) end)
 
-    repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
-    favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
+    repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || [])
+    favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
 
-    attachment_data = object["attachment"] || []
-    attachment_data = attachment_data ++ if object["type"] == "Video", do: [object], else: []
+    attachment_data = object.data["attachment"] || []
+    attachment_data = attachment_data ++ if object.data["type"] == "Video", do: [object], else: []
     attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment)
 
-    created_at = Utils.to_masto_date(object["published"])
+    created_at = Utils.to_masto_date(object.data["published"])
 
     reply_to = get_reply_to(activity, opts)
     reply_to_user = reply_to && User.get_cached_by_ap_id(reply_to.data["actor"])
 
     emojis =
-      (activity.data["object"]["emoji"] || [])
+      (object.data["emoji"] || [])
       |> Enum.map(fn {name, url} ->
         name = HTML.strip_tags(name)
 
@@ -124,13 +128,13 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       end)
 
     content =
-      render_content(object)
+      render_content(object.data)
       |> HTML.filter_tags(User.html_filter_policy(opts[:for]))
 
     %{
       id: to_string(activity.id),
-      uri: object["id"],
-      url: object["external_url"] || object["id"],
+      uri: object.data["id"],
+      url: object.data["external_url"] || object["id"],
       account: AccountView.render("account.json", %{user: user}),
       in_reply_to_id: reply_to && to_string(reply_to.id),
       in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id),
@@ -144,7 +148,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       favourited: !!favorited,
       muted: false,
       sensitive: sensitive,
-      spoiler_text: object["summary"] || "",
+      spoiler_text: object.data["summary"] || "",
       visibility: get_visibility(object),
       media_attachments: attachments |> Enum.take(4),
       mentions: mentions,
@@ -190,13 +194,15 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
   end
 
   def get_reply_to(activity, %{replied_to_activities: replied_to_activities}) do
-    _id = activity.data["object"]["inReplyTo"]
-    replied_to_activities[activity.data["object"]["inReplyTo"]]
+    object = Object.normalize(activity.data["object"])
+    replied_to_activities[object.data["inReplyTo"]]
   end
 
   def get_reply_to(%{data: %{"object" => object}}, _) do
-    if object["inReplyTo"] && object["inReplyTo"] != "" do
-      Activity.get_create_activity_by_object_ap_id(object["inReplyTo"])
+    object = Object.normalize(object)
+
+    if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do
+      Activity.get_create_activity_by_object_ap_id(object.data["inReplyTo"])
     else
       nil
     end
-- 
cgit v1.2.3


From 5d4a71906a15d4d6bff714ef0bb63fba1c5f4fb3 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 20:27:00 +0000
Subject: mastodon api: use Object.normalize() in a few missing spots

---
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 543fdf416..90225460b 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -464,7 +464,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def favourited_by(conn, %{"id" => id}) do
-    with %Activity{data: %{"object" => %{"likes" => likes}}} <- Repo.get(Activity, id) do
+    with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
+         %Object{data: %{"likes" => likes}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^likes)
       users = Repo.all(q)
       render(conn, AccountView, "accounts.json", %{users: users, as: :user})
@@ -474,7 +475,8 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def reblogged_by(conn, %{"id" => id}) do
-    with %Activity{data: %{"object" => %{"announcements" => announces}}} <- Repo.get(Activity, id) do
+    with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id),
+         %Object{data: %{"announcements" => announces}} <- Object.normalize(object) do
       q = from(u in User, where: u.ap_id in ^announces)
       users = Repo.all(q)
       render(conn, AccountView, "accounts.json", %{users: users, as: :user})
-- 
cgit v1.2.3


From dd66cc2ca6c99d5b1586133dd85d8d36d5cbaaa7 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 20:40:16 +0000
Subject: ostatus: use Object.normalize() where appropriate when representing
 activities

---
 lib/pleroma/web/ostatus/activity_representer.ex | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex
index 537bd9f77..ceb4d79db 100644
--- a/lib/pleroma/web/ostatus/activity_representer.ex
+++ b/lib/pleroma/web/ostatus/activity_representer.ex
@@ -84,11 +84,13 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
   def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do
     h = fn str -> [to_charlist(str)] end
 
-    updated_at = activity.data["object"]["published"]
-    inserted_at = activity.data["object"]["published"]
+    object = Object.normalize(activity.data["object"])
+
+    updated_at = object.data["published"]
+    inserted_at = object.data["published"]
 
     attachments =
-      Enum.map(activity.data["object"]["attachment"] || [], fn attachment ->
+      Enum.map(object.data["attachment"] || [], fn attachment ->
         url = hd(attachment["url"])
 
         {:link,
@@ -101,7 +103,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
     mentions = activity.recipients |> get_mentions
 
     categories =
-      (activity.data["object"]["tag"] || [])
+      (object.data["tag"] || [])
       |> Enum.map(fn tag ->
         if is_binary(tag) do
           {:category, [term: to_charlist(tag)], []}
@@ -111,11 +113,11 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
       end)
       |> Enum.filter(& &1)
 
-    emoji_links = get_emoji_links(activity.data["object"]["emoji"] || %{})
+    emoji_links = get_emoji_links(object.data["emoji"] || %{})
 
     summary =
-      if activity.data["object"]["summary"] do
-        [{:summary, [], h.(activity.data["object"]["summary"])}]
+      if object.data["summary"] do
+        [{:summary, [], h.(object.data["summary"])}]
       else
         []
       end
@@ -124,10 +126,9 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
       {:"activity:object-type", ['http://activitystrea.ms/schema/1.0/note']},
       {:"activity:verb", ['http://activitystrea.ms/schema/1.0/post']},
       # For notes, federate the object id.
-      {:id, h.(activity.data["object"]["id"])},
+      {:id, h.(object.data["id"])},
       {:title, ['New note by #{user.nickname}']},
-      {:content, [type: 'html'],
-       h.(activity.data["object"]["content"] |> String.replace(~r/[\n\r]/, ""))},
+      {:content, [type: 'html'], h.(object.data["content"] |> String.replace(~r/[\n\r]/, ""))},
       {:published, h.(inserted_at)},
       {:updated, h.(updated_at)},
       {:"ostatus:conversation", [ref: h.(activity.data["context"])],
-- 
cgit v1.2.3


From 3b8e5bcbeb03f3cfa5de1c4e7a4f3a04871094d1 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 20:56:12 +0000
Subject: fix most tests

---
 lib/pleroma/web/mastodon_api/views/status_view.ex                | 4 ++--
 lib/pleroma/web/twitter_api/representers/activity_representer.ex | 4 ++--
 lib/pleroma/web/twitter_api/views/activity_view.ex               | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 3b4911d53..31f4675c3 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -134,7 +134,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     %{
       id: to_string(activity.id),
       uri: object.data["id"],
-      url: object.data["external_url"] || object["id"],
+      url: object.data["external_url"] || object.data["id"],
       account: AccountView.render("account.json", %{user: user}),
       in_reply_to_id: reply_to && to_string(reply_to.id),
       in_reply_to_account_id: reply_to_user && to_string(reply_to_user.id),
@@ -149,7 +149,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
       muted: false,
       sensitive: sensitive,
       spoiler_text: object.data["summary"] || "",
-      visibility: get_visibility(object),
+      visibility: get_visibility(object.data),
       media_attachments: attachments |> Enum.take(4),
       mentions: mentions,
       # fix,
diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
index 4b4e202b3..436f9bf92 100644
--- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
@@ -167,14 +167,14 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
 
     tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags
 
-    {summary, content} = ActivityView.render_content(object)
+    {summary, content} = ActivityView.render_content(object.data)
 
     html =
       HTML.filter_tags(content, User.html_filter_policy(opts[:for]))
       |> Formatter.emojify(object.data["emoji"])
 
     video =
-      if object["type"] == "Video" do
+      if object.data["type"] == "Video" do
         vid = [object.data]
       else
         []
diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index 7839fe878..f202b6e97 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -265,7 +265,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       "tags" => tags,
       "activity_type" => "post",
       "possibly_sensitive" => possibly_sensitive,
-      "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object),
+      "visibility" => Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data),
       "summary" => summary
     }
   end
-- 
cgit v1.2.3


From e8570758f90f0ef040eab011d0584c59379ba743 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 21:16:44 +0000
Subject: twitterapi: fix remaining test failures

---
 lib/pleroma/web/twitter_api/representers/activity_representer.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/twitter_api/representers/activity_representer.ex b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
index 436f9bf92..8f91aeaf0 100644
--- a/lib/pleroma/web/twitter_api/representers/activity_representer.ex
+++ b/lib/pleroma/web/twitter_api/representers/activity_representer.ex
@@ -141,7 +141,7 @@ defmodule Pleroma.Web.TwitterAPI.Representers.ActivityRepresenter do
   end
 
   def to_map(
-        %Activity{data: %{"object" => %{"content" => content} = object}} = activity,
+        %Activity{data: %{"object" => object}} = activity,
         %{user: user} = opts
       ) do
     object = Object.normalize(object)
-- 
cgit v1.2.3


From 5ba5df1321b8c177419ae5342d37bc524f8f6656 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 21:20:29 +0000
Subject: object: normalize(): use object cache

---
 lib/pleroma/object.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 03a75dfbd..57a8b1d6b 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -27,8 +27,8 @@ defmodule Pleroma.Object do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
-  def normalize(obj) when is_map(obj), do: Object.get_by_ap_id(obj["id"])
-  def normalize(ap_id) when is_binary(ap_id), do: Object.get_by_ap_id(ap_id)
+  def normalize(obj) when is_map(obj), do: normalize(obj["id"])
+  def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   if Mix.env() == :test do
-- 
cgit v1.2.3


From 5ea64f4bf958d9801d89d1165fa20d0e17bf0570 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 21:34:36 +0000
Subject: activity: minor cleanups to normalization functions to align them
 with the object normalizers

---
 lib/pleroma/activity.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 3c7e150ee..34d0a34b8 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -79,8 +79,8 @@ defmodule Pleroma.Activity do
 
   def get_create_activity_by_object_ap_id(_), do: nil
 
-  def normalize(obj) when is_map(obj), do: Activity.get_by_ap_id(obj["id"])
-  def normalize(ap_id) when is_binary(ap_id), do: Activity.get_by_ap_id(ap_id)
+  def normalize(obj) when is_map(obj), do: normalize(obj["id"])
+  def normalize(ap_id) when is_binary(ap_id), do: get_by_ap_id(ap_id)
   def normalize(_), do: nil
 
   defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
-- 
cgit v1.2.3


From 57d90e7afe46a54cc7e1ef14a9b76b0650ed3db6 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 21:43:26 +0000
Subject: activitypub: relay: fix improper use of Object.normalize()

---
 lib/pleroma/web/activity_pub/relay.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/relay.ex b/lib/pleroma/web/activity_pub/relay.ex
index fcdc6b1c0..a48a91ef7 100644
--- a/lib/pleroma/web/activity_pub/relay.ex
+++ b/lib/pleroma/web/activity_pub/relay.ex
@@ -35,7 +35,7 @@ defmodule Pleroma.Web.ActivityPub.Relay do
 
   def publish(%Activity{data: %{"type" => "Create"}} = activity) do
     with %User{} = user <- get_actor(),
-         %Object{} = object <- Object.normalize(activity.data["object"]["id"]) do
+         %Object{} = object <- Object.normalize(activity.data["object"]) do
       ActivityPub.announce(user, object)
     else
       e -> Logger.error("error: #{inspect(e)}")
-- 
cgit v1.2.3


From d13d953385b7659d7a9eb381ac9303a41e6fc294 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 21:44:03 +0000
Subject: activitypub: implement activity flattening

---
 lib/pleroma/web/activity_pub/activity_pub.ex |  2 +-
 lib/pleroma/web/activity_pub/utils.ex        | 12 ++++++++----
 2 files changed, 9 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index b11d3221f..dcf670afb 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -57,7 +57,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
          map <- lazy_put_activity_defaults(map),
          :ok <- check_actor_is_active(map["actor"]),
          {:ok, map} <- MRF.filter(map),
-         :ok <- insert_full_object(map) do
+         {:ok, map} <- insert_full_object(map) do
       {recipients, _, _} = get_recipients(map)
 
       {:ok, activity} =
diff --git a/lib/pleroma/web/activity_pub/utils.ex b/lib/pleroma/web/activity_pub/utils.ex
index 549148989..bc5b98f1a 100644
--- a/lib/pleroma/web/activity_pub/utils.ex
+++ b/lib/pleroma/web/activity_pub/utils.ex
@@ -180,14 +180,18 @@ defmodule Pleroma.Web.ActivityPub.Utils do
   @doc """
   Inserts a full object if it is contained in an activity.
   """
-  def insert_full_object(%{"object" => %{"type" => type} = object_data})
+  def insert_full_object(%{"object" => %{"type" => type} = object_data} = map)
       when is_map(object_data) and type in @supported_object_types do
-    with {:ok, _} <- Object.create(object_data) do
-      :ok
+    with {:ok, object} <- Object.create(object_data) do
+      map =
+        map
+        |> Map.put("object", object.data["id"])
+
+      {:ok, map}
     end
   end
 
-  def insert_full_object(_), do: :ok
+  def insert_full_object(map), do: {:ok, map}
 
   def update_object_in_activities(%{data: %{"id" => id}} = object) do
     # TODO
-- 
cgit v1.2.3


From d6e65f9304c1598087a6dacc640bcd0bb4057009 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 21:50:46 +0000
Subject: common api: fix up improper Object.normalize() calls

---
 lib/pleroma/web/common_api/common_api.ex | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index e0c9dedd5..c83f8a6a9 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -6,7 +6,7 @@ defmodule Pleroma.Web.CommonAPI do
   import Pleroma.Web.CommonAPI.Utils
 
   def delete(activity_id, user) do
-    with %Activity{data: %{"object" => %{"id" => object_id}}} <- Repo.get(Activity, activity_id),
+    with %Activity{data: %{"object" => object_id}} <- Repo.get(Activity, activity_id),
          %Object{} = object <- Object.normalize(object_id),
          true <- user.info.is_moderator || user.ap_id == object.data["actor"],
          {:ok, delete} <- ActivityPub.delete(object) do
@@ -16,7 +16,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def repeat(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity.data["object"]) do
       ActivityPub.announce(user, object)
     else
       _ ->
@@ -26,7 +26,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def unrepeat(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity.data["object"]) do
       ActivityPub.unannounce(user, object)
     else
       _ ->
@@ -36,7 +36,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def favorite(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity.data["object"]) do
       ActivityPub.like(user, object)
     else
       _ ->
@@ -46,7 +46,7 @@ defmodule Pleroma.Web.CommonAPI do
 
   def unfavorite(id_or_ap_id, user) do
     with %Activity{} = activity <- get_by_id_or_ap_id(id_or_ap_id),
-         object <- Object.normalize(activity.data["object"]["id"]) do
+         object <- Object.normalize(activity.data["object"]) do
       ActivityPub.unlike(user, object)
     else
       _ ->
-- 
cgit v1.2.3


From cf139e3eec9bf978cfff24044dce5b5c388fa303 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sun, 25 Nov 2018 22:11:10 +0000
Subject: activitypub: transmogrifier: ensure we send nested object in Create

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 17b063609..5e3d40d9f 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -616,9 +616,9 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   #  internal -> Mastodon
   #  """
 
-  def prepare_outgoing(%{"type" => "Create", "object" => %{"type" => "Note"} = object} = data) do
+  def prepare_outgoing(%{"type" => "Create", "object" => object_id} = data) do
     object =
-      object
+      Object.normalize(object_id).data
       |> prepare_object
 
     data =
-- 
cgit v1.2.3


From f168a2add66a312aa9911c880806534899f3fe08 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 29 Nov 2018 03:06:20 +0000
Subject: ostatus: fix representing external objects

---
 lib/pleroma/web/ostatus/activity_representer.ex | 21 +++++++--------------
 1 file changed, 7 insertions(+), 14 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/ostatus/activity_representer.ex b/lib/pleroma/web/ostatus/activity_representer.ex
index ceb4d79db..fefd9459a 100644
--- a/lib/pleroma/web/ostatus/activity_representer.ex
+++ b/lib/pleroma/web/ostatus/activity_representer.ex
@@ -47,23 +47,16 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
     end)
   end
 
-  defp get_links(%{local: true, data: data}) do
+  defp get_links(%{local: true}, %{"id" => object_id}) do
     h = fn str -> [to_charlist(str)] end
 
     [
-      {:link, [type: ['application/atom+xml'], href: h.(data["object"]["id"]), rel: 'self'], []},
-      {:link, [type: ['text/html'], href: h.(data["object"]["id"]), rel: 'alternate'], []}
+      {:link, [type: ['application/atom+xml'], href: h.(object_id), rel: 'self'], []},
+      {:link, [type: ['text/html'], href: h.(object_id), rel: 'alternate'], []}
     ]
   end
 
-  defp get_links(%{
-         local: false,
-         data: %{
-           "object" => %{
-             "external_url" => external_url
-           }
-         }
-       }) do
+  defp get_links(%{local: false}, %{"external_url" => external_url}) do
     h = fn str -> [to_charlist(str)] end
 
     [
@@ -71,7 +64,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
     ]
   end
 
-  defp get_links(_activity), do: []
+  defp get_links(_activity, _object_data), do: []
 
   defp get_emoji_links(emojis) do
     Enum.map(emojis, fn {emoji, file} ->
@@ -81,7 +74,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
 
   def to_simple_form(activity, user, with_author \\ false)
 
-  def to_simple_form(%{data: %{"object" => %{"type" => "Note"}}} = activity, user, with_author) do
+  def to_simple_form(%{data: %{"type" => "Create"}} = activity, user, with_author) do
     h = fn str -> [to_charlist(str)] end
 
     object = Object.normalize(activity.data["object"])
@@ -136,7 +129,7 @@ defmodule Pleroma.Web.OStatus.ActivityRepresenter do
       {:link, [ref: h.(activity.data["context"]), rel: 'ostatus:conversation'], []}
     ] ++
       summary ++
-      get_links(activity) ++
+      get_links(activity, object.data) ++
       categories ++ attachments ++ in_reply_to ++ author ++ mentions ++ emoji_links
   end
 
-- 
cgit v1.2.3


From 6f90f2c3ac70c74b9d06debb09530d5f479b5a8c Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 29 Nov 2018 03:34:57 +0000
Subject: activitypub: rework thread filtering for split object view

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index dcf670afb..34a84b045 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -799,18 +799,21 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   # guard
   def entire_thread_visible_for_user?(nil, user), do: false
 
-  # child
+  # child / root
   def entire_thread_visible_for_user?(
-        %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
+        %Activity{data: %{"object" => object_id}} = tail,
         user
-      )
-      when is_binary(parent_id) do
+      ) do
     parent = Activity.get_in_reply_to_activity(tail)
-    visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
-  end
 
-  # root
-  def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user)
+    cond do
+      !is_nil(parent) ->
+        visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
+
+      true ->
+        visible_for_user?(tail, user)
+    end
+  end
 
   # filter out broken threads
   def contain_broken_threads(%Activity{} = activity, %User{} = user) do
-- 
cgit v1.2.3


From 5d753e1c7cb1b26ad224255c31b0b64ad917ebaa Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 29 Nov 2018 06:52:54 +0000
Subject: activity: add helpers for updating activities in the database

---
 lib/pleroma/activity.ex | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 34d0a34b8..e3aa4eb97 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -1,7 +1,7 @@
 defmodule Pleroma.Activity do
   use Ecto.Schema
   alias Pleroma.{Repo, Activity, Notification, Object}
-  import Ecto.Query
+  import Ecto.{Query, Changeset}
 
   schema "activities" do
     field(:data, :map)
@@ -22,6 +22,13 @@ defmodule Pleroma.Activity do
     )
   end
 
+  def change(struct, params \\ %{}) do
+    struct
+    |> cast(params, [:data])
+    |> validate_required([:data])
+    |> unique_constraint(:ap_id, name: :activities_unique_apid_index)
+  end
+
   # TODO:
   # Go through these and fix them everywhere.
   # Wrong name, only returns create activities
-- 
cgit v1.2.3


From ef56488349a257def67d6c906a1f71e9bbed397e Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Thu, 29 Nov 2018 06:53:17 +0000
Subject: mix: add task to compact the database

---
 lib/mix/tasks/compact_database.ex | 57 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 lib/mix/tasks/compact_database.ex

(limited to 'lib')

diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex
new file mode 100644
index 000000000..b84b340ac
--- /dev/null
+++ b/lib/mix/tasks/compact_database.ex
@@ -0,0 +1,57 @@
+defmodule Mix.Tasks.CompactDatabase do
+  @moduledoc """
+  Compact the database by flattening the object graph.
+  """
+
+  require Logger
+
+  use Mix.Task
+  import Mix.Ecto
+  import Ecto.Query
+  alias Pleroma.{Repo, Object, Activity}
+
+  defp maybe_compact(%Activity{data: %{"object" => %{"id" => object_id}}} = activity) do
+    data =
+      activity.data
+      |> Map.put("object", object_id)
+
+    {:ok, activity} =
+      Activity.change(activity, %{data: data})
+      |> Repo.update()
+
+    {:ok, activity}
+  end
+
+  defp maybe_compact(%Activity{} = activity), do: {:ok, activity}
+
+  defp activity_query(min_id, max_id) do
+    from(
+      a in Activity,
+      where: fragment("?->>'type' = 'Create'", a.data),
+      where: a.id >= ^min_id,
+      where: a.id < ^max_id
+    )
+  end
+
+  def run(args) do
+    Application.ensure_all_started(:pleroma)
+
+    max = Repo.aggregate(Activity, :max, :id)
+    Logger.info("Considering #{max} activities")
+
+    chunks = 0..(round(max / 100))
+
+    Enum.each(chunks, fn (i) ->
+      min = i * 100
+      max = min + 100
+
+      activity_query(min, max)
+      |> Repo.all()
+      |> Enum.each(&maybe_compact/1)
+
+      IO.write(".")
+    end)
+
+    Logger.info("Finished.")
+  end
+end
-- 
cgit v1.2.3


From 1a360a4eaa57d57ccc4f03dfa25e82a240d0175a Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sat, 1 Dec 2018 22:17:28 +0000
Subject: compact database task: fix formatting

---
 lib/mix/tasks/compact_database.ex | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex
index b84b340ac..7de50812a 100644
--- a/lib/mix/tasks/compact_database.ex
+++ b/lib/mix/tasks/compact_database.ex
@@ -39,9 +39,9 @@ defmodule Mix.Tasks.CompactDatabase do
     max = Repo.aggregate(Activity, :max, :id)
     Logger.info("Considering #{max} activities")
 
-    chunks = 0..(round(max / 100))
+    chunks = 0..round(max / 100)
 
-    Enum.each(chunks, fn (i) ->
+    Enum.each(chunks, fn i ->
       min = i * 100
       max = min + 100
 
-- 
cgit v1.2.3


From e8caecb5c7f49a829b857131ff98a46c705e3a80 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sat, 1 Dec 2018 22:29:41 +0000
Subject: object: move object containment out of transmogrifier into it's own
 module

---
 lib/pleroma/object/containment.ex              | 64 +++++++++++++++++++++++
 lib/pleroma/web/activity_pub/activity_pub.ex   |  5 +-
 lib/pleroma/web/activity_pub/transmogrifier.ex | 71 ++++----------------------
 3 files changed, 78 insertions(+), 62 deletions(-)
 create mode 100644 lib/pleroma/object/containment.ex

(limited to 'lib')

diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex
new file mode 100644
index 000000000..010b768bd
--- /dev/null
+++ b/lib/pleroma/object/containment.ex
@@ -0,0 +1,64 @@
+defmodule Pleroma.Object.Containment do
+  @moduledoc """
+  # Object Containment
+
+  This module contains some useful functions for containing objects to specific
+  origins and determining those origins.  They previously lived in the
+  ActivityPub `Transmogrifier` module.
+
+  Object containment is an important step in validating remote objects to prevent
+  spoofing, therefore removal of object containment functions is NOT recommended.
+  """
+
+  require Logger
+
+  def get_actor(%{"actor" => actor}) when is_binary(actor) do
+    actor
+  end
+
+  def get_actor(%{"actor" => actor}) when is_list(actor) do
+    if is_binary(Enum.at(actor, 0)) do
+      Enum.at(actor, 0)
+    else
+      Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
+      |> Map.get("id")
+    end
+  end
+
+  def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do
+    id
+  end
+
+  def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
+    get_actor(%{"actor" => actor})
+  end
+
+  @doc """
+  Checks that an imported AP object's actor matches the domain it came from.
+  """
+  def contain_origin(id, %{"actor" => nil}), do: :error
+
+  def contain_origin(id, %{"actor" => actor} = params) do
+    id_uri = URI.parse(id)
+    actor_uri = URI.parse(get_actor(params))
+
+    if id_uri.host == actor_uri.host do
+      :ok
+    else
+      :error
+    end
+  end
+
+  def contain_origin_from_id(id, %{"id" => nil}), do: :error
+
+  def contain_origin_from_id(id, %{"id" => other_id} = params) do
+    id_uri = URI.parse(id)
+    other_uri = URI.parse(other_id)
+
+    if id_uri.host == other_uri.host do
+      :ok
+    else
+      :error
+    end
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 34a84b045..517cf4b46 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1,5 +1,6 @@
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
+  alias Pleroma.Object.Containment
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
@@ -739,7 +740,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              "actor" => data["actor"] || data["attributedTo"],
              "object" => data
            },
-           :ok <- Transmogrifier.contain_origin(id, params),
+           :ok <- Containment.contain_origin(id, params),
            {:ok, activity} <- Transmogrifier.handle_incoming(params) do
         {:ok, Object.normalize(activity.data["object"])}
       else
@@ -773,7 +774,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
              recv_timeout: 20000
            ),
          {:ok, data} <- Jason.decode(body),
-         :ok <- Transmogrifier.contain_origin_from_id(id, data) do
+         :ok <- Containment.contain_origin_from_id(id, data) do
       {:ok, data}
     else
       e ->
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 5e3d40d9f..1b5e57294 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -4,6 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   """
   alias Pleroma.User
   alias Pleroma.Object
+  alias Pleroma.Object.Containment
   alias Pleroma.Activity
   alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -13,56 +14,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   require Logger
 
-  def get_actor(%{"actor" => actor}) when is_binary(actor) do
-    actor
-  end
-
-  def get_actor(%{"actor" => actor}) when is_list(actor) do
-    if is_binary(Enum.at(actor, 0)) do
-      Enum.at(actor, 0)
-    else
-      Enum.find(actor, fn %{"type" => type} -> type in ["Person", "Service", "Application"] end)
-      |> Map.get("id")
-    end
-  end
-
-  def get_actor(%{"actor" => %{"id" => id}}) when is_bitstring(id) do
-    id
-  end
-
-  def get_actor(%{"actor" => nil, "attributedTo" => actor}) when not is_nil(actor) do
-    get_actor(%{"actor" => actor})
-  end
-
-  @doc """
-  Checks that an imported AP object's actor matches the domain it came from.
-  """
-  def contain_origin(id, %{"actor" => nil}), do: :error
-
-  def contain_origin(id, %{"actor" => actor} = params) do
-    id_uri = URI.parse(id)
-    actor_uri = URI.parse(get_actor(params))
-
-    if id_uri.host == actor_uri.host do
-      :ok
-    else
-      :error
-    end
-  end
-
-  def contain_origin_from_id(id, %{"id" => nil}), do: :error
-
-  def contain_origin_from_id(id, %{"id" => other_id} = params) do
-    id_uri = URI.parse(id)
-    other_uri = URI.parse(other_id)
-
-    if id_uri.host == other_uri.host do
-      :ok
-    else
-      :error
-    end
-  end
-
   @doc """
   Modifies an incoming AP object (mastodon format) to our internal format.
   """
@@ -99,7 +50,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def fix_actor(%{"attributedTo" => actor} = object) do
     object
-    |> Map.put("actor", get_actor(%{"actor" => actor}))
+    |> Map.put("actor", Containment.get_actor(%{"actor" => actor}))
   end
 
   def fix_likes(%{"likes" => likes} = object)
@@ -277,7 +228,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   # - emoji
   def handle_incoming(%{"type" => "Create", "object" => %{"type" => objtype} = object} = data)
       when objtype in ["Article", "Note", "Video", "Page"] do
-    actor = get_actor(data)
+    actor = Containment.get_actor(data)
 
     data =
       Map.put(data, "actor", actor)
@@ -360,7 +311,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def handle_incoming(
         %{"type" => "Accept", "object" => follow_object, "actor" => actor, "id" => id} = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "accept"),
@@ -386,7 +337,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def handle_incoming(
         %{"type" => "Reject", "object" => follow_object, "actor" => actor, "id" => id} = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = followed <- User.get_or_fetch_by_ap_id(actor),
          {:ok, follow_activity} <- get_follow_activity(follow_object, followed),
          {:ok, follow_activity} <- Utils.update_follow_state(follow_activity, "reject"),
@@ -410,7 +361,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def handle_incoming(
         %{"type" => "Like", "object" => object_id, "actor" => actor, "id" => id} = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
@@ -423,7 +374,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   def handle_incoming(
         %{"type" => "Announce", "object" => object_id, "actor" => actor, "id" => id} = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do
@@ -477,10 +428,10 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     object_id = Utils.get_ap_id(object_id)
 
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
-         :ok <- contain_origin(actor.ap_id, object.data),
+         :ok <- Containment.contain_origin(actor.ap_id, object.data),
          {:ok, activity} <- ActivityPub.delete(object, false) do
       {:ok, activity}
     else
@@ -496,7 +447,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           "id" => id
         } = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do
@@ -566,7 +517,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           "id" => id
         } = data
       ) do
-    with actor <- get_actor(data),
+    with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
          {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
          {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do
-- 
cgit v1.2.3


From f85949cc696685496a8e4e17aebeb81168ede41a Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sat, 1 Dec 2018 22:53:10 +0000
Subject: object: factor out fetching functions into Pleroma.Object.Fetcher
 module

---
 lib/pleroma/object/fetcher.ex                      | 69 ++++++++++++++++++++++
 lib/pleroma/web/activity_pub/activity_pub.ex       | 63 +-------------------
 .../web/activity_pub/activity_pub_controller.ex    |  3 +-
 lib/pleroma/web/activity_pub/transmogrifier.ex     |  6 +-
 .../web/mastodon_api/mastodon_api_controller.ex    |  7 ++-
 5 files changed, 81 insertions(+), 67 deletions(-)
 create mode 100644 lib/pleroma/object/fetcher.ex

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
new file mode 100644
index 000000000..c27cb1577
--- /dev/null
+++ b/lib/pleroma/object/fetcher.ex
@@ -0,0 +1,69 @@
+defmodule Pleroma.Object.Fetcher do
+  alias Pleroma.{Object, Repo}
+  alias Pleroma.Object.Containment
+  alias Pleroma.Web.ActivityPub.Transmogrifier
+  alias Pleroma.Web.OStatus
+
+  require Logger
+
+  @httpoison Application.get_env(:pleroma, :httpoison)
+
+  # TODO:
+  # This will create a Create activity, which we need internally at the moment.
+  def fetch_object_from_id(id) do
+    if object = Object.get_cached_by_ap_id(id) do
+      {:ok, object}
+    else
+      Logger.info("Fetching #{id} via AP")
+
+      with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
+           nil <- Object.normalize(data),
+           params <- %{
+             "type" => "Create",
+             "to" => data["to"],
+             "cc" => data["cc"],
+             "actor" => data["actor"] || data["attributedTo"],
+             "object" => data
+           },
+           :ok <- Containment.contain_origin(id, params),
+           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
+        {:ok, Object.normalize(activity.data["object"])}
+      else
+        {:error, {:reject, nil}} ->
+          {:reject, nil}
+
+        object = %Object{} ->
+          {:ok, object}
+
+        _e ->
+          Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
+
+          case OStatus.fetch_activity_from_url(id) do
+            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
+            e -> e
+          end
+      end
+    end
+  end
+
+  def fetch_and_contain_remote_object_from_id(id) do
+    Logger.info("Fetching #{id} via AP")
+
+    with true <- String.starts_with?(id, "http"),
+         {:ok, %{body: body, status_code: code}} when code in 200..299 <-
+           @httpoison.get(
+             id,
+             [Accept: "application/activity+json"],
+             follow_redirect: true,
+             timeout: 10000,
+             recv_timeout: 20000
+           ),
+         {:ok, data} <- Jason.decode(body),
+         :ok <- Containment.contain_origin_from_id(id, data) do
+      {:ok, data}
+    else
+      e ->
+        {:error, e}
+    end
+  end
+end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 517cf4b46..fefefc320 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -1,6 +1,6 @@
 defmodule Pleroma.Web.ActivityPub.ActivityPub do
   alias Pleroma.{Activity, Repo, Object, Upload, User, Notification}
-  alias Pleroma.Object.Containment
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Web.ActivityPub.{Transmogrifier, MRF}
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Federator
@@ -629,7 +629,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   end
 
   def fetch_and_prepare_user_from_ap_id(ap_id) do
-    with {:ok, data} <- fetch_and_contain_remote_object_from_id(ap_id) do
+    with {:ok, data} <- Fetcher.fetch_and_contain_remote_object_from_id(ap_id) do
       user_data_from_user_object(data)
     else
       e -> Logger.error("Could not decode user at fetch #{ap_id}, #{inspect(e)}")
@@ -723,65 +723,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
     )
   end
 
-  # TODO:
-  # This will create a Create activity, which we need internally at the moment.
-  def fetch_object_from_id(id) do
-    if object = Object.get_cached_by_ap_id(id) do
-      {:ok, object}
-    else
-      Logger.info("Fetching #{id} via AP")
-
-      with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
-           nil <- Object.normalize(data),
-           params <- %{
-             "type" => "Create",
-             "to" => data["to"],
-             "cc" => data["cc"],
-             "actor" => data["actor"] || data["attributedTo"],
-             "object" => data
-           },
-           :ok <- Containment.contain_origin(id, params),
-           {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity.data["object"])}
-      else
-        {:error, {:reject, nil}} ->
-          {:reject, nil}
-
-        object = %Object{} ->
-          {:ok, object}
-
-        _e ->
-          Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
-
-          case OStatus.fetch_activity_from_url(id) do
-            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
-            e -> e
-          end
-      end
-    end
-  end
-
-  def fetch_and_contain_remote_object_from_id(id) do
-    Logger.info("Fetching #{id} via AP")
-
-    with true <- String.starts_with?(id, "http"),
-         {:ok, %{body: body, status_code: code}} when code in 200..299 <-
-           @httpoison.get(
-             id,
-             [Accept: "application/activity+json"],
-             follow_redirect: true,
-             timeout: 10000,
-             recv_timeout: 20000
-           ),
-         {:ok, data} <- Jason.decode(body),
-         :ok <- Containment.contain_origin_from_id(id, data) do
-      {:ok, data}
-    else
-      e ->
-        {:error, e}
-    end
-  end
-
   def is_public?(activity) do
     "https://www.w3.org/ns/activitystreams#Public" in (activity.data["to"] ++
                                                          (activity.data["cc"] || []))
diff --git a/lib/pleroma/web/activity_pub/activity_pub_controller.ex b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
index 3570a75cb..7b7c0e090 100644
--- a/lib/pleroma/web/activity_pub/activity_pub_controller.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub_controller.ex
@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.ActivityPub.ActivityPubController do
   use Pleroma.Web, :controller
   alias Pleroma.{User, Object}
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Web.ActivityPub.{ObjectView, UserView}
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -122,7 +123,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPubController do
       "Signature missing or not from author, relayed Create message, fetching object from source"
     )
 
-    ActivityPub.fetch_object_from_id(params["object"]["id"])
+    Fetcher.fetch_object_from_id(params["object"]["id"])
 
     json(conn, "ok")
   end
diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 1b5e57294..e76e29b95 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -4,7 +4,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   """
   alias Pleroma.User
   alias Pleroma.Object
-  alias Pleroma.Object.Containment
+  alias Pleroma.Object.{Containment, Fetcher}
   alias Pleroma.Activity
   alias Pleroma.Repo
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -529,8 +529,8 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def handle_incoming(_), do: :error
 
-  def fetch_obj_helper(id) when is_bitstring(id), do: ActivityPub.fetch_object_from_id(id)
-  def fetch_obj_helper(obj) when is_map(obj), do: ActivityPub.fetch_object_from_id(obj["id"])
+  def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id)
+  def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"])
 
   def get_obj_helper(id) do
     if object = Object.normalize(id), do: {:ok, object}, else: nil
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 90225460b..71390be0d 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1,6 +1,7 @@
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
   alias Pleroma.{Repo, Object, Activity, User, Notification, Stats}
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Web
   alias Pleroma.Web.MastodonAPI.{StatusView, AccountView, MastodonView, ListView, FilterView}
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -658,7 +659,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def status_search(query) do
     fetched =
       if Regex.match?(~r/https?:/, query) do
-        with {:ok, object} <- ActivityPub.fetch_object_from_id(query) do
+        with {:ok, object} <- Fetcher.fetch_object_from_id(query) do
           [Activity.get_create_activity_by_object_ap_id(object.data["id"])]
         else
           _e -> []
@@ -986,7 +987,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   def login(conn, _) do
     with {:ok, app} <- get_or_make_app() do
       path =
-        o_auth_path(conn, :authorize,
+        o_auth_path(
+          conn,
+          :authorize,
           response_type: "code",
           client_id: app.client_id,
           redirect_uri: ".",
-- 
cgit v1.2.3


From 7a57db0d3a18dbeb9fb3682b98f741ef6ba5f518 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sat, 1 Dec 2018 23:02:32 +0000
Subject: federator: fix up contain_origin_from_id() call

---
 lib/pleroma/web/federator/federator.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index ac3d7c132..0644f8d0a 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -2,6 +2,7 @@ defmodule Pleroma.Web.Federator do
   use GenServer
   alias Pleroma.User
   alias Pleroma.Activity
+  alias Pleroma.Object.Containment
   alias Pleroma.Web.{WebFinger, Websub}
   alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.ActivityPub.ActivityPub
@@ -106,7 +107,7 @@ defmodule Pleroma.Web.Federator do
     # actor shouldn't be acting on objects outside their own AP server.
     with {:ok, _user} <- ap_enabled_actor(params["actor"]),
          nil <- Activity.normalize(params["id"]),
-         :ok <- Transmogrifier.contain_origin_from_id(params["actor"], params),
+         :ok <- Containment.contain_origin_from_id(params["actor"], params),
          {:ok, activity} <- Transmogrifier.handle_incoming(params) do
       {:ok, activity}
     else
-- 
cgit v1.2.3


From 02288b5f1cd428c12f92b9ea4fb0b0a935bbdaaf Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Sat, 1 Dec 2018 23:08:23 +0000
Subject: twitterapi: fix bad rebase

---
 lib/pleroma/web/twitter_api/views/activity_view.ex | 24 ++++++++++++----------
 1 file changed, 13 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex
index f202b6e97..18b2ebb0b 100644
--- a/lib/pleroma/web/twitter_api/views/activity_view.ex
+++ b/lib/pleroma/web/twitter_api/views/activity_view.ex
@@ -207,15 +207,17 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
 
   def render(
         "activity.json",
-        %{activity: %{data: %{"type" => "Create", "object" => object}} = activity} = opts
+        %{activity: %{data: %{"type" => "Create", "object" => object_id}} = activity} = opts
       ) do
     user = get_user(activity.data["actor"], opts)
 
-    created_at = object["published"] |> Utils.date_to_asctime()
-    like_count = object["like_count"] || 0
-    announcement_count = object["announcement_count"] || 0
-    favorited = opts[:for] && opts[:for].ap_id in (object["likes"] || [])
-    repeated = opts[:for] && opts[:for].ap_id in (object["announcements"] || [])
+    object = Object.normalize(object_id)
+
+    created_at = object.data["published"] |> Utils.date_to_asctime()
+    like_count = object.data["like_count"] || 0
+    announcement_count = object.data["announcement_count"] || 0
+    favorited = opts[:for] && opts[:for].ap_id in (object.data["likes"] || [])
+    repeated = opts[:for] && opts[:for].ap_id in (object.data["announcements"] || [])
 
     attentions =
       activity.recipients
@@ -230,11 +232,11 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
 
     tags = if possibly_sensitive, do: Enum.uniq(["nsfw" | tags]), else: tags
 
-    {summary, content} = render_content(object)
+    {summary, content} = render_content(object.data)
 
     html =
       HTML.filter_tags(content, User.html_filter_policy(opts[:for]))
-      |> Formatter.emojify(object["emoji"])
+      |> Formatter.emojify(object.data["emoji"])
 
     reply_parent = Activity.get_in_reply_to_activity(activity)
 
@@ -249,19 +251,19 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do
       "is_local" => activity.local,
       "is_post_verb" => true,
       "created_at" => created_at,
-      "in_reply_to_status_id" => object["inReplyToStatusId"],
+      "in_reply_to_status_id" => object.data["inReplyToStatusId"],
       "in_reply_to_screen_name" => reply_user && reply_user.nickname,
       "in_reply_to_profileurl" => User.profile_url(reply_user),
       "in_reply_to_ostatus_uri" => reply_user && reply_user.ap_id,
       "in_reply_to_user_id" => reply_user && reply_user.id,
       "statusnet_conversation_id" => conversation_id,
-      "attachments" => (object["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts),
+      "attachments" => (object.data["attachment"] || []) |> ObjectRepresenter.enum_to_list(opts),
       "attentions" => attentions,
       "fave_num" => like_count,
       "repeat_num" => announcement_count,
       "favorited" => !!favorited,
       "repeated" => !!repeated,
-      "external_url" => object["external_url"] || object["id"],
+      "external_url" => object.data["external_url"] || object.data["id"],
       "tags" => tags,
       "activity_type" => "post",
       "possibly_sensitive" => possibly_sensitive,
-- 
cgit v1.2.3


From fed9b5404c1f30a695b60b67348604fc262c14c5 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 4 Dec 2018 03:17:25 +0000
Subject: object: rework Object.normalize() a bit to support transparent
 fetching

---
 lib/pleroma/object.ex         | 10 +++++++---
 lib/pleroma/object/fetcher.ex | 15 ++++++++++++---
 2 files changed, 19 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 57a8b1d6b..0e9aefb63 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -1,6 +1,7 @@
 defmodule Pleroma.Object do
   use Ecto.Schema
   alias Pleroma.{Repo, Object, Activity}
+  alias Pleroma.Object.Fetcher
   import Ecto.{Query, Changeset}
 
   schema "objects" do
@@ -27,9 +28,12 @@ defmodule Pleroma.Object do
     Repo.one(from(object in Object, where: fragment("(?)->>'id' = ?", object.data, ^ap_id)))
   end
 
-  def normalize(obj) when is_map(obj), do: normalize(obj["id"])
-  def normalize(ap_id) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
-  def normalize(_), do: nil
+  def normalize(_, fetch_remote \\ true)
+
+  def normalize(obj, fetch_remote) when is_map(obj), do: normalize(obj["id"], fetch_remote)
+  def normalize(ap_id, true) when is_binary(ap_id), do: Fetcher.fetch_object_from_id!(ap_id)
+  def normalize(ap_id, false) when is_binary(ap_id), do: get_cached_by_ap_id(ap_id)
+  def normalize(obj, _), do: nil
 
   if Mix.env() == :test do
     def get_cached_by_ap_id(ap_id) do
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index c27cb1577..c98722f39 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -17,7 +17,7 @@ defmodule Pleroma.Object.Fetcher do
       Logger.info("Fetching #{id} via AP")
 
       with {:ok, data} <- fetch_and_contain_remote_object_from_id(id),
-           nil <- Object.normalize(data),
+           nil <- Object.normalize(data, false),
            params <- %{
              "type" => "Create",
              "to" => data["to"],
@@ -27,7 +27,7 @@ defmodule Pleroma.Object.Fetcher do
            },
            :ok <- Containment.contain_origin(id, params),
            {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity.data["object"])}
+        {:ok, Object.normalize(activity.data["object"], false)}
       else
         {:error, {:reject, nil}} ->
           {:reject, nil}
@@ -39,13 +39,22 @@ defmodule Pleroma.Object.Fetcher do
           Logger.info("Couldn't get object via AP, trying out OStatus fetching...")
 
           case OStatus.fetch_activity_from_url(id) do
-            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"])}
+            {:ok, [activity | _]} -> {:ok, Object.normalize(activity.data["object"], false)}
             e -> e
           end
       end
     end
   end
 
+  def fetch_object_from_id!(id) do
+    with {:ok, object} <- fetch_object_from_id(id) do
+      object
+    else
+      _e ->
+        nil
+    end
+  end
+
   def fetch_and_contain_remote_object_from_id(id) do
     Logger.info("Fetching #{id} via AP")
 
-- 
cgit v1.2.3


From b3b52b58c3b31b22d3b3227d06d9a5336fa8edd0 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 4 Dec 2018 03:18:10 +0000
Subject: activitypub: transmogrifier: remove obsolete fetch_obj_helper()

---
 lib/pleroma/web/activity_pub/transmogrifier.ex | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index e76e29b95..c4567193f 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -85,7 +85,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
           ""
       end
 
-    case fetch_obj_helper(in_reply_to_id) do
+    case get_obj_helper(in_reply_to_id) do
       {:ok, replied_object} ->
         with %Activity{} = activity <-
                Activity.get_create_activity_by_object_ap_id(replied_object.data["id"]) do
@@ -363,7 +363,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.like(actor, object, id, false) do
       {:ok, activity}
     else
@@ -376,7 +376,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _object} <- ActivityPub.announce(actor, object, id, false) do
       {:ok, activity}
     else
@@ -430,7 +430,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
     with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          :ok <- Containment.contain_origin(actor.ap_id, object.data),
          {:ok, activity} <- ActivityPub.delete(object, false) do
       {:ok, activity}
@@ -449,7 +449,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _} <- ActivityPub.unannounce(actor, object, id, false) do
       {:ok, activity}
     else
@@ -519,7 +519,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
       ) do
     with actor <- Containment.get_actor(data),
          %User{} = actor <- User.get_or_fetch_by_ap_id(actor),
-         {:ok, object} <- get_obj_helper(object_id) || fetch_obj_helper(object_id),
+         {:ok, object} <- get_obj_helper(object_id),
          {:ok, activity, _, _} <- ActivityPub.unlike(actor, object, id, false) do
       {:ok, activity}
     else
@@ -529,9 +529,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def handle_incoming(_), do: :error
 
-  def fetch_obj_helper(id) when is_bitstring(id), do: Fetcher.fetch_object_from_id(id)
-  def fetch_obj_helper(obj) when is_map(obj), do: Fetcher.fetch_object_from_id(obj["id"])
-
   def get_obj_helper(id) do
     if object = Object.normalize(id), do: {:ok, object}, else: nil
   end
@@ -629,7 +626,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
 
   def maybe_fix_object_url(data) do
     if is_binary(data["object"]) and not String.starts_with?(data["object"], "http") do
-      case fetch_obj_helper(data["object"]) do
+      case get_obj_helper(data["object"]) do
         {:ok, relative_object} ->
           if relative_object.data["external_url"] do
             _data =
-- 
cgit v1.2.3


From d6ab701a14f7c9fb4d59953648c425e04725fc62 Mon Sep 17 00:00:00 2001
From: William Pitcock <nenolod@dereferenced.org>
Date: Tue, 4 Dec 2018 05:01:21 +0000
Subject: formatting

---
 lib/pleroma/web/ostatus/handlers/note_handler.ex | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/ostatus/handlers/note_handler.ex b/lib/pleroma/web/ostatus/handlers/note_handler.ex
index 39004367a..ba232b0ec 100644
--- a/lib/pleroma/web/ostatus/handlers/note_handler.ex
+++ b/lib/pleroma/web/ostatus/handlers/note_handler.ex
@@ -106,7 +106,8 @@ defmodule Pleroma.Web.OStatus.NoteHandler do
          cw <- OStatus.get_cw(entry),
          inReplyTo <- XML.string_from_xpath("//thr:in-reply-to[1]/@ref", entry),
          inReplyToActivity <- fetch_replied_to_activity(entry, inReplyTo),
-         inReplyToObject <- (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil,
+         inReplyToObject <-
+           (inReplyToActivity && Object.normalize(inReplyToActivity.data["object"])) || nil,
          inReplyTo <- (inReplyToObject && inReplyToObject.data["id"]) || inReplyTo,
          attachments <- OStatus.get_attachments(entry),
          context <- get_context(entry, inReplyTo),
-- 
cgit v1.2.3


From 462028688b7050bb2335914b0987632082fdf3c8 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 12:34:19 +0300
Subject: Fix pinned posts relying on embded objects

---
 lib/pleroma/web/common_api/common_api.ex | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index 9c3daac2c..d6eb843f7 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -217,8 +217,10 @@ defmodule Pleroma.Web.CommonAPI do
     with %Activity{
            actor: ^user_ap_id,
            data: %{
-             "type" => "Create",
-             "object" => %{
+             "type" => "Create"
+           },
+           object: %Object{
+             data: %{
                "to" => object_to,
                "type" => "Note"
              }
-- 
cgit v1.2.3


From b09ae02c04d66e58c2bcc6ce10277c88d5fed576 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 13:04:58 +0300
Subject: Added some more normalization calls all in mastodon api controller

---
 lib/pleroma/web/common_api/common_api.ex                | 2 +-
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 5 +++--
 lib/pleroma/web/mastodon_api/views/status_view.ex       | 8 ++++----
 3 files changed, 8 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex
index d6eb843f7..6458a3449 100644
--- a/lib/pleroma/web/common_api/common_api.ex
+++ b/lib/pleroma/web/common_api/common_api.ex
@@ -127,7 +127,7 @@ defmodule Pleroma.Web.CommonAPI do
       in_reply_to ->
         # XXX: these heuristics should be moved out of MastodonAPI.
         with %Object{} = object <- Object.normalize(in_reply_to) do
-          Pleroma.Web.MastodonAPI.StatusView.get_visibility(object.data)
+          Pleroma.Web.MastodonAPI.StatusView.get_visibility(object)
         end
     end
   end
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 24a2d4cb9..d2e3da449 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -543,10 +543,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def bookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with %Activity{} = activity <- Activity.get_by_id(id),
+    with %Activity{} = activity <- Activity.get_by_id_with_object(id),
+         %Object{} = object <- Object.normalize(activity),
          %User{} = user <- User.get_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
-         {:ok, user} <- User.bookmark(user, activity.data["object"]["id"]) do
+         {:ok, user} <- User.bookmark(user, object.data["id"]) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index e4de5ecfb..03dc587d9 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -125,8 +125,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     }
   end
 
-  def render("status.json", %{activity: %{data: %{"object" => object}} = activity} = opts) do
-    object = Object.normalize(object)
+  def render("status.json", %{activity: %{data: %{"object" => _object}} = activity} = opts) do
+    object = Object.normalize(activity)
 
     user = get_user(activity.data["actor"])
 
@@ -320,8 +320,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
     end
   end
 
-  def get_reply_to(%{data: %{"object" => object}}, _) do
-    object = Object.normalize(object)
+  def get_reply_to(%{data: %{"object" => _object}} = activity, _) do
+    object = Object.normalize(activity)
 
     if object.data["inReplyTo"] && object.data["inReplyTo"] != "" do
       Activity.get_create_by_object_ap_id(object.data["inReplyTo"])
-- 
cgit v1.2.3


From e7c3c367667e96ef7fe31ef9dd8337b563a3ccaa Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 14:21:39 +0300
Subject: Update functions in object fetcher for tesla and set up a proper mock
 for tests

---
 lib/pleroma/object/fetcher.ex | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 19d9c51af..138e7866f 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -27,7 +27,7 @@ defmodule Pleroma.Object.Fetcher do
            },
            :ok <- Containment.contain_origin(id, params),
            {:ok, activity} <- Transmogrifier.handle_incoming(params) do
-        {:ok, Object.normalize(activity.data["object"], false)}
+        {:ok, Object.normalize(activity, false)}
       else
         {:error, {:reject, nil}} ->
           {:reject, nil}
@@ -56,16 +56,13 @@ defmodule Pleroma.Object.Fetcher do
   end
 
   def fetch_and_contain_remote_object_from_id(id) do
-    Logger.info("Fetching #{id} via AP")
+    Logger.info("Fetching object #{id} via AP")
 
     with true <- String.starts_with?(id, "http"),
-         {:ok, %{body: body, status_code: code}} when code in 200..299 <-
+         {:ok, %{body: body, status: code}} when code in 200..299 <-
            @httpoison.get(
              id,
-             [Accept: "application/activity+json"],
-             follow_redirect: true,
-             timeout: 10000,
-             recv_timeout: 20000
+             [{:Accept, "application/activity+json"}]
            ),
          {:ok, data} <- Jason.decode(body),
          :ok <- Containment.contain_origin_from_id(id, data) do
-- 
cgit v1.2.3


From e641651e2b0ba9e6c4d0a99ebf4f7c3b25f7ce67 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 14:27:02 +0300
Subject: Fix unbookmarking

---
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index d2e3da449..1e82b2f68 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -555,10 +555,11 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   end
 
   def unbookmark_status(%{assigns: %{user: user}} = conn, %{"id" => id}) do
-    with %Activity{} = activity <- Activity.get_by_id(id),
+    with %Activity{} = activity <- Activity.get_by_id_with_object(id),
+         %Object{} = object <- Object.normalize(activity),
          %User{} = user <- User.get_by_nickname(user.nickname),
          true <- Visibility.visible_for_user?(activity, user),
-         {:ok, user} <- User.unbookmark(user, activity.data["object"]["id"]) do
+         {:ok, user} <- User.unbookmark(user, object.data["id"]) do
       conn
       |> put_view(StatusView)
       |> try_render("status.json", %{activity: activity, for: user, as: :activity})
-- 
cgit v1.2.3


From ad681877df46c151ee20b58401bda9f84d884109 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 14:52:01 +0300
Subject: Make credo happy

---
 lib/pleroma/web/activity_pub/transmogrifier.ex          | 5 ++---
 lib/pleroma/web/common_api/utils.ex                     | 8 ++++----
 lib/pleroma/web/federator/federator.ex                  | 2 +-
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 2 +-
 lib/pleroma/web/mastodon_api/views/status_view.ex       | 2 +-
 5 files changed, 9 insertions(+), 10 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex
index 0637b18dc..a80aa52c6 100644
--- a/lib/pleroma/web/activity_pub/transmogrifier.ex
+++ b/lib/pleroma/web/activity_pub/transmogrifier.ex
@@ -6,13 +6,12 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do
   @moduledoc """
   A module to handle coding from internal to wire ActivityPub and back.
   """
-  alias Pleroma.User
-  alias Pleroma.Object
-  alias Pleroma.Object.Containment
   alias Pleroma.Activity
   alias Pleroma.Object
+  alias Pleroma.Object.Containment
   alias Pleroma.Repo
   alias Pleroma.User
+  alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Utils
   alias Pleroma.Web.ActivityPub.Visibility
diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 7781f1635..50a72aee5 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -208,7 +208,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
         context,
         content_html,
         attachments,
-        inReplyTo,
+        in_reply_to,
         tags,
         cw \\ nil,
         cc \\ []
@@ -225,11 +225,11 @@ defmodule Pleroma.Web.CommonAPI.Utils do
       "tag" => tags |> Enum.map(fn {_, tag} -> tag end) |> Enum.uniq()
     }
 
-    if inReplyTo do
-      inReplyToObject = Object.normalize(inReplyTo.data["object"])
+    if in_reply_to do
+      in_reply_to_object = Object.normalize(inReplyTo.data["object"])
 
       object
-      |> Map.put("inReplyTo", inReplyToObject.data["id"])
+      |> Map.put("inReplyTo", in_reply_to_object.data["id"])
     else
       object
     end
diff --git a/lib/pleroma/web/federator/federator.ex b/lib/pleroma/web/federator/federator.ex
index a1f6373a4..1b4deb6dc 100644
--- a/lib/pleroma/web/federator/federator.ex
+++ b/lib/pleroma/web/federator/federator.ex
@@ -4,6 +4,7 @@
 
 defmodule Pleroma.Web.Federator do
   alias Pleroma.Activity
+  alias Pleroma.Object.Containment
   alias Pleroma.User
   alias Pleroma.Web.ActivityPub.ActivityPub
   alias Pleroma.Web.ActivityPub.Relay
@@ -12,7 +13,6 @@ defmodule Pleroma.Web.Federator do
   alias Pleroma.Web.ActivityPub.Visibility
   alias Pleroma.Web.Federator.RetryQueue
   alias Pleroma.Web.OStatus
-  alias Pleroma.Object.Containment
   alias Pleroma.Web.Salmon
   alias Pleroma.Web.WebFinger
   alias Pleroma.Web.Websub
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 1e82b2f68..4cec26c9b 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -4,13 +4,13 @@
 
 defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
   use Pleroma.Web, :controller
-  alias Pleroma.Object.Fetcher
   alias Ecto.Changeset
   alias Pleroma.Activity
   alias Pleroma.Config
   alias Pleroma.Filter
   alias Pleroma.Notification
   alias Pleroma.Object
+  alias Pleroma.Object.Fetcher
   alias Pleroma.Pagination
   alias Pleroma.Repo
   alias Pleroma.ScheduledActivity
diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex
index 03dc587d9..f8961eb6c 100644
--- a/lib/pleroma/web/mastodon_api/views/status_view.ex
+++ b/lib/pleroma/web/mastodon_api/views/status_view.ex
@@ -7,8 +7,8 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do
 
   alias Pleroma.Activity
   alias Pleroma.HTML
-  alias Pleroma.Repo
   alias Pleroma.Object
+  alias Pleroma.Repo
   alias Pleroma.User
   alias Pleroma.Web.CommonAPI
   alias Pleroma.Web.CommonAPI.Utils
-- 
cgit v1.2.3


From a53b917e7ff2df74f53f1f4578c212fe539acb00 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 14:55:26 +0300
Subject: oof

---
 lib/pleroma/web/common_api/utils.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex
index 50a72aee5..25f498fcb 100644
--- a/lib/pleroma/web/common_api/utils.ex
+++ b/lib/pleroma/web/common_api/utils.ex
@@ -226,7 +226,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do
     }
 
     if in_reply_to do
-      in_reply_to_object = Object.normalize(inReplyTo.data["object"])
+      in_reply_to_object = Object.normalize(in_reply_to.data["object"])
 
       object
       |> Map.put("inReplyTo", in_reply_to_object.data["id"])
-- 
cgit v1.2.3


From c3a20528060e4fb95292dd93768d9afc8926e66e Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 15:11:22 +0300
Subject: If it's an object struct it is already normalized

---
 lib/pleroma/object.ex | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 3f1d0fea1..740d687a3 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -44,6 +44,7 @@ defmodule Pleroma.Object do
   def normalize(_, fetch_remote \\ true)
   # If we pass an Activity to Object.normalize(), we can try to use the preloaded object.
   # Use this whenever possible, especially when walking graphs in an O(N) loop!
+  def normalize(%Object{} = object, _), do: object
   def normalize(%Activity{object: %Object{} = object}, _), do: object
 
   # A hack for fake activities
-- 
cgit v1.2.3


From 8e4d950f31ec3ea956f6892f9f36b419344bf930 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 15:54:09 +0300
Subject: Remove updating reply count for embeded objects

---
 lib/pleroma/activity.ex                      | 50 ----------------------------
 lib/pleroma/web/activity_pub/activity_pub.ex |  2 --
 2 files changed, 52 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 99cc9c077..478d16356 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -257,54 +257,4 @@ defmodule Pleroma.Activity do
     |> where([s], s.actor == ^actor)
     |> Repo.all()
   end
-
-  def increase_replies_count(nil), do: nil
-
-  def increase_replies_count(object_ap_id) do
-    from(a in create_by_object_ap_id(object_ap_id),
-      update: [
-        set: [
-          data:
-            fragment(
-              """
-              jsonb_set(?, '{object, repliesCount}',
-                (coalesce((?->'object'->>'repliesCount')::int, 0) + 1)::varchar::jsonb, true)
-              """,
-              a.data,
-              a.data
-            )
-        ]
-      ]
-    )
-    |> Repo.update_all([])
-    |> case do
-      {1, [activity]} -> activity
-      _ -> {:error, "Not found"}
-    end
-  end
-
-  def decrease_replies_count(nil), do: nil
-
-  def decrease_replies_count(object_ap_id) do
-    from(a in create_by_object_ap_id(object_ap_id),
-      update: [
-        set: [
-          data:
-            fragment(
-              """
-              jsonb_set(?, '{object, repliesCount}',
-                (greatest(0, (?->'object'->>'repliesCount')::int - 1))::varchar::jsonb, true)
-              """,
-              a.data,
-              a.data
-            )
-        ]
-      ]
-    )
-    |> Repo.update_all([])
-    |> case do
-      {1, [activity]} -> activity
-      _ -> {:error, "Not found"}
-    end
-  end
 end
diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 1a3b47cb3..28fca6116 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -95,7 +95,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         "type" => "Create"
       }) do
     if is_public?(object) do
-      Activity.increase_replies_count(reply_ap_id)
       Object.increase_replies_count(reply_ap_id)
     end
   end
@@ -106,7 +105,6 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
         data: %{"inReplyTo" => reply_ap_id} = object
       }) do
     if is_public?(object) do
-      Activity.decrease_replies_count(reply_ap_id)
       Object.decrease_replies_count(reply_ap_id)
     end
   end
-- 
cgit v1.2.3


From 4c289e924e2ef7863a2c95b74f71fd83969b7827 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 16:35:01 +0300
Subject: Fix delete-by_ap_id to expect not only embeded objects

---
 lib/pleroma/activity.ex | 1 +
 1 file changed, 1 insertion(+)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 478d16356..9043530c9 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -230,6 +230,7 @@ defmodule Pleroma.Activity do
     |> Repo.delete_all()
     |> elem(1)
     |> Enum.find(fn
+      %{data: %{"type" => "Create", "object" => ap_id}} when is_binary(ap_id) -> ap_id == id
       %{data: %{"type" => "Create", "object" => %{"id" => ap_id}}} -> ap_id == id
       _ -> nil
     end)
-- 
cgit v1.2.3


From 35ac672b8d4a6711754a5f88ad65e52d356c4c67 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Wed, 17 Apr 2019 17:59:15 +0300
Subject: Remove containment tests from transmogrifier and fix thread
 visibility solver

---
 lib/pleroma/object/containment.ex          |  3 ---
 lib/pleroma/web/activity_pub/visibility.ex | 21 ++++++++++++---------
 2 files changed, 12 insertions(+), 12 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/object/containment.ex b/lib/pleroma/object/containment.ex
index 27e89d87f..25bd911fb 100644
--- a/lib/pleroma/object/containment.ex
+++ b/lib/pleroma/object/containment.ex
@@ -9,9 +9,6 @@ defmodule Pleroma.Object.Containment do
   Object containment is an important step in validating remote objects to prevent
   spoofing, therefore removal of object containment functions is NOT recommended.
   """
-
-  require Logger
-
   def get_actor(%{"actor" => actor}) when is_binary(actor) do
     actor
   end
diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex
index db52fe933..3da709b3d 100644
--- a/lib/pleroma/web/activity_pub/visibility.ex
+++ b/lib/pleroma/web/activity_pub/visibility.ex
@@ -41,16 +41,19 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
   # guard
   def entire_thread_visible_for_user?(nil, _user), do: false
 
-  # child
+  # XXX: Probably even more inefficient than the previous implementation, intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop
   def entire_thread_visible_for_user?(
-        %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
+        %Activity{} = tail,
+        # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
         user
-      )
-      when is_binary(parent_id) do
-    parent = Activity.get_in_reply_to_activity(tail)
-    visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
+      ) do
+    case Object.normalize(tail) do
+      %{data: %{"inReplyTo" => parent_id}} when is_binary(parent_id) ->
+        parent = Activity.get_in_reply_to_activity(tail)
+        visible_for_user?(tail, user) && entire_thread_visible_for_user?(parent, user)
+
+      _ ->
+        visible_for_user?(tail, user)
+    end
   end
-
-  # root
-  def entire_thread_visible_for_user?(tail, user), do: visible_for_user?(tail, user)
 end
-- 
cgit v1.2.3


From 2abc09570f40352e949f0142d11778f89a70c920 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 18 Apr 2019 01:37:04 +0300
Subject: Use the preloaded object in tag queries

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 28fca6116..6b2fb17a4 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -574,8 +574,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_tag_reject(query, %{"tag_reject" => tag_reject})
        when is_list(tag_reject) and tag_reject != [] do
     from(
-      activity in query,
-      where: fragment(~s(\(not \(? #> '{"object","tag"}'\) \\?| ?\)), activity.data, ^tag_reject)
+      [_activity, object] in query,
+      where: fragment("not (?)->'tag' \\?| (?)", object.data, ^tag_reject)
     )
   end
 
@@ -584,8 +584,8 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   defp restrict_tag_all(query, %{"tag_all" => tag_all})
        when is_list(tag_all) and tag_all != [] do
     from(
-      activity in query,
-      where: fragment(~s(\(? #> '{"object","tag"}'\) \\?& ?), activity.data, ^tag_all)
+      [_activity, object] in query,
+      where: fragment("(?)->'tag' \\?& (?)", object.data, ^tag_all)
     )
   end
 
@@ -593,15 +593,15 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do
     from(
-      activity in query,
-      where: fragment(~s(\(? #> '{"object","tag"}'\) \\?| ?), activity.data, ^tag)
+      [_activity, object] in query,
+      where: fragment("(?)->'tag' \\?| (?)", object.data, ^tag)
     )
   end
 
   defp restrict_tag(query, %{"tag" => tag}) when is_binary(tag) do
     from(
-      activity in query,
-      where: fragment(~s(? <@ (? #> '{"object","tag"}'\)), ^tag, activity.data)
+      [_activity, object] in query,
+      where: fragment("(?)->'tag' \\? (?)", object.data, ^tag)
     )
   end
 
-- 
cgit v1.2.3


From e31a22043bcf74d1d85f6fe007bd4606291d41e9 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 18 Apr 2019 08:31:08 +0300
Subject: Fix media timeline depending on embeded object and add some guards

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 6b2fb17a4..03be8b06f 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -571,6 +571,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_since(query, _), do: query
 
+  defp restrict_tag_reject(_query, %{"tag_reject" => _tag_reject, "skip_preload" => true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
   defp restrict_tag_reject(query, %{"tag_reject" => tag_reject})
        when is_list(tag_reject) and tag_reject != [] do
     from(
@@ -581,6 +585,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_tag_reject(query, _), do: query
 
+  defp restrict_tag_all(_query, %{"tag_all" => _tag_all, "skip_preload" => true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
   defp restrict_tag_all(query, %{"tag_all" => tag_all})
        when is_list(tag_all) and tag_all != [] do
     from(
@@ -591,6 +599,10 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_tag_all(query, _), do: query
 
+  defp restrict_tag(_query, %{"tag" => _tag, "skip_preload" => true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
   defp restrict_tag(query, %{"tag" => tag}) when is_list(tag) do
     from(
       [_activity, object] in query,
@@ -666,10 +678,14 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
 
   defp restrict_favorited_by(query, _), do: query
 
+  defp restrict_media(_query, %{"only_media" => _val, "skip_preload" => true}) do
+    raise "Can't use the child object without preloading!"
+  end
+
   defp restrict_media(query, %{"only_media" => val}) when val == "true" or val == "1" do
     from(
-      activity in query,
-      where: fragment(~s(not (? #> '{"object","attachment"}' = ?\)), activity.data, ^[])
+      [_activity, object] in query,
+      where: fragment("not (?)->'attachment' = (?)", object.data, ^[])
     )
   end
 
-- 
cgit v1.2.3


From 6069d0fd361a9971f2297d3babc596d60634eb43 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 18 Apr 2019 09:28:20 +0300
Subject: Fix object search depending on embeded object

---
 lib/pleroma/web/activity_pub/visibility.ex              | 4 +++-
 lib/pleroma/web/mastodon_api/mastodon_api_controller.ex | 6 +++---
 2 files changed, 6 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/visibility.ex b/lib/pleroma/web/activity_pub/visibility.ex
index 3da709b3d..6dee61dd6 100644
--- a/lib/pleroma/web/activity_pub/visibility.ex
+++ b/lib/pleroma/web/activity_pub/visibility.ex
@@ -41,7 +41,9 @@ defmodule Pleroma.Web.ActivityPub.Visibility do
   # guard
   def entire_thread_visible_for_user?(nil, _user), do: false
 
-  # XXX: Probably even more inefficient than the previous implementation, intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop
+  # XXX: Probably even more inefficient than the previous implementation intended to be a placeholder untill https://git.pleroma.social/pleroma/pleroma/merge_requests/971 is in develop
+  # credo:disable-for-previous-line Credo.Check.Readability.MaxLineLength
+
   def entire_thread_visible_for_user?(
         %Activity{} = tail,
         # %Activity{data: %{"object" => %{"inReplyTo" => parent_id}}} = tail,
diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
index 4cec26c9b..3916d7c41 100644
--- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
+++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex
@@ -1012,13 +1012,13 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do
 
     q =
       from(
-        a in Activity,
+        [a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
         where:
           fragment(
-            "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)",
-            a.data,
+            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+            o.data,
             ^query
           ),
         limit: 20,
-- 
cgit v1.2.3


From ac04311b3f0a611b5008747037d6cd5874fa3ae9 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 18 Apr 2019 10:24:06 +0300
Subject: Fix search in TwitterAPI

---
 lib/pleroma/web/twitter_api/twitter_api.ex | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/twitter_api/twitter_api.ex b/lib/pleroma/web/twitter_api/twitter_api.ex
index d6ce0a7c6..8e44dbeb8 100644
--- a/lib/pleroma/web/twitter_api/twitter_api.ex
+++ b/lib/pleroma/web/twitter_api/twitter_api.ex
@@ -266,6 +266,7 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
   defp parse_int(_, default), do: default
 
+  # TODO: unify the search query with MastoAPI one and do only pagination here
   def search(_user, %{"q" => query} = params) do
     limit = parse_int(params["rpp"], 20)
     page = parse_int(params["page"], 1)
@@ -273,13 +274,13 @@ defmodule Pleroma.Web.TwitterAPI.TwitterAPI do
 
     q =
       from(
-        a in Activity,
+        [a, o] in Activity.with_preloaded_object(Activity),
         where: fragment("?->>'type' = 'Create'", a.data),
         where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients,
         where:
           fragment(
-            "to_tsvector('english', ?->'object'->>'content') @@ plainto_tsquery('english', ?)",
-            a.data,
+            "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)",
+            o.data,
             ^query
           ),
         limit: ^limit,
-- 
cgit v1.2.3


From a11ca87f40fd85341afa4445d3b7303ae8e92b76 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 18 Apr 2019 13:10:38 +0300
Subject: Add a migration to remove embeded objects

---
 lib/mix/tasks/compact_database.ex | 57 ---------------------------------------
 1 file changed, 57 deletions(-)
 delete mode 100644 lib/mix/tasks/compact_database.ex

(limited to 'lib')

diff --git a/lib/mix/tasks/compact_database.ex b/lib/mix/tasks/compact_database.ex
deleted file mode 100644
index 17b9721f7..000000000
--- a/lib/mix/tasks/compact_database.ex
+++ /dev/null
@@ -1,57 +0,0 @@
-defmodule Mix.Tasks.CompactDatabase do
-  @moduledoc """
-  Compact the database by flattening the object graph.
-  """
-
-  require Logger
-
-  use Mix.Task
-  import Ecto.Query
-  alias Pleroma.Activity
-  alias Pleroma.Repo
-
-  defp maybe_compact(%Activity{data: %{"object" => %{"id" => object_id}}} = activity) do
-    data =
-      activity.data
-      |> Map.put("object", object_id)
-
-    {:ok, activity} =
-      Activity.change(activity, %{data: data})
-      |> Repo.update()
-
-    {:ok, activity}
-  end
-
-  defp maybe_compact(%Activity{} = activity), do: {:ok, activity}
-
-  defp activity_query(min_id, max_id) do
-    from(
-      a in Activity,
-      where: fragment("?->>'type' = 'Create'", a.data),
-      where: a.id >= ^min_id,
-      where: a.id < ^max_id
-    )
-  end
-
-  def run(_args) do
-    Application.ensure_all_started(:pleroma)
-
-    max = Repo.aggregate(Activity, :max, :id)
-    Logger.info("Considering #{max} activities")
-
-    chunks = 0..round(max / 100)
-
-    Enum.each(chunks, fn i ->
-      min = i * 100
-      max = min + 100
-
-      activity_query(min, max)
-      |> Repo.all()
-      |> Enum.each(&maybe_compact/1)
-
-      IO.write(".")
-    end)
-
-    Logger.info("Finished.")
-  end
-end
-- 
cgit v1.2.3


From 9238dccac1310fc2e281b242768a2de79f405f35 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 18 Apr 2019 21:40:40 +0300
Subject: Add a guard to fetching reply activity

---
 lib/pleroma/activity.ex | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex
index 9043530c9..4a2ded518 100644
--- a/lib/pleroma/activity.ex
+++ b/lib/pleroma/activity.ex
@@ -204,12 +204,14 @@ defmodule Pleroma.Activity do
 
   def create_by_object_ap_id_with_object(_), do: nil
 
-  def get_create_by_object_ap_id_with_object(ap_id) do
+  def get_create_by_object_ap_id_with_object(ap_id) when is_binary(ap_id) do
     ap_id
     |> create_by_object_ap_id_with_object()
     |> Repo.one()
   end
 
+  def get_create_by_object_ap_id_with_object(_), do: nil
+
   defp get_in_reply_to_activity_from_object(%Object{data: %{"inReplyTo" => ap_id}}) do
     get_create_by_object_ap_id_with_object(ap_id)
   end
-- 
cgit v1.2.3


From 099f89367efaf4032b8e937258b2c1a90f16b047 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 18 Apr 2019 23:34:01 +0300
Subject: Replace embedded object migration with a mix task

---
 lib/mix/tasks/pleroma/database.ex | 45 +++++++++++++++++++++++++++++++++++++++
 1 file changed, 45 insertions(+)
 create mode 100644 lib/mix/tasks/pleroma/database.ex

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
new file mode 100644
index 000000000..ce3252af5
--- /dev/null
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -0,0 +1,45 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Mix.Tasks.Pleroma.Database do
+  alias Mix.Tasks.Pleroma.Common
+  use Mix.Task
+
+  @shortdoc "A collection of database related tasks"
+  @moduledoc """
+   A collection of database related tasks
+
+   ## Replace embedded objects with their references
+
+   Replaces embedded objects with references to them in the `objects` table. Only needs to be ran once. The reason why this is not a migration is because it could significantly increase the database size after being ran, however after this `VACUUM FULL` will be able to reclaim about 20% (really depends on what is in the database, your mileage may vary) of the db size before the migration.
+
+       mix pleroma.database remove_embedded_objects
+
+    Options:
+    - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
+  """
+  def run(["remove_embedded_objects" | args]) do
+    {options, [], []} =
+      OptionParser.parse(
+        args,
+        strict: [
+          vacuum: :boolean
+        ]
+      )
+
+    Common.start_pleroma()
+
+    Ecto.Adapters.SQL.query!(
+      Pleroma.Repo,
+      "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;"
+    )
+
+    if Keyword.get(options, :vacuum) do
+      Ecto.Adapters.SQL.query!(
+        Pleroma.Repo,
+        "vacuum full;"
+      )
+    end
+  end
+end
-- 
cgit v1.2.3


From 945325013af6dde3f1da2417753bb97f55911a84 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Thu, 18 Apr 2019 23:58:59 +0300
Subject: remove query timeouts

---
 lib/mix/tasks/pleroma/database.ex | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index ce3252af5..d657c1ef0 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -30,15 +30,17 @@ defmodule Mix.Tasks.Pleroma.Database do
 
     Common.start_pleroma()
 
-    Ecto.Adapters.SQL.query!(
-      Pleroma.Repo,
-      "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;"
+    Pleroma.Repo.query!(
+      "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
+      [],
+      timeout: :infinity
     )
 
     if Keyword.get(options, :vacuum) do
-      Ecto.Adapters.SQL.query!(
-        Pleroma.Repo,
-        "vacuum full;"
+      Pleroma.Repo.query!(
+        "vacuum full;",
+        [],
+        timeout: :infinity
       )
     end
   end
-- 
cgit v1.2.3


From 73b8c5387b25caaf27734f7018dc4702d49af7de Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 19 Apr 2019 00:17:37 +0300
Subject: Add some logging

---
 lib/mix/tasks/pleroma/database.ex | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'lib')

diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index d657c1ef0..ab9a3a7ff 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -4,6 +4,7 @@
 
 defmodule Mix.Tasks.Pleroma.Database do
   alias Mix.Tasks.Pleroma.Common
+  require Logger
   use Mix.Task
 
   @shortdoc "A collection of database related tasks"
@@ -29,6 +30,7 @@ defmodule Mix.Tasks.Pleroma.Database do
       )
 
     Common.start_pleroma()
+    Logger.info("Removing embedded objects")
 
     Pleroma.Repo.query!(
       "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;",
@@ -37,6 +39,8 @@ defmodule Mix.Tasks.Pleroma.Database do
     )
 
     if Keyword.get(options, :vacuum) do
+      Logger.info("Runnning VACUUM FULL")
+
       Pleroma.Repo.query!(
         "vacuum full;",
         [],
-- 
cgit v1.2.3


From f9865cf9439e2e9273f55d3b82c1e68166178b07 Mon Sep 17 00:00:00 2001
From: rinpatch <rinpatch@sdf.org>
Date: Fri, 19 Apr 2019 00:47:02 +0300
Subject: Stream out deletes, mistakingly removed when resolving merge
 conflicts

---
 lib/pleroma/web/activity_pub/activity_pub.ex | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'lib')

diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex
index 9e2574419..0b99a169c 100644
--- a/lib/pleroma/web/activity_pub/activity_pub.ex
+++ b/lib/pleroma/web/activity_pub/activity_pub.ex
@@ -167,7 +167,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do
   def stream_out(activity) do
     public = "https://www.w3.org/ns/activitystreams#Public"
 
-    if activity.data["type"] in ["Create", "Announce"] do
+    if activity.data["type"] in ["Create", "Announce", "Delete"] do
       object = Object.normalize(activity.data["object"])
       Pleroma.Web.Streamer.stream("user", activity)
       Pleroma.Web.Streamer.stream("list", activity)
-- 
cgit v1.2.3