From 06a3998013aca1f74c563d261d050543056c1255 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sun, 5 Jun 2022 15:02:25 -0400 Subject: Create Update notifications --- ...0605185734_add_update_to_notifications_enum.exs | 51 ++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs (limited to 'priv/repo/migrations') diff --git a/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs b/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs new file mode 100644 index 000000000..0656c885f --- /dev/null +++ b/priv/repo/migrations/20220605185734_add_update_to_notifications_enum.exs @@ -0,0 +1,51 @@ +defmodule Pleroma.Repo.Migrations.AddUpdateToNotificationsEnum do + use Ecto.Migration + + @disable_ddl_transaction true + + def up do + """ + alter type notification_type add value 'update' + """ + |> execute() + end + + # 20210717000000_add_poll_to_notifications_enum.exs + def down do + alter table(:notifications) do + modify(:type, :string) + end + + """ + delete from notifications where type = 'update' + """ + |> execute() + + """ + drop type if exists notification_type + """ + |> execute() + + """ + create type notification_type as enum ( + 'follow', + 'follow_request', + 'mention', + 'move', + 'pleroma:emoji_reaction', + 'pleroma:chat_mention', + 'reblog', + 'favourite', + 'pleroma:report', + 'poll' + ) + """ + |> execute() + + """ + alter table notifications + alter column type type notification_type using (type::notification_type) + """ + |> execute() + end +end -- cgit v1.2.3 From c559c240d1a56f05fc70f69ae6b8c0809026fa2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Sun, 7 Aug 2022 20:41:24 +0200 Subject: Migrations: delete context objects These objects represent from 30 to 70% of the rows on the objects table, based on numbers from a few live instances (single-user, small, large.) As those pseudo-objects prevent creating objects with those actual IDs, deleting them is a better solution. This could have happened if an object used another object's ID as its context. --- ...220807125023_data_migration_delete_context_objects.exs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs (limited to 'priv/repo/migrations') diff --git a/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs b/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs new file mode 100644 index 000000000..debb474b2 --- /dev/null +++ b/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs @@ -0,0 +1,15 @@ +defmodule Pleroma.Repo.Migrations.DataMigrationDeleteContextObjects do + use Ecto.Migration + + require Logger + + @doc "This migration removes objects created exclusively for contexts, containing only an `id` field." + + def change do + Logger.warn( + "This migration can take a very long time to execute, depending on your database size. Please be patient, Pleroma-tan is doing her best!\n" + ) + + execute("DELETE FROM objects WHERE (data->>'type') IS NULL;") + end +end -- cgit v1.2.3 From 88c1c76d3eca3412d1e02008f1b8d96fe8fe0b96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9l=C3=A8ne?= Date: Mon, 15 Aug 2022 01:15:23 +0200 Subject: Migrations: delete contexts with BaseMigrator Due to the lengthiness of this task, the migration has been adapted into a BaseMigrator migration, running in the background instead. --- ...20220807125023_data_migration_delete_context_objects.exs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'priv/repo/migrations') diff --git a/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs b/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs index debb474b2..84365dbe3 100644 --- a/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs +++ b/priv/repo/migrations/20220807125023_data_migration_delete_context_objects.exs @@ -3,13 +3,16 @@ defmodule Pleroma.Repo.Migrations.DataMigrationDeleteContextObjects do require Logger - @doc "This migration removes objects created exclusively for contexts, containing only an `id` field." + def up do + dt = NaiveDateTime.utc_now() - def change do - Logger.warn( - "This migration can take a very long time to execute, depending on your database size. Please be patient, Pleroma-tan is doing her best!\n" + execute( + "INSERT INTO data_migrations(name, inserted_at, updated_at) " <> + "VALUES ('delete_context_objects', '#{dt}', '#{dt}') ON CONFLICT DO NOTHING;" ) + end - execute("DELETE FROM objects WHERE (data->>'type') IS NULL;") + def down do + execute("DELETE FROM data_migrations WHERE name = 'delete_context_objects';") end end -- cgit v1.2.3 From 06678fb4ad42fcaecb99eccc2237c3b863a2b9a5 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 11 Jul 2022 14:58:38 -0400 Subject: Add function to calculate associated object id --- ...711182322_add_associated_object_id_function.exs | 37 ++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 priv/repo/migrations/20220711182322_add_associated_object_id_function.exs (limited to 'priv/repo/migrations') diff --git a/priv/repo/migrations/20220711182322_add_associated_object_id_function.exs b/priv/repo/migrations/20220711182322_add_associated_object_id_function.exs new file mode 100644 index 000000000..76348f31a --- /dev/null +++ b/priv/repo/migrations/20220711182322_add_associated_object_id_function.exs @@ -0,0 +1,37 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.AddAssociatedObjectIdFunction do + use Ecto.Migration + + def up do + statement = """ + CREATE OR REPLACE FUNCTION associated_object_id(data jsonb) RETURNS varchar AS $$ + DECLARE + object_data jsonb; + BEGIN + IF jsonb_typeof(data->'object') = 'array' THEN + object_data := data->'object'->0; + ELSE + object_data := data->'object'; + END IF; + + IF jsonb_typeof(object_data->'id') = 'string' THEN + RETURN object_data->>'id'; + ELSIF jsonb_typeof(object_data) = 'string' THEN + RETURN object_data#>>'{}'; + ELSE + RETURN NULL; + END IF; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + + execute(statement) + end + + def down do + execute("DROP FUNCTION IF EXISTS associated_object_id(data jsonb)") + end +end -- cgit v1.2.3 From 3885ee182a572a10b326ae553703ee0d38f3b66d Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 11 Jul 2022 15:49:58 -0400 Subject: Switch to associated_object_id index --- ...192750_switch_to_associated_object_id_index.exs | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs (limited to 'priv/repo/migrations') diff --git a/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs b/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs new file mode 100644 index 000000000..c0b89731b --- /dev/null +++ b/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs @@ -0,0 +1,39 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.SwitchToAssociatedObjectIdIndex do + use Ecto.Migration + @disable_ddl_transaction true + @disable_migration_lock true + + def up do + drop_if_exists( + index(:activities, ["(coalesce(data->'object'->>'id', data->>'object'))"], + name: :activities_create_objects_index + ) + ) + + create( + index(:activities, ["associated_object_id(data)"], + name: :activities_create_objects_index, + concurrently: true + ) + ) + end + + def down do + drop_if_exists( + index(:activities, ["associated_object_id(data)"], + name: :activities_create_objects_index + ) + ) + + create( + index(:activities, ["(coalesce(data->'object'->>'id', data->>'object'))"], + name: :activities_create_objects_index, + concurrently: true + ) + ) + end +end -- cgit v1.2.3 From 4e7ed563c050e3781990e6c62ea5996d61b63d37 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Mon, 11 Jul 2022 16:24:38 -0400 Subject: Lint --- .../20220711192750_switch_to_associated_object_id_index.exs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'priv/repo/migrations') diff --git a/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs b/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs index c0b89731b..75c1cd40b 100644 --- a/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs +++ b/priv/repo/migrations/20220711192750_switch_to_associated_object_id_index.exs @@ -24,9 +24,7 @@ defmodule Pleroma.Repo.Migrations.SwitchToAssociatedObjectIdIndex do def down do drop_if_exists( - index(:activities, ["associated_object_id(data)"], - name: :activities_create_objects_index - ) + index(:activities, ["associated_object_id(data)"], name: :activities_create_objects_index) ) create( -- cgit v1.2.3 From f047088a937ddf95d5fd7f84ad69fd97decbffc0 Mon Sep 17 00:00:00 2001 From: Tusooa Zhu Date: Sat, 20 Aug 2022 21:06:12 -0400 Subject: Update thread visibility function --- ...hread_visibility_to_use_new_object_id_index.exs | 156 +++++++++++++++++++++ 1 file changed, 156 insertions(+) create mode 100644 priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs (limited to 'priv/repo/migrations') diff --git a/priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs b/priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs new file mode 100644 index 000000000..bb56843cb --- /dev/null +++ b/priv/repo/migrations/20220821004840_change_thread_visibility_to_use_new_object_id_index.exs @@ -0,0 +1,156 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2022 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Repo.Migrations.ChangeThreadVisibilityToUseNewObjectIdIndex do + use Ecto.Migration + + def up do + execute(update_thread_visibility()) + end + + def down do + execute(restore_thread_visibility()) + end + + def update_thread_visibility do + """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar, local_public varchar default '') RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + author_fa varchar; + valid_recipients varchar[]; + actor_user_following varchar[]; + BEGIN + --- Fetch actor following + SELECT array_agg(following.follower_address) INTO actor_user_following FROM following_relationships + JOIN users ON users.id = following_relationships.follower_id + JOIN users AS following ON following.id = following_relationships.following_id + WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. + IF activity IS NULL THEN + RETURN false; + END IF; + + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON associated_object_id(activities.data) = objects.data->>'id' + WHERE associated_object_id(activity.data) = objects.data->>'id'; + + --- Fetch the author's AS2 following collection. + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + --- If we specified local public, add it. + IF local_public <> '' THEN + valid_recipients := valid_recipients || local_public; + END IF; + IF ARRAY[author_fa] && actor_user_following THEN + valid_recipients := valid_recipients || author_fa; + END IF; + + --- Check visibility. + IF NOT valid_recipients && activity.recipients THEN + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON associated_object_id(activities.data) = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + end + + # priv/repo/migrations/20220509180452_change_thread_visibility_to_be_local_only_aware.exs + def restore_thread_visibility do + """ + CREATE OR REPLACE FUNCTION thread_visibility(actor varchar, activity_id varchar, local_public varchar default '') RETURNS boolean AS $$ + DECLARE + public varchar := 'https://www.w3.org/ns/activitystreams#Public'; + child objects%ROWTYPE; + activity activities%ROWTYPE; + author_fa varchar; + valid_recipients varchar[]; + actor_user_following varchar[]; + BEGIN + --- Fetch actor following + SELECT array_agg(following.follower_address) INTO actor_user_following FROM following_relationships + JOIN users ON users.id = following_relationships.follower_id + JOIN users AS following ON following.id = following_relationships.following_id + WHERE users.ap_id = actor; + + --- Fetch our initial activity. + SELECT * INTO activity FROM activities WHERE activities.data->>'id' = activity_id; + + LOOP + --- Ensure that we have an activity before continuing. + --- If we don't, the thread is not satisfiable. + IF activity IS NULL THEN + RETURN false; + END IF; + + --- We only care about Create activities. + IF activity.data->>'type' != 'Create' THEN + RETURN true; + END IF; + + --- Normalize the child object into child. + SELECT * INTO child FROM objects + INNER JOIN activities ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE COALESCE(activity.data->'object'->>'id', activity.data->>'object') = objects.data->>'id'; + + --- Fetch the author's AS2 following collection. + SELECT COALESCE(users.follower_address, '') INTO author_fa FROM users WHERE users.ap_id = activity.actor; + + --- Prepare valid recipients array. + valid_recipients := ARRAY[actor, public]; + --- If we specified local public, add it. + IF local_public <> '' THEN + valid_recipients := valid_recipients || local_public; + END IF; + IF ARRAY[author_fa] && actor_user_following THEN + valid_recipients := valid_recipients || author_fa; + END IF; + + --- Check visibility. + IF NOT valid_recipients && activity.recipients THEN + --- activity not visible, break out of the loop + RETURN false; + END IF; + + --- If there's a parent, load it and do this all over again. + IF (child.data->'inReplyTo' IS NOT NULL) AND (child.data->'inReplyTo' != 'null'::jsonb) THEN + SELECT * INTO activity FROM activities + INNER JOIN objects ON COALESCE(activities.data->'object'->>'id', activities.data->>'object') = objects.data->>'id' + WHERE child.data->>'inReplyTo' = objects.data->>'id'; + ELSE + RETURN true; + END IF; + END LOOP; + END; + $$ LANGUAGE plpgsql IMMUTABLE; + """ + end +end -- cgit v1.2.3