From f11e7037c21d611cddd7f2eab64ebfc39630a078 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 1 May 2019 16:09:53 +0700 Subject: test fixes --- lib/mix/tasks/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index b396ff0de..9e2523b18 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -126,7 +126,7 @@ defmodule Mix.Tasks.Pleroma.User do proceed? = assume_yes? or Mix.shell().yes?("Continue?") - unless not proceed? do + if proceed? do Common.start_pleroma() params = %{ -- cgit v1.2.3 From 0e37fddd5abad5db0378c8186513fbba0fbd6586 Mon Sep 17 00:00:00 2001 From: lain Date: Fri, 3 May 2019 19:15:55 +0200 Subject: Search: Add fts index on objects table. --- lib/mix/tasks/benchmark.ex | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 lib/mix/tasks/benchmark.ex (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/benchmark.ex b/lib/mix/tasks/benchmark.ex new file mode 100644 index 000000000..0fbb4dbb1 --- /dev/null +++ b/lib/mix/tasks/benchmark.ex @@ -0,0 +1,25 @@ +defmodule Mix.Tasks.Pleroma.Benchmark do + use Mix.Task + alias Mix.Tasks.Pleroma.Common + + def run(["search"]) do + Common.start_pleroma() + + Benchee.run(%{ + "search" => fn -> + Pleroma.Web.MastodonAPI.MastodonAPIController.status_search(nil, "cofe") + end + }) + end + + def run(["tag"]) do + Common.start_pleroma() + + Benchee.run(%{ + "tag" => fn -> + %{"type" => "Create", "tag" => "cofe"} + |> Pleroma.Web.ActivityPub.ActivityPub.fetch_public_activities() + end + }) + end +end -- cgit v1.2.3 From ce6ca0fefe7feb1c31447287aec117d40233652a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 6 May 2019 16:45:22 +0000 Subject: Merge branch 'develop' of https://git.pleroma.social/pleroma/pleroma into feature/845-improve-status-deletion --- lib/mix/tasks/pleroma/user.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 9e2523b18..6a83a8c0d 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -163,7 +163,7 @@ defmodule Mix.Tasks.Pleroma.User do Common.start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.delete(user) + User.perform(:delete, user) Mix.shell().info("User #{nickname} deleted.") else _ -> @@ -380,7 +380,7 @@ defmodule Mix.Tasks.Pleroma.User do Common.start_pleroma() with %User{local: true} = user <- User.get_cached_by_nickname(nickname) do - User.delete_user_activities(user) + {:ok, _} = User.delete_user_activities(user) Mix.shell().info("User #{nickname} statuses deleted.") else _ -> -- cgit v1.2.3 From a76e2622ca0201dfd78616cbad58c9fb14111796 Mon Sep 17 00:00:00 2001 From: Alex S Date: Wed, 8 May 2019 19:05:25 +0700 Subject: emoji dowload follow redirects --- lib/mix/tasks/pleroma/emoji.ex | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index cced73226..5cb54c3ca 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -109,7 +109,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do ]) ) - binary_archive = Tesla.get!(src_url).body + binary_archive = Tesla.get!(client(), src_url).body archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() sha_status_text = ["SHA256 of ", :bright, pack_name, :normal, " source file is ", :bright] @@ -137,7 +137,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do ]) ) - files = Tesla.get!(files_url).body |> Poison.decode!() + files = Tesla.get!(client(), files_url).body |> Poison.decode!() IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name])) @@ -213,7 +213,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts("Downloading the pack and generating SHA256") - binary_archive = Tesla.get!(src).body + binary_archive = Tesla.get!(client(), src).body archive_sha = :crypto.hash(:sha256, binary_archive) |> Base.encode16() IO.puts("SHA256 is #{archive_sha}") @@ -272,7 +272,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do defp fetch_manifest(from) do Poison.decode!( if String.starts_with?(from, "http") do - Tesla.get!(from).body + Tesla.get!(client(), from).body else File.read!(from) end @@ -290,4 +290,12 @@ defmodule Mix.Tasks.Pleroma.Emoji do ] ) end + + defp client do + middleware = [ + {Tesla.Middleware.FollowRedirects, [max_redirects: 3]} + ] + + Tesla.client(middleware) + end end -- cgit v1.2.3 From 6d19bb4eae43270099a68f749519ba0f323da01a Mon Sep 17 00:00:00 2001 From: lain Date: Wed, 8 May 2019 18:09:07 +0200 Subject: Conversations: Add mix task to 'import' old DMs. --- lib/mix/tasks/pleroma/conversations.ex | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 lib/mix/tasks/pleroma/conversations.ex (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex new file mode 100644 index 000000000..125d8851a --- /dev/null +++ b/lib/mix/tasks/pleroma/conversations.ex @@ -0,0 +1,23 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2018 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Mix.Tasks.Pleroma.Conversations do + use Mix.Task + alias Mix.Tasks.Pleroma.Common + alias Pleroma.Conversation + + @shortdoc "Manages Pleroma users" + @moduledoc """ + Manages Pleroma conversations. + + ## Create a conversation for all existing DMs. Can be safely re-run. + + mix pleroma.conversations bump_all + + """ + def run(["bump_all"]) do + Common.start_pleroma() + Conversation.bump_for_all_activities() + end +end -- cgit v1.2.3 From 786f2c7a849bc4fa2bd4aac18de59ef6b2ed18c5 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 13 May 2019 11:16:54 -0500 Subject: Update shortdoc description --- lib/mix/tasks/pleroma/conversations.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex index 125d8851a..b52b9921a 100644 --- a/lib/mix/tasks/pleroma/conversations.ex +++ b/lib/mix/tasks/pleroma/conversations.ex @@ -7,7 +7,7 @@ defmodule Mix.Tasks.Pleroma.Conversations do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation - @shortdoc "Manages Pleroma users" + @shortdoc "Manages Pleroma conversations." @moduledoc """ Manages Pleroma conversations. -- cgit v1.2.3 From a2be420f940fb8f181feeb9b0fb9759d433dcae1 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Mon, 13 May 2019 18:35:45 +0000 Subject: differences_in_mastoapi_responses.md: fullname & bio are optionnal [ci skip] --- lib/mix/tasks/pleroma/user.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index 6a83a8c0d..d130ff8c9 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -138,7 +138,7 @@ defmodule Mix.Tasks.Pleroma.User do bio: bio } - changeset = User.register_changeset(%User{}, params, confirmed: true) + changeset = User.register_changeset(%User{}, params, need_confirmation: false) {:ok, _user} = User.register(changeset) Mix.shell().info("User #{nickname} created") -- cgit v1.2.3 From 498bfdf403b6d20f92363a4b650dd0b926351e17 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Mon, 13 May 2019 15:37:38 -0500 Subject: Switch to Jason over Poison --- lib/mix/tasks/pleroma/emoji.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/emoji.ex b/lib/mix/tasks/pleroma/emoji.ex index 5cb54c3ca..d2ddf450a 100644 --- a/lib/mix/tasks/pleroma/emoji.ex +++ b/lib/mix/tasks/pleroma/emoji.ex @@ -137,7 +137,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do ]) ) - files = Tesla.get!(client(), files_url).body |> Poison.decode!() + files = Tesla.get!(client(), files_url).body |> Jason.decode!() IO.puts(IO.ANSI.format(["Unpacking ", :bright, pack_name])) @@ -239,7 +239,7 @@ defmodule Mix.Tasks.Pleroma.Emoji do emoji_map = Pleroma.Emoji.make_shortcode_to_file_map(tmp_pack_dir, exts) - File.write!(files_name, Poison.encode!(emoji_map, pretty: true)) + File.write!(files_name, Jason.encode!(emoji_map, pretty: true)) IO.puts(""" @@ -248,11 +248,11 @@ defmodule Mix.Tasks.Pleroma.Emoji do """) if File.exists?("index.json") do - existing_data = File.read!("index.json") |> Poison.decode!() + existing_data = File.read!("index.json") |> Jason.decode!() File.write!( "index.json", - Poison.encode!( + Jason.encode!( Map.merge( existing_data, pack_json @@ -263,14 +263,14 @@ defmodule Mix.Tasks.Pleroma.Emoji do IO.puts("index.json file has been update with the #{name} pack") else - File.write!("index.json", Poison.encode!(pack_json, pretty: true)) + File.write!("index.json", Jason.encode!(pack_json, pretty: true)) IO.puts("index.json has been created with the #{name} pack") end end defp fetch_manifest(from) do - Poison.decode!( + Jason.decode!( if String.starts_with?(from, "http") do Tesla.get!(client(), from).body else -- cgit v1.2.3 From 73ae58fdfaf0b9dc9630929b0b84ae3b6083684a Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Thu, 16 May 2019 13:23:41 +0000 Subject: Feature/896 toggling confirmation --- lib/mix/tasks/pleroma/user.ex | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/user.ex b/lib/mix/tasks/pleroma/user.ex index d130ff8c9..25fc40ea7 100644 --- a/lib/mix/tasks/pleroma/user.ex +++ b/lib/mix/tasks/pleroma/user.ex @@ -77,6 +77,10 @@ defmodule Mix.Tasks.Pleroma.User do ## Delete tags from a user. mix pleroma.user untag NICKNAME TAGS + + ## Toggle confirmation of the user's account. + + mix pleroma.user toggle_confirmed NICKNAME """ def run(["new", nickname, email | rest]) do {options, [], []} = @@ -388,6 +392,21 @@ defmodule Mix.Tasks.Pleroma.User do end end + def run(["toggle_confirmed", nickname]) do + Common.start_pleroma() + + with %User{} = user <- User.get_cached_by_nickname(nickname) do + {:ok, user} = User.toggle_confirmation(user) + + message = if user.info.confirmation_pending, do: "needs", else: "doesn't need" + + Mix.shell().info("#{nickname} #{message} confirmation.") + else + _ -> + Mix.shell().error("No local user #{nickname}") + end + end + defp set_moderator(user, value) do info_cng = User.Info.admin_api_update(user.info, %{is_moderator: value}) -- cgit v1.2.3 From efa61c161085fd24f7e85ccf7f32ef823e335d7b Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Thu, 16 May 2019 13:14:48 -0500 Subject: Move to pleroma.database mix task --- lib/mix/tasks/pleroma/conversations.ex | 23 ----------------------- lib/mix/tasks/pleroma/database.ex | 11 +++++++++++ 2 files changed, 11 insertions(+), 23 deletions(-) delete mode 100644 lib/mix/tasks/pleroma/conversations.ex (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/conversations.ex b/lib/mix/tasks/pleroma/conversations.ex deleted file mode 100644 index b52b9921a..000000000 --- a/lib/mix/tasks/pleroma/conversations.ex +++ /dev/null @@ -1,23 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2018 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Mix.Tasks.Pleroma.Conversations do - use Mix.Task - alias Mix.Tasks.Pleroma.Common - alias Pleroma.Conversation - - @shortdoc "Manages Pleroma conversations." - @moduledoc """ - Manages Pleroma conversations. - - ## Create a conversation for all existing DMs. Can be safely re-run. - - mix pleroma.conversations bump_all - - """ - def run(["bump_all"]) do - Common.start_pleroma() - Conversation.bump_for_all_activities() - end -end diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index ab9a3a7ff..42753a1a4 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -4,6 +4,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common + alias Pleroma.Conversation require Logger use Mix.Task @@ -19,6 +20,11 @@ defmodule Mix.Tasks.Pleroma.Database do Options: - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + + ## Create a conversation for all existing DMs. Can be safely re-run. + + mix pleroma.database bump_all_conversations + """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -48,4 +54,9 @@ defmodule Mix.Tasks.Pleroma.Database do ) end end + + def run(["bump_all_conversations"]) do + Common.start_pleroma() + Conversation.bump_for_all_activities() + end end -- cgit v1.2.3 From 2c303afc8b25ca92eaa229df3e9e4737856e890c Mon Sep 17 00:00:00 2001 From: Sergey Suprunenko Date: Thu, 16 May 2019 20:04:08 +0000 Subject: Remove duplicated entries in users' following lists --- lib/mix/tasks/pleroma/database.ex | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index 42753a1a4..f650b447d 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -5,6 +5,8 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation + alias Pleroma.Repo + alias Pleroma.User require Logger use Mix.Task @@ -25,6 +27,9 @@ defmodule Mix.Tasks.Pleroma.Database do mix pleroma.database bump_all_conversations + ## Remove duplicated items from following and update followers count for all users + + mix pleroma.database update_users_following_followers_counts """ def run(["remove_embedded_objects" | args]) do {options, [], []} = @@ -38,7 +43,7 @@ defmodule Mix.Tasks.Pleroma.Database do Common.start_pleroma() Logger.info("Removing embedded objects") - Pleroma.Repo.query!( + Repo.query!( "update activities set data = jsonb_set(data, '{object}'::text[], data->'object'->'id') where data->'object'->>'id' is not null;", [], timeout: :infinity @@ -47,7 +52,7 @@ defmodule Mix.Tasks.Pleroma.Database do if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") - Pleroma.Repo.query!( + Repo.query!( "vacuum full;", [], timeout: :infinity @@ -59,4 +64,12 @@ defmodule Mix.Tasks.Pleroma.Database do Common.start_pleroma() Conversation.bump_for_all_activities() end + + def run(["update_users_following_followers_counts"]) do + Common.start_pleroma() + + users = Repo.all(User) + Enum.each(users, &User.remove_duplicated_following/1) + Enum.each(users, &User.update_follower_count/1) + end end -- cgit v1.2.3 From 16b260fb19cca02463766c2e36a41bfcc823af9b Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Tue, 21 May 2019 01:21:28 +0000 Subject: add mix task to prune the object database using a configured retention period --- lib/mix/tasks/pleroma/database.ex | 40 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f650b447d..fdb216037 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -23,6 +23,10 @@ defmodule Mix.Tasks.Pleroma.Database do Options: - `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references + ## Prune old objects from the database + + mix pleroma.database prune_objects + ## Create a conversation for all existing DMs. Can be safely re-run. mix pleroma.database bump_all_conversations @@ -72,4 +76,40 @@ defmodule Mix.Tasks.Pleroma.Database do Enum.each(users, &User.remove_duplicated_following/1) Enum.each(users, &User.update_follower_count/1) end + + def run(["prune_objects" | args]) do + {options, [], []} = + OptionParser.parse( + args, + strict: [ + vacuum: :boolean + ] + ) + + Common.start_pleroma() + + deadline = Pleroma.Config.get([:instance, :remote_post_retention_days]) + + Logger.info("Pruning objects older than #{deadline} days") + + time_deadline = + NaiveDateTime.utc_now() + |> NaiveDateTime.add(-(deadline * 86_400)) + + Repo.query!( + "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2", + [time_deadline, Pleroma.Web.Endpoint.host()], + timeout: :infinity + ) + + if Keyword.get(options, :vacuum) do + Logger.info("Runnning VACUUM FULL") + + Repo.query!( + "vacuum full;", + [], + timeout: :infinity + ) + end + end end -- cgit v1.2.3 From a023ca004cbd90e330cab35e4dfda16346d08668 Mon Sep 17 00:00:00 2001 From: William Pitcock Date: Wed, 22 May 2019 03:12:48 +0000 Subject: prune objects task: use Repo.delete_all() --- lib/mix/tasks/pleroma/database.ex | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index fdb216037..f9bafb277 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -5,6 +5,7 @@ defmodule Mix.Tasks.Pleroma.Database do alias Mix.Tasks.Pleroma.Common alias Pleroma.Conversation + alias Pleroma.Object alias Pleroma.Repo alias Pleroma.User require Logger @@ -78,6 +79,8 @@ defmodule Mix.Tasks.Pleroma.Database do end def run(["prune_objects" | args]) do + import Ecto.Query + {options, [], []} = OptionParser.parse( args, @@ -96,11 +99,15 @@ defmodule Mix.Tasks.Pleroma.Database do NaiveDateTime.utc_now() |> NaiveDateTime.add(-(deadline * 86_400)) - Repo.query!( - "DELETE FROM objects WHERE inserted_at < $1 AND split_part(data->>'actor', '/', 3) != $2", - [time_deadline, Pleroma.Web.Endpoint.host()], - timeout: :infinity + public = "https://www.w3.org/ns/activitystreams#Public" + + from(o in Object, + where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public), + where: o.inserted_at < ^time_deadline, + where: + fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) ) + |> Repo.delete_all() if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") -- cgit v1.2.3 From 54e10a3e55fe46b71ef7f330605baf8bcccd5a44 Mon Sep 17 00:00:00 2001 From: rinpatch Date: Wed, 22 May 2019 20:10:52 +0300 Subject: Disable timeouts for object pruning query --- lib/mix/tasks/pleroma/database.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/mix/tasks') diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex index f9bafb277..4d480ac3f 100644 --- a/lib/mix/tasks/pleroma/database.ex +++ b/lib/mix/tasks/pleroma/database.ex @@ -107,7 +107,7 @@ defmodule Mix.Tasks.Pleroma.Database do where: fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host()) ) - |> Repo.delete_all() + |> Repo.delete_all(timeout: :infinity) if Keyword.get(options, :vacuum) do Logger.info("Runnning VACUUM FULL") -- cgit v1.2.3