summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--CHANGELOG.md2
-rw-r--r--config/config.exs7
-rw-r--r--docs/config.md3
-rw-r--r--lib/mix/tasks/pleroma/database.ex47
-rw-r--r--lib/pleroma/application.ex1
-rw-r--r--lib/pleroma/object.ex7
-rw-r--r--lib/pleroma/object/fetcher.ex22
-rw-r--r--lib/pleroma/web/oauth/token.ex36
-rw-r--r--lib/pleroma/web/oauth/token/clean_worker.ex41
-rw-r--r--lib/pleroma/web/oauth/token/query.ex55
-rw-r--r--mix.exs5
-rw-r--r--mix.lock2
-rw-r--r--test/object/fetcher_test.exs19
-rw-r--r--test/web/oauth/token_test.exs13
14 files changed, 231 insertions, 29 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bb2306fc4..6c94669db 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -41,6 +41,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
- Metadata: RelMe provider
- OAuth: added support for refresh tokens
- Emoji packs and emoji pack manager
+- Object pruning (`mix pleroma.database prune_objects`)
+- OAuth: added job to clean expired access tokens
### Changed
- **Breaking:** Configuration: move from Pleroma.Mailer to Pleroma.Emails.Mailer
diff --git a/config/config.exs b/config/config.exs
index c3301b2ed..33b7e713d 100644
--- a/config/config.exs
+++ b/config/config.exs
@@ -239,7 +239,8 @@ config :pleroma, :instance,
welcome_message: nil,
max_report_comment_size: 1000,
safe_dm_mentions: false,
- healthcheck: false
+ healthcheck: false,
+ remote_post_retention_days: 90
config :pleroma, :app_account_creation, enabled: true, max_requests: 25, interval: 1800
@@ -480,7 +481,9 @@ config :pleroma, Pleroma.ScheduledActivity,
config :pleroma, :oauth2,
token_expires_in: 600,
- issue_new_refresh_token: true
+ issue_new_refresh_token: true,
+ clean_expired_tokens: false,
+ clean_expired_tokens_interval: 86_400_000
config :pleroma, :database, rum_enabled: false
diff --git a/docs/config.md b/docs/config.md
index 63ca61d1e..be0cbc2dc 100644
--- a/docs/config.md
+++ b/docs/config.md
@@ -104,6 +104,7 @@ config :pleroma, Pleroma.Emails.Mailer,
* `max_report_comment_size`: The maximum size of the report comment (Default: `1000`)
* `safe_dm_mentions`: If set to true, only mentions at the beginning of a post will be used to address people in direct messages. This is to prevent accidental mentioning of people when talking about them (e.g. "@friend hey i really don't like @enemy"). (Default: `false`)
* `healthcheck`: if set to true, system data will be shown on ``/api/pleroma/healthcheck``.
+* `remote_post_retention_days`: the default amount of days to retain remote posts when pruning the database
## :app_account_creation
REST API for creating an account settings
@@ -549,6 +550,8 @@ Configure OAuth 2 provider capabilities:
* `token_expires_in` - The lifetime in seconds of the access token.
* `issue_new_refresh_token` - Keeps old refresh token or generate new refresh token when to obtain an access token.
+* `clean_expired_tokens` - Enable a background job to clean expired oauth tokens. Defaults to `false`.
+* `clean_expired_tokens_interval` - Interval to run the job to clean expired tokens. Defaults to `86_400_000` (24 hours).
## :emoji
* `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]`
diff --git a/lib/mix/tasks/pleroma/database.ex b/lib/mix/tasks/pleroma/database.ex
index f650b447d..4d480ac3f 100644
--- a/lib/mix/tasks/pleroma/database.ex
+++ b/lib/mix/tasks/pleroma/database.ex
@@ -5,6 +5,7 @@
defmodule Mix.Tasks.Pleroma.Database do
alias Mix.Tasks.Pleroma.Common
alias Pleroma.Conversation
+ alias Pleroma.Object
alias Pleroma.Repo
alias Pleroma.User
require Logger
@@ -23,6 +24,10 @@ defmodule Mix.Tasks.Pleroma.Database do
Options:
- `--vacuum` - run `VACUUM FULL` after the embedded objects are replaced with their references
+ ## Prune old objects from the database
+
+ mix pleroma.database prune_objects
+
## Create a conversation for all existing DMs. Can be safely re-run.
mix pleroma.database bump_all_conversations
@@ -72,4 +77,46 @@ defmodule Mix.Tasks.Pleroma.Database do
Enum.each(users, &User.remove_duplicated_following/1)
Enum.each(users, &User.update_follower_count/1)
end
+
+ def run(["prune_objects" | args]) do
+ import Ecto.Query
+
+ {options, [], []} =
+ OptionParser.parse(
+ args,
+ strict: [
+ vacuum: :boolean
+ ]
+ )
+
+ Common.start_pleroma()
+
+ deadline = Pleroma.Config.get([:instance, :remote_post_retention_days])
+
+ Logger.info("Pruning objects older than #{deadline} days")
+
+ time_deadline =
+ NaiveDateTime.utc_now()
+ |> NaiveDateTime.add(-(deadline * 86_400))
+
+ public = "https://www.w3.org/ns/activitystreams#Public"
+
+ from(o in Object,
+ where: fragment("?->'to' \\? ? OR ?->'cc' \\? ?", o.data, ^public, o.data, ^public),
+ where: o.inserted_at < ^time_deadline,
+ where:
+ fragment("split_part(?->>'actor', '/', 3) != ?", o.data, ^Pleroma.Web.Endpoint.host())
+ )
+ |> Repo.delete_all(timeout: :infinity)
+
+ if Keyword.get(options, :vacuum) do
+ Logger.info("Runnning VACUUM FULL")
+
+ Repo.query!(
+ "vacuum full;",
+ [],
+ timeout: :infinity
+ )
+ end
+ end
end
diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex
index dab45a0b2..76df3945e 100644
--- a/lib/pleroma/application.ex
+++ b/lib/pleroma/application.ex
@@ -110,6 +110,7 @@ defmodule Pleroma.Application do
hackney_pool_children() ++
[
worker(Pleroma.Web.Federator.RetryQueue, []),
+ worker(Pleroma.Web.OAuth.Token.CleanWorker, []),
worker(Pleroma.Stats, []),
worker(Task, [&Pleroma.Web.Push.init/0], restart: :temporary, id: :web_push_init),
worker(Task, [&Pleroma.Web.Federator.init/0], restart: :temporary, id: :federator_init)
diff --git a/lib/pleroma/object.ex b/lib/pleroma/object.ex
index 740d687a3..cc6fc9c5d 100644
--- a/lib/pleroma/object.ex
+++ b/lib/pleroma/object.ex
@@ -130,6 +130,13 @@ defmodule Pleroma.Object do
end
end
+ def prune(%Object{data: %{"id" => id}} = object) do
+ with {:ok, object} <- Repo.delete(object),
+ {:ok, true} <- Cachex.del(:object_cache, "object:#{id}") do
+ {:ok, object}
+ end
+ end
+
def set_cache(%Object{data: %{"id" => ap_id}} = object) do
Cachex.put(:object_cache, "object:#{ap_id}", object)
{:ok, object}
diff --git a/lib/pleroma/object/fetcher.ex b/lib/pleroma/object/fetcher.ex
index 8d4bcc95e..bb9388d4f 100644
--- a/lib/pleroma/object/fetcher.ex
+++ b/lib/pleroma/object/fetcher.ex
@@ -8,6 +8,19 @@ defmodule Pleroma.Object.Fetcher do
@httpoison Application.get_env(:pleroma, :httpoison)
+ defp reinject_object(data) do
+ Logger.debug("Reinjecting object #{data["id"]}")
+
+ with data <- Transmogrifier.fix_object(data),
+ {:ok, object} <- Object.create(data) do
+ {:ok, object}
+ else
+ e ->
+ Logger.error("Error while processing object: #{inspect(e)}")
+ {:error, e}
+ end
+ end
+
# TODO:
# This will create a Create activity, which we need internally at the moment.
def fetch_object_from_id(id) do
@@ -26,12 +39,17 @@ defmodule Pleroma.Object.Fetcher do
"object" => data
},
:ok <- Containment.contain_origin(id, params),
- {:ok, activity} <- Transmogrifier.handle_incoming(params) do
- {:ok, Object.normalize(activity, false)}
+ {:ok, activity} <- Transmogrifier.handle_incoming(params),
+ {:object, _data, %Object{} = object} <-
+ {:object, data, Object.normalize(activity, false)} do
+ {:ok, object}
else
{:error, {:reject, nil}} ->
{:reject, nil}
+ {:object, data, nil} ->
+ reinject_object(data)
+
object = %Object{} ->
{:ok, object}
diff --git a/lib/pleroma/web/oauth/token.ex b/lib/pleroma/web/oauth/token.ex
index 66c95c2e9..f412f7eb2 100644
--- a/lib/pleroma/web/oauth/token.ex
+++ b/lib/pleroma/web/oauth/token.ex
@@ -5,7 +5,6 @@
defmodule Pleroma.Web.OAuth.Token do
use Ecto.Schema
- import Ecto.Query
import Ecto.Changeset
alias Pleroma.Repo
@@ -13,6 +12,7 @@ defmodule Pleroma.Web.OAuth.Token do
alias Pleroma.Web.OAuth.App
alias Pleroma.Web.OAuth.Authorization
alias Pleroma.Web.OAuth.Token
+ alias Pleroma.Web.OAuth.Token.Query
@expires_in Pleroma.Config.get([:oauth2, :token_expires_in], 600)
@type t :: %__MODULE__{}
@@ -31,17 +31,17 @@ defmodule Pleroma.Web.OAuth.Token do
@doc "Gets token for app by access token"
@spec get_by_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
def get_by_token(%App{id: app_id} = _app, token) do
- from(t in __MODULE__, where: t.app_id == ^app_id and t.token == ^token)
+ Query.get_by_app(app_id)
+ |> Query.get_by_token(token)
|> Repo.find_resource()
end
@doc "Gets token for app by refresh token"
@spec get_by_refresh_token(App.t(), String.t()) :: {:ok, t()} | {:error, :not_found}
def get_by_refresh_token(%App{id: app_id} = _app, token) do
- from(t in __MODULE__,
- where: t.app_id == ^app_id and t.refresh_token == ^token,
- preload: [:user]
- )
+ Query.get_by_app(app_id)
+ |> Query.get_by_refresh_token(token)
+ |> Query.preload([:user])
|> Repo.find_resource()
end
@@ -97,29 +97,25 @@ defmodule Pleroma.Web.OAuth.Token do
end
def delete_user_tokens(%User{id: user_id}) do
- from(
- t in Token,
- where: t.user_id == ^user_id
- )
+ Query.get_by_user(user_id)
|> Repo.delete_all()
end
def delete_user_token(%User{id: user_id}, token_id) do
- from(
- t in Token,
- where: t.user_id == ^user_id,
- where: t.id == ^token_id
- )
+ Query.get_by_user(user_id)
+ |> Query.get_by_id(token_id)
+ |> Repo.delete_all()
+ end
+
+ def delete_expired_tokens do
+ Query.get_expired_tokens()
|> Repo.delete_all()
end
def get_user_tokens(%User{id: user_id}) do
- from(
- t in Token,
- where: t.user_id == ^user_id
- )
+ Query.get_by_user(user_id)
+ |> Query.preload([:app])
|> Repo.all()
- |> Repo.preload(:app)
end
def is_expired?(%__MODULE__{valid_until: valid_until}) do
diff --git a/lib/pleroma/web/oauth/token/clean_worker.ex b/lib/pleroma/web/oauth/token/clean_worker.ex
new file mode 100644
index 000000000..dca852449
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/clean_worker.ex
@@ -0,0 +1,41 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Token.CleanWorker do
+ @moduledoc """
+ The module represents functions to clean an expired oauth tokens.
+ """
+
+ # 10 seconds
+ @start_interval 10_000
+ @interval Pleroma.Config.get(
+ # 24 hours
+ [:oauth2, :clean_expired_tokens_interval],
+ 86_400_000
+ )
+ @queue :background
+
+ alias Pleroma.Web.OAuth.Token
+
+ def start_link, do: GenServer.start_link(__MODULE__, nil)
+
+ def init(_) do
+ if Pleroma.Config.get([:oauth2, :clean_expired_tokens], false) do
+ Process.send_after(self(), :perform, @start_interval)
+ {:ok, nil}
+ else
+ :ignore
+ end
+ end
+
+ @doc false
+ def handle_info(:perform, state) do
+ Process.send_after(self(), :perform, @interval)
+ PleromaJobQueue.enqueue(@queue, __MODULE__, [:clean])
+ {:noreply, state}
+ end
+
+ # Job Worker Callbacks
+ def perform(:clean), do: Token.delete_expired_tokens()
+end
diff --git a/lib/pleroma/web/oauth/token/query.ex b/lib/pleroma/web/oauth/token/query.ex
new file mode 100644
index 000000000..d92e1f071
--- /dev/null
+++ b/lib/pleroma/web/oauth/token/query.ex
@@ -0,0 +1,55 @@
+# Pleroma: A lightweight social networking server
+# Copyright © 2017-2018 Pleroma Authors <https://pleroma.social/>
+# SPDX-License-Identifier: AGPL-3.0-only
+
+defmodule Pleroma.Web.OAuth.Token.Query do
+ @moduledoc """
+ Contains queries for OAuth Token.
+ """
+
+ import Ecto.Query, only: [from: 2]
+
+ @type query :: Ecto.Queryable.t() | Token.t()
+
+ alias Pleroma.Web.OAuth.Token
+
+ @spec get_by_refresh_token(query, String.t()) :: query
+ def get_by_refresh_token(query \\ Token, refresh_token) do
+ from(q in query, where: q.refresh_token == ^refresh_token)
+ end
+
+ @spec get_by_token(query, String.t()) :: query
+ def get_by_token(query \\ Token, token) do
+ from(q in query, where: q.token == ^token)
+ end
+
+ @spec get_by_app(query, String.t()) :: query
+ def get_by_app(query \\ Token, app_id) do
+ from(q in query, where: q.app_id == ^app_id)
+ end
+
+ @spec get_by_id(query, String.t()) :: query
+ def get_by_id(query \\ Token, id) do
+ from(q in query, where: q.id == ^id)
+ end
+
+ @spec get_expired_tokens(query, DateTime.t() | nil) :: query
+ def get_expired_tokens(query \\ Token, date \\ nil) do
+ expired_date = date || Timex.now()
+ from(q in query, where: fragment("?", q.valid_until) < ^expired_date)
+ end
+
+ @spec get_by_user(query, String.t()) :: query
+ def get_by_user(query \\ Token, user_id) do
+ from(q in query, where: q.user_id == ^user_id)
+ end
+
+ @spec preload(query, any) :: query
+ def preload(query \\ Token, assoc_preload \\ [])
+
+ def preload(query, assoc_preload) when is_list(assoc_preload) do
+ from(q in query, preload: ^assoc_preload)
+ end
+
+ def preload(query, _assoc_preload), do: query
+end
diff --git a/mix.exs b/mix.exs
index b2c075c85..b2017ef9b 100644
--- a/mix.exs
+++ b/mix.exs
@@ -66,10 +66,7 @@ defmodule Pleroma.Mixfile do
{:plug_cowboy, "~> 2.0"},
{:phoenix_pubsub, "~> 1.1"},
{:phoenix_ecto, "~> 4.0"},
- {:ecto_sql,
- git: "https://github.com/elixir-ecto/ecto_sql",
- ref: "14cb065a74c488d737d973f7a91bc036c6245f78",
- override: true},
+ {:ecto_sql, "~> 3.1"},
{:postgrex, ">= 0.13.5"},
{:gettext, "~> 0.15"},
{:comeonin, "~> 4.1.1"},
diff --git a/mix.lock b/mix.lock
index bacc09787..857bfca79 100644
--- a/mix.lock
+++ b/mix.lock
@@ -21,7 +21,7 @@
"deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm"},
"earmark": {:hex, :earmark, "1.3.2", "b840562ea3d67795ffbb5bd88940b1bed0ed9fa32834915125ea7d02e35888a5", [:mix], [], "hexpm"},
"ecto": {:hex, :ecto, "3.1.4", "69d852da7a9f04ede725855a35ede48d158ca11a404fe94f8b2fb3b2162cd3c9", [:mix], [{:decimal, "~> 1.6", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}], "hexpm"},
- "ecto_sql": {:git, "https://github.com/elixir-ecto/ecto_sql", "14cb065a74c488d737d973f7a91bc036c6245f78", [ref: "14cb065a74c488d737d973f7a91bc036c6245f78"]},
+ "ecto_sql": {:hex, :ecto_sql, "3.1.3", "2c536139190492d9de33c5fefac7323c5eaaa82e1b9bf93482a14649042f7cd9", [:mix], [{:db_connection, "~> 2.0", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.1.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:mariaex, "~> 0.9.1", [hex: :mariaex, repo: "hexpm", optional: true]}, {:myxql, "~> 0.2.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.14.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm"},
"esshd": {:hex, :esshd, "0.1.0", "6f93a2062adb43637edad0ea7357db2702a4b80dd9683482fe00f5134e97f4c1", [:mix], [], "hexpm"},
"eternal": {:hex, :eternal, "1.2.0", "e2a6b6ce3b8c248f7dc31451aefca57e3bdf0e48d73ae5043229380a67614c41", [:mix], [], "hexpm"},
"ex2ms": {:hex, :ex2ms, "1.5.0", "19e27f9212be9a96093fed8cdfbef0a2b56c21237196d26760f11dfcfae58e97", [:mix], [], "hexpm"},
diff --git a/test/object/fetcher_test.exs b/test/object/fetcher_test.exs
index 72f616782..d604fd5f5 100644
--- a/test/object/fetcher_test.exs
+++ b/test/object/fetcher_test.exs
@@ -87,4 +87,23 @@ defmodule Pleroma.Object.FetcherTest do
)
end
end
+
+ describe "pruning" do
+ test "it can refetch pruned objects" do
+ object_id = "http://mastodon.example.org/@admin/99541947525187367"
+
+ {:ok, object} = Fetcher.fetch_object_from_id(object_id)
+
+ assert object
+
+ {:ok, _object} = Object.prune(object)
+
+ refute Object.get_by_ap_id(object_id)
+
+ {:ok, %Object{} = object_two} = Fetcher.fetch_object_from_id(object_id)
+
+ assert object.data["id"] == object_two.data["id"]
+ assert object.id != object_two.id
+ end
+ end
end
diff --git a/test/web/oauth/token_test.exs b/test/web/oauth/token_test.exs
index ad2a49f09..3c07309b7 100644
--- a/test/web/oauth/token_test.exs
+++ b/test/web/oauth/token_test.exs
@@ -69,4 +69,17 @@ defmodule Pleroma.Web.OAuth.TokenTest do
assert tokens == 2
end
+
+ test "deletes expired tokens" do
+ insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3))
+ insert(:oauth_token, valid_until: Timex.shift(Timex.now(), days: -3))
+ t3 = insert(:oauth_token)
+ t4 = insert(:oauth_token, valid_until: Timex.shift(Timex.now(), minutes: 10))
+ {tokens, _} = Token.delete_expired_tokens()
+ assert tokens == 2
+ available_tokens = Pleroma.Repo.all(Token)
+
+ token_ids = available_tokens |> Enum.map(& &1.id)
+ assert token_ids == [t3.id, t4.id]
+ end
end
generated by cgit v1.2.3 (git 2.25.1) at 2025-04-25 16:32:28 +0000