From e1b07402ab077899dd5b9c0023fbe1c48af259e9 Mon Sep 17 00:00:00 2001 From: "Haelwenn (lanodan) Monnier" Date: Mon, 23 Mar 2020 22:52:25 +0100 Subject: User: Add raw_bio, storing unformatted bio Related: https://git.pleroma.social/pleroma/pleroma/issues/1643 --- lib/pleroma/user.ex | 13 ++++++++++++- .../web/mastodon_api/controllers/account_controller.ex | 1 + lib/pleroma/web/mastodon_api/views/account_view.ex | 13 +------------ 3 files changed, 14 insertions(+), 13 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 72ee2d58e..23ca8c9f3 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -79,6 +79,7 @@ defmodule Pleroma.User do schema "users" do field(:bio, :string) + field(:raw_bio, :string) field(:email, :string) field(:name, :string) field(:nickname, :string) @@ -432,6 +433,7 @@ defmodule Pleroma.User do params, [ :bio, + :raw_bio, :name, :emoji, :avatar, @@ -607,7 +609,16 @@ defmodule Pleroma.User do struct |> confirmation_changeset(need_confirmation: need_confirmation?) - |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji]) + |> cast(params, [ + :bio, + :raw_bio, + :email, + :name, + :nickname, + :password, + :password_confirmation, + :emoji + ]) |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) |> unique_constraint(:email) diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 5734bb854..ebfa533dd 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -165,6 +165,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do end) |> Maps.put_if_present(:name, params[:display_name]) |> Maps.put_if_present(:bio, params[:note]) + |> Maps.put_if_present(:raw_bio, params[:note]) |> Maps.put_if_present(:avatar, params[:avatar]) |> Maps.put_if_present(:banner, params[:header]) |> Maps.put_if_present(:background, params[:pleroma_background_image]) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 04c419d2f..5326b02c6 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -224,7 +224,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do fields: user.fields, bot: bot, source: %{ - note: prepare_user_bio(user), + note: user.raw_bio || "", sensitive: false, fields: user.raw_fields, pleroma: %{ @@ -259,17 +259,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> maybe_put_unread_notification_count(user, opts[:for]) end - defp prepare_user_bio(%User{bio: ""}), do: "" - - defp prepare_user_bio(%User{bio: bio}) when is_binary(bio) do - bio - |> String.replace(~r(
), "\n") - |> Pleroma.HTML.strip_tags() - |> HtmlEntities.decode() - end - - defp prepare_user_bio(_), do: "" - defp username_from_nickname(string) when is_binary(string) do hd(String.split(string, "@")) end -- cgit v1.2.3 From f9dcf15ecb684b4b802d731a216448c76913d462 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Fri, 12 Jun 2020 14:49:54 +0300 Subject: added admin api for MediaProxy cache invalidation --- .../controllers/media_proxy_cache_controller.ex | 38 +++++++ .../web/admin_api/views/media_proxy_cache_view.ex | 11 +++ .../admin/media_proxy_cache_operation.ex | 109 +++++++++++++++++++++ lib/pleroma/web/router.ex | 4 + 4 files changed, 162 insertions(+) create mode 100644 lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex create mode 100644 lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex create mode 100644 lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex new file mode 100644 index 000000000..7b28f7c72 --- /dev/null +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do + use Pleroma.Web, :controller + + alias Pleroma.Plugs.OAuthScopesPlug + alias Pleroma.Web.ApiSpec.Admin, as: Spec + + plug(Pleroma.Web.ApiSpec.CastAndValidate) + + plug( + OAuthScopesPlug, + %{scopes: ["read:media_proxy_caches"], admin: true} when action in [:index] + ) + + plug( + OAuthScopesPlug, + %{scopes: ["write:media_proxy_caches"], admin: true} when action in [:purge, :delete] + ) + + action_fallback(Pleroma.Web.AdminAPI.FallbackController) + + defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation + + def index(%{assigns: %{user: _}} = conn, _) do + render(conn, "index.json", urls: []) + end + + def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do + render(conn, "index.json", urls: urls) + end + + def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: _ban}} = conn, _) do + render(conn, "index.json", urls: urls) + end +end diff --git a/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex new file mode 100644 index 000000000..c97400beb --- /dev/null +++ b/lib/pleroma/web/admin_api/views/media_proxy_cache_view.ex @@ -0,0 +1,11 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.AdminAPI.MediaProxyCacheView do + use Pleroma.Web, :view + + def render("index.json", %{urls: urls}) do + %{urls: urls} + end +end diff --git a/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex new file mode 100644 index 000000000..0358cfbad --- /dev/null +++ b/lib/pleroma/web/api_spec/operations/admin/media_proxy_cache_operation.ex @@ -0,0 +1,109 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.ApiSpec.Admin.MediaProxyCacheOperation do + alias OpenApiSpex.Operation + alias OpenApiSpex.Schema + alias Pleroma.Web.ApiSpec.Schemas.ApiError + + import Pleroma.Web.ApiSpec.Helpers + + def open_api_operation(action) do + operation = String.to_existing_atom("#{action}_operation") + apply(__MODULE__, operation, []) + end + + def index_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Fetch a paginated list of all banned MediaProxy URLs in Cachex", + operationId: "AdminAPI.MediaProxyCacheController.index", + security: [%{"oAuth" => ["read:media_proxy_caches"]}], + parameters: [ + Operation.parameter( + :page, + :query, + %Schema{type: :integer, default: 1}, + "Page" + ), + Operation.parameter( + :page_size, + :query, + %Schema{type: :integer, default: 50}, + "Number of statuses to return" + ) + ], + responses: %{ + 200 => success_response() + } + } + end + + def delete_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Remove a banned MediaProxy URL from Cachex", + operationId: "AdminAPI.MediaProxyCacheController.delete", + security: [%{"oAuth" => ["write:media_proxy_caches"]}], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + required: [:urls], + properties: %{ + urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}} + } + }, + required: true + ), + responses: %{ + 200 => success_response(), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + def purge_operation do + %Operation{ + tags: ["Admin", "MediaProxyCache"], + summary: "Purge and optionally ban a MediaProxy URL", + operationId: "AdminAPI.MediaProxyCacheController.purge", + security: [%{"oAuth" => ["write:media_proxy_caches"]}], + requestBody: + request_body( + "Parameters", + %Schema{ + type: :object, + required: [:urls], + properties: %{ + urls: %Schema{type: :array, items: %Schema{type: :string, format: :uri}}, + ban: %Schema{type: :boolean, default: true} + } + }, + required: true + ), + responses: %{ + 200 => success_response(), + 400 => Operation.response("Error", "application/json", ApiError) + } + } + end + + defp success_response do + Operation.response("Array of banned MediaProxy URLs in Cachex", "application/json", %Schema{ + type: :object, + properties: %{ + urls: %Schema{ + type: :array, + items: %Schema{ + type: :string, + format: :uri, + description: "MediaProxy URLs" + } + } + } + }) + end +end diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 57570b672..eda74a171 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -209,6 +209,10 @@ defmodule Pleroma.Web.Router do post("/oauth_app", OAuthAppController, :create) patch("/oauth_app/:id", OAuthAppController, :update) delete("/oauth_app/:id", OAuthAppController, :delete) + + get("/media_proxy_caches", MediaProxyCacheController, :index) + post("/media_proxy_caches/delete", MediaProxyCacheController, :delete) + post("/media_proxy_caches/purge", MediaProxyCacheController, :purge) end scope "/api/pleroma/emoji", Pleroma.Web.PleromaAPI do -- cgit v1.2.3 From 2e8a236cef28c0b754aecb04a5c60c3b7655c5a6 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Sun, 14 Jun 2020 21:02:57 +0300 Subject: fix invalidates media url's --- lib/pleroma/application.ex | 3 +- lib/pleroma/plugs/uploaded_media.ex | 16 ++- lib/pleroma/web/media_proxy/invalidation.ex | 29 +++-- lib/pleroma/web/media_proxy/invalidations/http.ex | 8 +- .../web/media_proxy/invalidations/script.ex | 36 +++--- lib/pleroma/web/media_proxy/media_proxy.ex | 35 +++++- .../web/media_proxy/media_proxy_controller.ex | 3 +- lib/pleroma/workers/attachments_cleanup_worker.ex | 129 +++++++++++---------- 8 files changed, 163 insertions(+), 96 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index 9d3d92b38..adebebc7a 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -148,7 +148,8 @@ defmodule Pleroma.Application do build_cachex("idempotency", expiration: idempotency_expiration(), limit: 2500), build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), - build_cachex("failed_proxy_url", limit: 2500) + build_cachex("failed_proxy_url", limit: 2500), + build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 94147e0c4..2f3fde002 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -10,6 +10,8 @@ defmodule Pleroma.Plugs.UploadedMedia do import Pleroma.Web.Gettext require Logger + alias Pleroma.Web.MediaProxy + @behaviour Plug # no slashes @path "media" @@ -35,8 +37,7 @@ defmodule Pleroma.Plugs.UploadedMedia do %{query_params: %{"name" => name}} = conn -> name = String.replace(name, "\"", "\\\"") - conn - |> put_resp_header("content-disposition", "filename=\"#{name}\"") + put_resp_header(conn, "content-disposition", "filename=\"#{name}\"") conn -> conn @@ -47,7 +48,8 @@ defmodule Pleroma.Plugs.UploadedMedia do with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), - {:ok, get_method} <- uploader.get_file(file) do + {:ok, get_method} <- uploader.get_file(file), + false <- media_is_deleted(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else _ -> @@ -59,6 +61,14 @@ defmodule Pleroma.Plugs.UploadedMedia do def call(conn, _opts), do: conn + defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do + MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path) + end + + defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url) + + defp media_is_deleted(_, _), do: false + defp get_media(conn, {:static_dir, directory}, _, opts) do static_opts = Map.get(opts, :static_plug_opts) diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index c037ff13e..83ff8589c 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -5,22 +5,33 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do @moduledoc false - @callback purge(list(String.t()), map()) :: {:ok, String.t()} | {:error, String.t()} + @callback purge(list(String.t()), Keyword.t()) :: {:ok, list(String.t())} | {:error, String.t()} alias Pleroma.Config + alias Pleroma.Web.MediaProxy - @spec purge(list(String.t())) :: {:ok, String.t()} | {:error, String.t()} + @spec enabled?() :: boolean() + def enabled?, do: Config.get([:media_proxy, :invalidation, :enabled]) + + @spec purge(list(String.t()) | String.t()) :: {:ok, list(String.t())} | {:error, String.t()} def purge(urls) do - [:media_proxy, :invalidation, :enabled] - |> Config.get() - |> do_purge(urls) + prepared_urls = prepare_urls(urls) + + if enabled?() do + do_purge(prepared_urls) + else + {:ok, prepared_urls} + end end - defp do_purge(true, urls) do + defp do_purge(urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) - options = Config.get(provider) - provider.purge(urls, options) + provider.purge(urls, Config.get(provider)) end - defp do_purge(_, _), do: :ok + def prepare_urls(urls) do + urls + |> List.wrap() + |> Enum.map(&MediaProxy.url(&1)) + end end diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 07248df6e..3694b56e8 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -10,9 +10,9 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do @impl Pleroma.Web.MediaProxy.Invalidation def purge(urls, opts) do - method = Map.get(opts, :method, :purge) - headers = Map.get(opts, :headers, []) - options = Map.get(opts, :options, []) + method = Keyword.get(opts, :method, :purge) + headers = Keyword.get(opts, :headers, []) + options = Keyword.get(opts, :options, []) Logger.debug("Running cache purge: #{inspect(urls)}") @@ -22,7 +22,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do end end) - {:ok, "success"} + {:ok, urls} end defp do_purge(method, url, headers, options) do diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 6be782132..d41d647bb 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,32 +10,34 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, %{script_path: script_path} = _options) do + def purge(urls, opts) do args = urls |> List.wrap() |> Enum.uniq() |> Enum.join(" ") - path = Path.expand(script_path) - - Logger.debug("Running cache purge: #{inspect(urls)}, #{path}") - - case do_purge(path, [args]) do - {result, exit_status} when exit_status > 0 -> - Logger.error("Error while cache purge: #{inspect(result)}") - {:error, inspect(result)} - - _ -> - {:ok, "success"} - end + opts + |> Keyword.get(:script_path, nil) + |> do_purge([args]) + |> handle_result(urls) end - def purge(_, _), do: {:error, "not found script path"} - - defp do_purge(path, args) do + defp do_purge(script_path, args) when is_binary(script_path) do + path = Path.expand(script_path) + Logger.debug("Running cache purge: #{inspect(args)}, #{inspect(path)}") System.cmd(path, args) rescue - error -> {inspect(error), 1} + error -> error + end + + defp do_purge(_, _), do: {:error, "not found script path"} + + defp handle_result({_result, 0}, urls), do: {:ok, urls} + defp handle_result({:error, error}, urls), do: handle_result(error, urls) + + defp handle_result(error, _) do + Logger.error("Error while cache purge: #{inspect(error)}") + {:error, inspect(error)} end end diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index b2b524524..59ca217ab 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -6,20 +6,53 @@ defmodule Pleroma.Web.MediaProxy do alias Pleroma.Config alias Pleroma.Upload alias Pleroma.Web + alias Pleroma.Web.MediaProxy.Invalidation @base64_opts [padding: false] + @spec in_deleted_urls(String.t()) :: boolean() + def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1) + + def remove_from_deleted_urls(urls) when is_list(urls) do + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) + end) + end + + def remove_from_deleted_urls(url) when is_binary(url) do + Cachex.del(:deleted_urls_cache, url(url)) + end + + def put_in_deleted_urls(urls) when is_list(urls) do + Cachex.execute!(:deleted_urls_cache, fn cache -> + Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) + end) + end + + def put_in_deleted_urls(url) when is_binary(url) do + Cachex.put(:deleted_urls_cache, url(url), true) + end + def url(url) when is_nil(url) or url == "", do: nil def url("/" <> _ = url), do: url def url(url) do - if disabled?() or local?(url) or whitelisted?(url) do + if disabled?() or not is_url_proxiable?(url) do url else encode_url(url) end end + @spec is_url_proxiable?(String.t()) :: boolean() + def is_url_proxiable?(url) do + if local?(url) or whitelisted?(url) do + false + else + true + end + end + defp disabled?, do: !Config.get([:media_proxy, :enabled], false) defp local?(url), do: String.starts_with?(url, Pleroma.Web.base_url()) diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index 4657a4383..ff0158d83 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -14,10 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), + {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)}, :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else - false -> + error when error in [false, {:in_deleted_urls, true}] -> send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) {:error, :invalid_signature} -> diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 49352db2a..4ad19c0fc 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -23,8 +23,25 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do Enum.map(attachment["url"], & &1["href"]) end) - names = Enum.map(attachments, & &1["name"]) + # find all objects for copies of the attachments, name and actor doesn't matter here + hrefs + |> fetch_objects + |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) + |> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> + with 1 <- count do + {ids ++ [id], hrefs ++ [href]} + else + _ -> {ids ++ [id], hrefs} + end + end) + |> do_clean + + {:ok, :success} + end + def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + + defp do_clean({object_ids, attachment_urls}) do uploader = Pleroma.Config.get([Pleroma.Upload, :uploader]) prefix = @@ -39,68 +56,60 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do "/" ) - # find all objects for copies of the attachments, name and actor doesn't matter here - object_ids_and_hrefs = - from(o in Object, - where: - fragment( - "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", - o.data, - o.data, - ^hrefs - ) - ) - # The query above can be time consumptive on large instances until we - # refactor how uploads are stored - |> Repo.all(timeout: :infinity) - # we should delete 1 object for any given attachment, but don't delete - # files if there are more than 1 object for it - |> Enum.reduce(%{}, fn %{ - id: id, - data: %{ - "url" => [%{"href" => href}], - "actor" => obj_actor, - "name" => name - } - }, - acc -> - Map.update(acc, href, %{id: id, count: 1}, fn val -> - case obj_actor == actor and name in names do - true -> - # set id of the actor's object that will be deleted - %{val | id: id, count: val.count + 1} - - false -> - # another actor's object, just increase count to not delete file - %{val | count: val.count + 1} - end - end) - end) - |> Enum.map(fn {href, %{id: id, count: count}} -> - # only delete files that have single instance - with 1 <- count do - href - |> String.trim_leading("#{base_url}/#{prefix}") - |> uploader.delete_file() - - {id, href} - else - _ -> {id, nil} - end - end) - - object_ids = Enum.map(object_ids_and_hrefs, fn {id, _} -> id end) + Enum.each(attachment_urls, fn href -> + href + |> String.trim_leading("#{base_url}/#{prefix}") + |> uploader.delete_file() + end) - from(o in Object, where: o.id in ^object_ids) - |> Repo.delete_all() + delete_objects(object_ids) + end - object_ids_and_hrefs - |> Enum.filter(fn {_, href} -> not is_nil(href) end) - |> Enum.map(&elem(&1, 1)) - |> Pleroma.Web.MediaProxy.Invalidation.purge() + defp delete_objects([_ | _] = object_ids) do + Repo.delete_all(from(o in Object, where: o.id in ^object_ids)) + end - {:ok, :success} + defp delete_objects(_), do: :ok + + # we should delete 1 object for any given attachment, but don't delete + # files if there are more than 1 object for it + def prepare_objects(objects, actor, names) do + objects + |> Enum.reduce(%{}, fn %{ + id: id, + data: %{ + "url" => [%{"href" => href}], + "actor" => obj_actor, + "name" => name + } + }, + acc -> + Map.update(acc, href, %{id: id, count: 1}, fn val -> + case obj_actor == actor and name in names do + true -> + # set id of the actor's object that will be deleted + %{val | id: id, count: val.count + 1} + + false -> + # another actor's object, just increase count to not delete file + %{val | count: val.count + 1} + end + end) + end) end - def perform(%{"op" => "cleanup_attachments", "object" => _object}, _job), do: {:ok, :skip} + def fetch_objects(hrefs) do + from(o in Object, + where: + fragment( + "to_jsonb(array(select jsonb_array_elements((?)#>'{url}') ->> 'href' where jsonb_typeof((?)#>'{url}') = 'array'))::jsonb \\?| (?)", + o.data, + o.data, + ^hrefs + ) + ) + # The query above can be time consumptive on large instances until we + # refactor how uploads are stored + |> Repo.all(timeout: :infinity) + end end -- cgit v1.2.3 From b7df7436c813bfcb4f27ac64c85ebc1507153601 Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 15 Jun 2020 12:27:13 +0200 Subject: Conversations: Return last dm for conversation, not last message. --- lib/pleroma/conversation/participation.ex | 11 +++++++---- lib/pleroma/web/activity_pub/activity_pub.ex | 7 ++++--- lib/pleroma/web/mastodon_api/views/conversation_view.ex | 11 +++++++---- 3 files changed, 18 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index ce7bd2396..8bc3e85d6 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -162,10 +162,13 @@ defmodule Pleroma.Conversation.Participation do for_user(user, params) |> Enum.map(fn participation -> activity_id = - ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ - user: user, - blocking_user: user - }) + ActivityPub.fetch_latest_direct_activity_id_for_context( + participation.conversation.ap_id, + %{ + user: user, + blocking_user: user + } + ) %{ participation diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c9dc6135c..3e4f3ad30 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -210,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do conversation = Repo.preload(conversation, :participations) last_activity_id = - fetch_latest_activity_id_for_context(conversation.ap_id, %{ + fetch_latest_direct_activity_id_for_context(conversation.ap_id, %{ user: user, blocking_user: user }) @@ -517,11 +517,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.all() end - @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) :: + @spec fetch_latest_direct_activity_id_for_context(String.t(), keyword() | map()) :: FlakeId.Ecto.CompatType.t() | nil - def fetch_latest_activity_id_for_context(context, opts \\ %{}) do + def fetch_latest_direct_activity_id_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(Map.merge(%{skip_preload: true}, opts)) + |> restrict_visibility(%{visibility: "direct"}) |> limit(1) |> select([a], a.id) |> Repo.one() diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index fbe618377..06f0c1728 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -23,10 +23,13 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do last_activity_id = with nil <- participation.last_activity_id do - ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ - user: user, - blocking_user: user - }) + ActivityPub.fetch_latest_direct_activity_id_for_context( + participation.conversation.ap_id, + %{ + user: user, + blocking_user: user + } + ) end activity = Activity.get_by_id_with_object(last_activity_id) -- cgit v1.2.3 From 62b8c31b7a84dadb2a46861fe0f2dd1dbf9d40f0 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 15 Jun 2020 14:55:00 +0300 Subject: added tests --- .../controllers/media_proxy_cache_controller.ex | 31 +++++++++++++++++++--- .../web/media_proxy/invalidations/script.ex | 2 +- 2 files changed, 29 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex index 7b28f7c72..e3fa0ac28 100644 --- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -7,6 +7,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do alias Pleroma.Plugs.OAuthScopesPlug alias Pleroma.Web.ApiSpec.Admin, as: Spec + alias Pleroma.Web.MediaProxy plug(Pleroma.Web.ApiSpec.CastAndValidate) @@ -24,15 +25,39 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do defdelegate open_api_operation(action), to: Spec.MediaProxyCacheOperation - def index(%{assigns: %{user: _}} = conn, _) do - render(conn, "index.json", urls: []) + def index(%{assigns: %{user: _}} = conn, params) do + cursor = + :deleted_urls_cache + |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}]) + |> :qlc.cursor() + + urls = + case params.page do + 1 -> + :qlc.next_answers(cursor, params.page_size) + + _ -> + :qlc.next_answers(cursor, (params.page - 1) * params.page_size) + :qlc.next_answers(cursor, params.page_size) + end + + :qlc.delete_cursor(cursor) + + render(conn, "index.json", urls: urls) end def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do + MediaProxy.remove_from_deleted_urls(urls) render(conn, "index.json", urls: urls) end - def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: _ban}} = conn, _) do + def purge(%{assigns: %{user: _}, body_params: %{urls: urls, ban: ban}} = conn, _) do + MediaProxy.Invalidation.purge(urls) + + if ban do + MediaProxy.put_in_deleted_urls(urls) + end + render(conn, "index.json", urls: urls) end end diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index d41d647bb..0217b119d 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts) do + def purge(urls, opts \\ %{}) do args = urls |> List.wrap() -- cgit v1.2.3 From e1ee8bc1da17a356c88b535db7a9228fccc5251f Mon Sep 17 00:00:00 2001 From: lain Date: Mon, 15 Jun 2020 14:29:34 +0200 Subject: User: update_follower_count refactor. --- lib/pleroma/user.ex | 32 ++++++++++---------------------- 1 file changed, 10 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 52ac9052b..39a9e13e8 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -747,7 +747,6 @@ defmodule Pleroma.User do follower |> update_following_count() - |> set_cache() end end @@ -776,7 +775,6 @@ defmodule Pleroma.User do {:ok, follower} = follower |> update_following_count() - |> set_cache() {:ok, follower, followed} @@ -1128,35 +1126,25 @@ defmodule Pleroma.User do ]) end + @spec update_follower_count(User.t()) :: {:ok, User.t()} def update_follower_count(%User{} = user) do if user.local or !Pleroma.Config.get([:instance, :external_user_synchronization]) do - follower_count_query = - User.Query.build(%{followers: user, deactivated: false}) - |> select([u], %{count: count(u.id)}) - - User - |> where(id: ^user.id) - |> join(:inner, [u], s in subquery(follower_count_query)) - |> update([u, s], - set: [follower_count: s.count] - ) - |> select([u], u) - |> Repo.update_all([]) - |> case do - {1, [user]} -> set_cache(user) - _ -> {:error, user} - end + follower_count = FollowingRelationship.follower_count(user) + + user + |> follow_information_changeset(%{follower_count: follower_count}) + |> update_and_set_cache else {:ok, maybe_fetch_follow_information(user)} end end - @spec update_following_count(User.t()) :: User.t() + @spec update_following_count(User.t()) :: {:ok, User.t()} def update_following_count(%User{local: false} = user) do if Pleroma.Config.get([:instance, :external_user_synchronization]) do - maybe_fetch_follow_information(user) + {:ok, maybe_fetch_follow_information(user)} else - user + {:ok, user} end end @@ -1165,7 +1153,7 @@ defmodule Pleroma.User do user |> follow_information_changeset(%{following_count: following_count}) - |> Repo.update!() + |> update_and_set_cache() end def set_unread_conversation_count(%User{local: true} = user) do -- cgit v1.2.3 From b02311079961c5193af1c144516a3caeee72b582 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Mon, 15 Jun 2020 20:47:02 +0300 Subject: fixed a visibility of functions --- lib/pleroma/workers/attachments_cleanup_worker.ex | 31 +++++++++++------------ 1 file changed, 15 insertions(+), 16 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/workers/attachments_cleanup_worker.ex b/lib/pleroma/workers/attachments_cleanup_worker.ex index 4ad19c0fc..8deeabda0 100644 --- a/lib/pleroma/workers/attachments_cleanup_worker.ex +++ b/lib/pleroma/workers/attachments_cleanup_worker.ex @@ -18,22 +18,11 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do }, _job ) do - hrefs = - Enum.flat_map(attachments, fn attachment -> - Enum.map(attachment["url"], & &1["href"]) - end) - - # find all objects for copies of the attachments, name and actor doesn't matter here - hrefs + attachments + |> Enum.flat_map(fn item -> Enum.map(item["url"], & &1["href"]) end) |> fetch_objects |> prepare_objects(actor, Enum.map(attachments, & &1["name"])) - |> Enum.reduce({[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> - with 1 <- count do - {ids ++ [id], hrefs ++ [href]} - else - _ -> {ids ++ [id], hrefs} - end - end) + |> filter_objects |> do_clean {:ok, :success} @@ -73,7 +62,17 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do # we should delete 1 object for any given attachment, but don't delete # files if there are more than 1 object for it - def prepare_objects(objects, actor, names) do + defp filter_objects(objects) do + Enum.reduce(objects, {[], []}, fn {href, %{id: id, count: count}}, {ids, hrefs} -> + with 1 <- count do + {ids ++ [id], hrefs ++ [href]} + else + _ -> {ids ++ [id], hrefs} + end + end) + end + + defp prepare_objects(objects, actor, names) do objects |> Enum.reduce(%{}, fn %{ id: id, @@ -98,7 +97,7 @@ defmodule Pleroma.Workers.AttachmentsCleanupWorker do end) end - def fetch_objects(hrefs) do + defp fetch_objects(hrefs) do from(o in Object, where: fragment( -- cgit v1.2.3 From 9a4fde97661595630ea840917ef83b4786f2e2d3 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sun, 31 May 2020 10:46:02 +0300 Subject: Mogrify args as custom tuples --- lib/mix/tasks/pleroma/config.ex | 10 +- lib/pleroma/config/config_db.ex | 276 +++++++++------------ lib/pleroma/config/transfer_task.ex | 12 +- lib/pleroma/config/type/atom.ex | 22 ++ lib/pleroma/config/type/binary_value.ex | 23 ++ .../web/admin_api/controllers/config_controller.ex | 34 +-- lib/pleroma/web/admin_api/views/config_view.ex | 19 +- 7 files changed, 196 insertions(+), 200 deletions(-) create mode 100644 lib/pleroma/config/type/atom.ex create mode 100644 lib/pleroma/config/type/binary_value.ex (limited to 'lib') diff --git a/lib/mix/tasks/pleroma/config.ex b/lib/mix/tasks/pleroma/config.ex index 5c9ef6904..f1b3a8766 100644 --- a/lib/mix/tasks/pleroma/config.ex +++ b/lib/mix/tasks/pleroma/config.ex @@ -72,8 +72,7 @@ defmodule Mix.Tasks.Pleroma.Config do group |> Pleroma.Config.Loader.filter_group(settings) |> Enum.each(fn {key, value} -> - key = inspect(key) - {:ok, _} = ConfigDB.update_or_create(%{group: inspect(group), key: key, value: value}) + {:ok, _} = ConfigDB.update_or_create(%{group: group, key: key, value: value}) shell_info("Settings for key #{key} migrated.") end) @@ -131,12 +130,9 @@ defmodule Mix.Tasks.Pleroma.Config do end defp write(config, file) do - value = - config.value - |> ConfigDB.from_binary() - |> inspect(limit: :infinity) + value = inspect(config.value, limit: :infinity) - IO.write(file, "config #{config.group}, #{config.key}, #{value}\r\n\r\n") + IO.write(file, "config #{inspect(config.group)}, #{inspect(config.key)}, #{value}\r\n\r\n") config end diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 2b43d4c36..39b37c42e 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -6,7 +6,7 @@ defmodule Pleroma.ConfigDB do use Ecto.Schema import Ecto.Changeset - import Ecto.Query + import Ecto.Query, only: [select: 3] import Pleroma.Web.Gettext alias __MODULE__ @@ -14,16 +14,6 @@ defmodule Pleroma.ConfigDB do @type t :: %__MODULE__{} - @full_key_update [ - {:pleroma, :ecto_repos}, - {:quack, :meta}, - {:mime, :types}, - {:cors_plug, [:max_age, :methods, :expose, :headers]}, - {:auto_linker, :opts}, - {:swarm, :node_blacklist}, - {:logger, :backends} - ] - @full_subkey_update [ {:pleroma, :assets, :mascots}, {:pleroma, :emoji, :groups}, @@ -32,14 +22,10 @@ defmodule Pleroma.ConfigDB do {:pleroma, :mrf_keyword, :replace} ] - @regex ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u - - @delimiters ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}] - schema "config" do - field(:key, :string) - field(:group, :string) - field(:value, :binary) + field(:key, Pleroma.Config.Type.Atom) + field(:group, Pleroma.Config.Type.Atom) + field(:value, Pleroma.Config.Type.BinaryValue) field(:db, {:array, :string}, virtual: true, default: []) timestamps() @@ -51,10 +37,6 @@ defmodule Pleroma.ConfigDB do |> select([c], {c.group, c.key, c.value}) |> Repo.all() |> Enum.reduce([], fn {group, key, value}, acc -> - group = ConfigDB.from_string(group) - key = ConfigDB.from_string(key) - value = from_binary(value) - Keyword.update(acc, group, [{key, value}], &Keyword.merge(&1, [{key, value}])) end) end @@ -64,50 +46,41 @@ defmodule Pleroma.ConfigDB do @spec changeset(ConfigDB.t(), map()) :: Changeset.t() def changeset(config, params \\ %{}) do - params = Map.put(params, :value, transform(params[:value])) - config |> cast(params, [:key, :group, :value]) |> validate_required([:key, :group, :value]) |> unique_constraint(:key, name: :config_group_key_index) end - @spec create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} - def create(params) do + defp create(params) do %ConfigDB{} |> changeset(params) |> Repo.insert() end - @spec update(ConfigDB.t(), map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} - def update(%ConfigDB{} = config, %{value: value}) do + defp update(%ConfigDB{} = config, %{value: value}) do config |> changeset(%{value: value}) |> Repo.update() end - @spec get_db_keys(ConfigDB.t()) :: [String.t()] - def get_db_keys(%ConfigDB{} = config) do - config.value - |> ConfigDB.from_binary() - |> get_db_keys(config.key) - end - @spec get_db_keys(keyword(), any()) :: [String.t()] def get_db_keys(value, key) do - if Keyword.keyword?(value) do - value |> Keyword.keys() |> Enum.map(&convert(&1)) - else - [convert(key)] - end + keys = + if Keyword.keyword?(value) do + Keyword.keys(value) + else + [key] + end + + Enum.map(keys, &to_json_types(&1)) end @spec merge_group(atom(), atom(), keyword(), keyword()) :: keyword() def merge_group(group, key, old_value, new_value) do - new_keys = to_map_set(new_value) + new_keys = to_mapset(new_value) - intersect_keys = - old_value |> to_map_set() |> MapSet.intersection(new_keys) |> MapSet.to_list() + intersect_keys = old_value |> to_mapset() |> MapSet.intersection(new_keys) |> MapSet.to_list() merged_value = ConfigDB.merge(old_value, new_value) @@ -120,12 +93,10 @@ defmodule Pleroma.ConfigDB do [] end) |> List.flatten() - |> Enum.reduce(merged_value, fn subkey, acc -> - Keyword.put(acc, subkey, new_value[subkey]) - end) + |> Enum.reduce(merged_value, &Keyword.put(&2, &1, new_value[&1])) end - defp to_map_set(keyword) do + defp to_mapset(keyword) do keyword |> Keyword.keys() |> MapSet.new() @@ -159,43 +130,39 @@ defmodule Pleroma.ConfigDB do @spec update_or_create(map()) :: {:ok, ConfigDB.t()} | {:error, Changeset.t()} def update_or_create(params) do + params = Map.put(params, :value, to_elixir_types(params[:value])) search_opts = Map.take(params, [:group, :key]) with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts), - {:partial_update, true, config} <- - {:partial_update, can_be_partially_updated?(config), config}, - old_value <- from_binary(config.value), - transformed_value <- do_transform(params[:value]), - {:can_be_merged, true, config} <- {:can_be_merged, is_list(transformed_value), config}, - new_value <- - merge_group( - ConfigDB.from_string(config.group), - ConfigDB.from_string(config.key), - old_value, - transformed_value - ) do - ConfigDB.update(config, %{value: new_value}) + {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config}, + {_, true, config} <- {:can_be_merged, is_list(params[:value]), config} do + new_value = merge_group(config.group, config.key, config.value, params[:value]) + update(config, %{value: new_value}) else {reason, false, config} when reason in [:partial_update, :can_be_merged] -> - ConfigDB.update(config, params) + update(config, params) nil -> - ConfigDB.create(params) + create(params) end end defp can_be_partially_updated?(%ConfigDB{} = config), do: not only_full_update?(config) - defp only_full_update?(%ConfigDB{} = config) do - config_group = ConfigDB.from_string(config.group) - config_key = ConfigDB.from_string(config.key) - - Enum.any?(@full_key_update, fn - {group, key} when is_list(key) -> - config_group == group and config_key in key - - {group, key} -> - config_group == group and config_key == key + defp only_full_update?(%ConfigDB{group: group, key: key}) do + full_key_update = [ + {:pleroma, :ecto_repos}, + {:quack, :meta}, + {:mime, :types}, + {:cors_plug, [:max_age, :methods, :expose, :headers]}, + {:auto_linker, :opts}, + {:swarm, :node_blacklist}, + {:logger, :backends} + ] + + Enum.any?(full_key_update, fn + {s_group, s_key} -> + group == s_group and ((is_list(s_key) and key in s_key) or key == s_key) end) end @@ -205,11 +172,10 @@ defmodule Pleroma.ConfigDB do with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts), {config, sub_keys} when is_list(sub_keys) <- {config, params[:subkeys]}, - old_value <- from_binary(config.value), - keys <- Enum.map(sub_keys, &do_transform_string(&1)), - {:partial_remove, config, new_value} when new_value != [] <- - {:partial_remove, config, Keyword.drop(old_value, keys)} do - ConfigDB.update(config, %{value: new_value}) + keys <- Enum.map(sub_keys, &string_to_elixir_types(&1)), + {_, config, new_value} when new_value != [] <- + {:partial_remove, config, Keyword.drop(config.value, keys)} do + update(config, %{value: new_value}) else {:partial_remove, config, []} -> Repo.delete(config) @@ -225,37 +191,32 @@ defmodule Pleroma.ConfigDB do end end - @spec from_binary(binary()) :: term() - def from_binary(binary), do: :erlang.binary_to_term(binary) - - @spec from_binary_with_convert(binary()) :: any() - def from_binary_with_convert(binary) do - binary - |> from_binary() - |> do_convert() + @spec to_json_types(term()) :: map() | list() | boolean() | String.t() + def to_json_types(entity) when is_list(entity) do + Enum.map(entity, &to_json_types/1) end - @spec from_string(String.t()) :: atom() | no_return() - def from_string(string), do: do_transform_string(string) - - @spec convert(any()) :: any() - def convert(entity), do: do_convert(entity) + def to_json_types(%Regex{} = entity), do: inspect(entity) - defp do_convert(entity) when is_list(entity) do - for v <- entity, into: [], do: do_convert(v) + def to_json_types(entity) when is_map(entity) do + Map.new(entity, fn {k, v} -> {to_json_types(k), to_json_types(v)} end) end - defp do_convert(%Regex{} = entity), do: inspect(entity) + def to_json_types({:args, args}) when is_list(args) do + arguments = + Enum.map(args, fn + arg when is_tuple(arg) -> inspect(arg) + arg -> to_json_types(arg) + end) - defp do_convert(entity) when is_map(entity) do - for {k, v} <- entity, into: %{}, do: {do_convert(k), do_convert(v)} + %{"tuple" => [":args", arguments]} end - defp do_convert({:proxy_url, {type, :localhost, port}}) do - %{"tuple" => [":proxy_url", %{"tuple" => [do_convert(type), "localhost", port]}]} + def to_json_types({:proxy_url, {type, :localhost, port}}) do + %{"tuple" => [":proxy_url", %{"tuple" => [to_json_types(type), "localhost", port]}]} end - defp do_convert({:proxy_url, {type, host, port}}) when is_tuple(host) do + def to_json_types({:proxy_url, {type, host, port}}) when is_tuple(host) do ip = host |> :inet_parse.ntoa() @@ -264,66 +225,64 @@ defmodule Pleroma.ConfigDB do %{ "tuple" => [ ":proxy_url", - %{"tuple" => [do_convert(type), ip, port]} + %{"tuple" => [to_json_types(type), ip, port]} ] } end - defp do_convert({:proxy_url, {type, host, port}}) do + def to_json_types({:proxy_url, {type, host, port}}) do %{ "tuple" => [ ":proxy_url", - %{"tuple" => [do_convert(type), to_string(host), port]} + %{"tuple" => [to_json_types(type), to_string(host), port]} ] } end - defp do_convert({:partial_chain, entity}), do: %{"tuple" => [":partial_chain", inspect(entity)]} + def to_json_types({:partial_chain, entity}), + do: %{"tuple" => [":partial_chain", inspect(entity)]} - defp do_convert(entity) when is_tuple(entity) do + def to_json_types(entity) when is_tuple(entity) do value = entity |> Tuple.to_list() - |> do_convert() + |> to_json_types() %{"tuple" => value} end - defp do_convert(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do + def to_json_types(entity) when is_binary(entity), do: entity + + def to_json_types(entity) when is_boolean(entity) or is_number(entity) or is_nil(entity) do entity end - defp do_convert(entity) - when is_atom(entity) and entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do + def to_json_types(entity) when entity in [:"tlsv1.1", :"tlsv1.2", :"tlsv1.3"] do ":#{entity}" end - defp do_convert(entity) when is_atom(entity), do: inspect(entity) + def to_json_types(entity) when is_atom(entity), do: inspect(entity) - defp do_convert(entity) when is_binary(entity), do: entity + @spec to_elixir_types(boolean() | String.t() | map() | list()) :: term() + def to_elixir_types(%{"tuple" => [":args", args]}) when is_list(args) do + arguments = + Enum.map(args, fn arg -> + if String.contains?(arg, ["{", "}"]) do + {elem, []} = Code.eval_string(arg) + elem + else + to_elixir_types(arg) + end + end) - @spec transform(any()) :: binary() | no_return() - def transform(entity) when is_binary(entity) or is_map(entity) or is_list(entity) do - entity - |> do_transform() - |> to_binary() + {:args, arguments} end - def transform(entity), do: to_binary(entity) - - @spec transform_with_out_binary(any()) :: any() - def transform_with_out_binary(entity), do: do_transform(entity) - - @spec to_binary(any()) :: binary() - def to_binary(entity), do: :erlang.term_to_binary(entity) - - defp do_transform(%Regex{} = entity), do: entity - - defp do_transform(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do - {:proxy_url, {do_transform_string(type), parse_host(host), port}} + def to_elixir_types(%{"tuple" => [":proxy_url", %{"tuple" => [type, host, port]}]}) do + {:proxy_url, {string_to_elixir_types(type), parse_host(host), port}} end - defp do_transform(%{"tuple" => [":partial_chain", entity]}) do + def to_elixir_types(%{"tuple" => [":partial_chain", entity]}) do {partial_chain, []} = entity |> String.replace(~r/[^\w|^{:,[|^,|^[|^\]^}|^\/|^\.|^"]^\s/, "") @@ -332,25 +291,51 @@ defmodule Pleroma.ConfigDB do {:partial_chain, partial_chain} end - defp do_transform(%{"tuple" => entity}) do - Enum.reduce(entity, {}, fn val, acc -> Tuple.append(acc, do_transform(val)) end) + def to_elixir_types(%{"tuple" => entity}) do + Enum.reduce(entity, {}, &Tuple.append(&2, to_elixir_types(&1))) end - defp do_transform(entity) when is_map(entity) do - for {k, v} <- entity, into: %{}, do: {do_transform(k), do_transform(v)} + def to_elixir_types(entity) when is_map(entity) do + Map.new(entity, fn {k, v} -> {to_elixir_types(k), to_elixir_types(v)} end) end - defp do_transform(entity) when is_list(entity) do - for v <- entity, into: [], do: do_transform(v) + def to_elixir_types(entity) when is_list(entity) do + Enum.map(entity, &to_elixir_types/1) end - defp do_transform(entity) when is_binary(entity) do + def to_elixir_types(entity) when is_binary(entity) do entity |> String.trim() - |> do_transform_string() + |> string_to_elixir_types() + end + + def to_elixir_types(entity), do: entity + + @spec string_to_elixir_types(String.t()) :: + atom() | Regex.t() | module() | String.t() | no_return() + def string_to_elixir_types("~r" <> _pattern = regex) do + pattern = + ~r/^~r(?'delimiter'[\/|"'([{<]{1})(?'pattern'.+)[\/|"')\]}>]{1}(?'modifier'[uismxfU]*)/u + + delimiters = ["/", "|", "\"", "'", {"(", ")"}, {"[", "]"}, {"{", "}"}, {"<", ">"}] + + with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <- + Regex.named_captures(pattern, regex), + {:ok, {leading, closing}} <- find_valid_delimiter(delimiters, pattern, regex_delimiter), + {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do + result + end end - defp do_transform(entity), do: entity + def string_to_elixir_types(":" <> atom), do: String.to_atom(atom) + + def string_to_elixir_types(value) do + if is_module_name?(value) do + String.to_existing_atom("Elixir." <> value) + else + value + end + end defp parse_host("localhost"), do: :localhost @@ -387,25 +372,6 @@ defmodule Pleroma.ConfigDB do end end - defp do_transform_string("~r" <> _pattern = regex) do - with %{"modifier" => modifier, "pattern" => pattern, "delimiter" => regex_delimiter} <- - Regex.named_captures(@regex, regex), - {:ok, {leading, closing}} <- find_valid_delimiter(@delimiters, pattern, regex_delimiter), - {result, _} <- Code.eval_string("~r#{leading}#{pattern}#{closing}#{modifier}") do - result - end - end - - defp do_transform_string(":" <> atom), do: String.to_atom(atom) - - defp do_transform_string(value) do - if is_module_name?(value) do - String.to_existing_atom("Elixir." <> value) - else - value - end - end - @spec is_module_name?(String.t()) :: boolean() def is_module_name?(string) do Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or diff --git a/lib/pleroma/config/transfer_task.ex b/lib/pleroma/config/transfer_task.ex index c02b70e96..eb86b8ff4 100644 --- a/lib/pleroma/config/transfer_task.ex +++ b/lib/pleroma/config/transfer_task.ex @@ -28,10 +28,6 @@ defmodule Pleroma.Config.TransferTask do {:pleroma, Pleroma.Captcha, [:seconds_valid]}, {:pleroma, Pleroma.Upload, [:proxy_remote]}, {:pleroma, :instance, [:upload_limit]}, - {:pleroma, :email_notifications, [:digest]}, - {:pleroma, :oauth2, [:clean_expired_tokens]}, - {:pleroma, Pleroma.ActivityExpiration, [:enabled]}, - {:pleroma, Pleroma.ScheduledActivity, [:enabled]}, {:pleroma, :gopher, [:enabled]} ] @@ -48,7 +44,7 @@ defmodule Pleroma.Config.TransferTask do {logger, other} = (Repo.all(ConfigDB) ++ deleted_settings) - |> Enum.map(&transform_and_merge/1) + |> Enum.map(&merge_with_default/1) |> Enum.split_with(fn {group, _, _, _} -> group in [:logger, :quack] end) logger @@ -92,11 +88,7 @@ defmodule Pleroma.Config.TransferTask do end end - defp transform_and_merge(%{group: group, key: key, value: value} = setting) do - group = ConfigDB.from_string(group) - key = ConfigDB.from_string(key) - value = ConfigDB.from_binary(value) - + defp merge_with_default(%{group: group, key: key, value: value} = setting) do default = Config.Holder.default_config(group, key) merged = diff --git a/lib/pleroma/config/type/atom.ex b/lib/pleroma/config/type/atom.ex new file mode 100644 index 000000000..387869284 --- /dev/null +++ b/lib/pleroma/config/type/atom.ex @@ -0,0 +1,22 @@ +defmodule Pleroma.Config.Type.Atom do + use Ecto.Type + + def type, do: :atom + + def cast(key) when is_atom(key) do + {:ok, key} + end + + def cast(key) when is_binary(key) do + {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} + end + + def cast(_), do: :error + + def load(key) do + {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} + end + + def dump(key) when is_atom(key), do: {:ok, inspect(key)} + def dump(_), do: :error +end diff --git a/lib/pleroma/config/type/binary_value.ex b/lib/pleroma/config/type/binary_value.ex new file mode 100644 index 000000000..17c5524a3 --- /dev/null +++ b/lib/pleroma/config/type/binary_value.ex @@ -0,0 +1,23 @@ +defmodule Pleroma.Config.Type.BinaryValue do + use Ecto.Type + + def type, do: :term + + def cast(value) when is_binary(value) do + if String.valid?(value) do + {:ok, value} + else + {:ok, :erlang.binary_to_term(value)} + end + end + + def cast(value), do: {:ok, value} + + def load(value) when is_binary(value) do + {:ok, :erlang.binary_to_term(value)} + end + + def dump(value) do + {:ok, :erlang.term_to_binary(value)} + end +end diff --git a/lib/pleroma/web/admin_api/controllers/config_controller.ex b/lib/pleroma/web/admin_api/controllers/config_controller.ex index d6e2019bc..7f60470cb 100644 --- a/lib/pleroma/web/admin_api/controllers/config_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/config_controller.ex @@ -33,7 +33,11 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do def show(conn, %{only_db: true}) do with :ok <- configurable_from_database() do configs = Pleroma.Repo.all(ConfigDB) - render(conn, "index.json", %{configs: configs}) + + render(conn, "index.json", %{ + configs: configs, + need_reboot: Restarter.Pleroma.need_reboot?() + }) end end @@ -61,17 +65,20 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do value end - %{ - group: ConfigDB.convert(group), - key: ConfigDB.convert(key), - value: ConfigDB.convert(merged_value) + %ConfigDB{ + group: group, + key: key, + value: merged_value } |> Pleroma.Maps.put_if_present(:db, db) end) end) |> List.flatten() - json(conn, %{configs: merged, need_reboot: Restarter.Pleroma.need_reboot?()}) + render(conn, "index.json", %{ + configs: merged, + need_reboot: Restarter.Pleroma.need_reboot?() + }) end end @@ -91,24 +98,17 @@ defmodule Pleroma.Web.AdminAPI.ConfigController do {deleted, updated} = results - |> Enum.map(fn {:ok, config} -> - Map.put(config, :db, ConfigDB.get_db_keys(config)) - end) - |> Enum.split_with(fn config -> - Ecto.get_meta(config, :state) == :deleted + |> Enum.map(fn {:ok, %{key: key, value: value} = config} -> + Map.put(config, :db, ConfigDB.get_db_keys(value, key)) end) + |> Enum.split_with(&(Ecto.get_meta(&1, :state) == :deleted)) Config.TransferTask.load_and_update_env(deleted, false) if not Restarter.Pleroma.need_reboot?() do changed_reboot_settings? = (updated ++ deleted) - |> Enum.any?(fn config -> - group = ConfigDB.from_string(config.group) - key = ConfigDB.from_string(config.key) - value = ConfigDB.from_binary(config.value) - Config.TransferTask.pleroma_need_restart?(group, key, value) - end) + |> Enum.any?(&Config.TransferTask.pleroma_need_restart?(&1.group, &1.key, &1.value)) if changed_reboot_settings?, do: Restarter.Pleroma.need_reboot() end diff --git a/lib/pleroma/web/admin_api/views/config_view.ex b/lib/pleroma/web/admin_api/views/config_view.ex index 587ef760e..d2d8b5907 100644 --- a/lib/pleroma/web/admin_api/views/config_view.ex +++ b/lib/pleroma/web/admin_api/views/config_view.ex @@ -5,23 +5,20 @@ defmodule Pleroma.Web.AdminAPI.ConfigView do use Pleroma.Web, :view + alias Pleroma.ConfigDB + def render("index.json", %{configs: configs} = params) do - map = %{ - configs: render_many(configs, __MODULE__, "show.json", as: :config) + %{ + configs: render_many(configs, __MODULE__, "show.json", as: :config), + need_reboot: params[:need_reboot] } - - if params[:need_reboot] do - Map.put(map, :need_reboot, true) - else - map - end end def render("show.json", %{config: config}) do map = %{ - key: config.key, - group: config.group, - value: Pleroma.ConfigDB.from_binary_with_convert(config.value) + key: ConfigDB.to_json_types(config.key), + group: ConfigDB.to_json_types(config.group), + value: ConfigDB.to_json_types(config.value) } if config.db != [] do -- cgit v1.2.3 From e1603ac8fee2a660c3dc510dee5967e0fd1bbd98 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sun, 31 May 2020 13:25:04 +0300 Subject: fix attemps to merge map --- lib/pleroma/config/config_db.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 39b37c42e..70be17ecf 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -135,7 +135,8 @@ defmodule Pleroma.ConfigDB do with %ConfigDB{} = config <- ConfigDB.get_by_params(search_opts), {_, true, config} <- {:partial_update, can_be_partially_updated?(config), config}, - {_, true, config} <- {:can_be_merged, is_list(params[:value]), config} do + {_, true, config} <- + {:can_be_merged, is_list(params[:value]) and is_list(config.value), config} do new_value = merge_group(config.group, config.key, config.value, params[:value]) update(config, %{value: new_value}) else -- cgit v1.2.3 From 32c6576b600e2f24310f429f4b2391f95a5b5ba0 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Sun, 31 May 2020 14:42:15 +0300 Subject: naming --- lib/pleroma/config/config_db.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 70be17ecf..30bd51b05 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -331,7 +331,7 @@ defmodule Pleroma.ConfigDB do def string_to_elixir_types(":" <> atom), do: String.to_atom(atom) def string_to_elixir_types(value) do - if is_module_name?(value) do + if module_name?(value) do String.to_existing_atom("Elixir." <> value) else value @@ -373,8 +373,8 @@ defmodule Pleroma.ConfigDB do end end - @spec is_module_name?(String.t()) :: boolean() - def is_module_name?(string) do + @spec module_name?(String.t()) :: boolean() + def module_name?(string) do Regex.match?(~r/^(Pleroma|Phoenix|Tesla|Quack|Ueberauth|Swoosh)\./, string) or string in ["Oban", "Ueberauth", "ExSyslogger"] end -- cgit v1.2.3 From 3c2cee33adcd79a76b192a66f6f2d3772e2fda99 Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Tue, 16 Jun 2020 17:50:33 +0300 Subject: moving custom ecto types in context folders --- lib/pleroma/config/config_db.ex | 6 ++-- lib/pleroma/config/type/atom.ex | 22 ------------ lib/pleroma/config/type/binary_value.ex | 23 ------------- .../activity_pub/object_validators/date_time.ex | 38 ++++++++++++++++++++ .../activity_pub/object_validators/object_id.ex | 27 +++++++++++++++ .../activity_pub/object_validators/recipients.ex | 40 ++++++++++++++++++++++ .../activity_pub/object_validators/safe_text.ex | 25 ++++++++++++++ .../activity_pub/object_validators/uri.ex | 24 +++++++++++++ lib/pleroma/ecto_type/config/atom.ex | 26 ++++++++++++++ lib/pleroma/ecto_type/config/binary_value.ex | 27 +++++++++++++++ lib/pleroma/signature.ex | 4 +-- lib/pleroma/user.ex | 4 +-- lib/pleroma/web/activity_pub/object_validator.ex | 4 +-- .../object_validators/announce_validator.ex | 14 ++++---- .../object_validators/chat_message_validator.ex | 12 +++---- .../create_chat_message_validator.ex | 10 +++--- .../object_validators/create_note_validator.ex | 6 ++-- .../object_validators/delete_validator.ex | 14 ++++---- .../object_validators/emoji_react_validator.ex | 8 ++--- .../object_validators/like_validator.ex | 14 ++++---- .../object_validators/note_validator.ex | 12 +++---- .../object_validators/types/date_time.ex | 34 ------------------ .../object_validators/types/object_id.ex | 23 ------------- .../object_validators/types/recipients.ex | 36 ------------------- .../object_validators/types/safe_text.ex | 25 -------------- .../activity_pub/object_validators/types/uri.ex | 20 ----------- .../object_validators/undo_validator.ex | 8 ++--- .../object_validators/url_object_validator.ex | 8 +++-- lib/pleroma/web/activity_pub/transmogrifier.ex | 4 +-- 29 files changed, 273 insertions(+), 245 deletions(-) delete mode 100644 lib/pleroma/config/type/atom.ex delete mode 100644 lib/pleroma/config/type/binary_value.ex create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex create mode 100644 lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex create mode 100644 lib/pleroma/ecto_type/config/atom.ex create mode 100644 lib/pleroma/ecto_type/config/binary_value.ex delete mode 100644 lib/pleroma/web/activity_pub/object_validators/types/date_time.ex delete mode 100644 lib/pleroma/web/activity_pub/object_validators/types/object_id.ex delete mode 100644 lib/pleroma/web/activity_pub/object_validators/types/recipients.ex delete mode 100644 lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex delete mode 100644 lib/pleroma/web/activity_pub/object_validators/types/uri.ex (limited to 'lib') diff --git a/lib/pleroma/config/config_db.ex b/lib/pleroma/config/config_db.ex index 30bd51b05..2f4eb8581 100644 --- a/lib/pleroma/config/config_db.ex +++ b/lib/pleroma/config/config_db.ex @@ -23,9 +23,9 @@ defmodule Pleroma.ConfigDB do ] schema "config" do - field(:key, Pleroma.Config.Type.Atom) - field(:group, Pleroma.Config.Type.Atom) - field(:value, Pleroma.Config.Type.BinaryValue) + field(:key, Pleroma.EctoType.Config.Atom) + field(:group, Pleroma.EctoType.Config.Atom) + field(:value, Pleroma.EctoType.Config.BinaryValue) field(:db, {:array, :string}, virtual: true, default: []) timestamps() diff --git a/lib/pleroma/config/type/atom.ex b/lib/pleroma/config/type/atom.ex deleted file mode 100644 index 387869284..000000000 --- a/lib/pleroma/config/type/atom.ex +++ /dev/null @@ -1,22 +0,0 @@ -defmodule Pleroma.Config.Type.Atom do - use Ecto.Type - - def type, do: :atom - - def cast(key) when is_atom(key) do - {:ok, key} - end - - def cast(key) when is_binary(key) do - {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} - end - - def cast(_), do: :error - - def load(key) do - {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} - end - - def dump(key) when is_atom(key), do: {:ok, inspect(key)} - def dump(_), do: :error -end diff --git a/lib/pleroma/config/type/binary_value.ex b/lib/pleroma/config/type/binary_value.ex deleted file mode 100644 index 17c5524a3..000000000 --- a/lib/pleroma/config/type/binary_value.ex +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Pleroma.Config.Type.BinaryValue do - use Ecto.Type - - def type, do: :term - - def cast(value) when is_binary(value) do - if String.valid?(value) do - {:ok, value} - else - {:ok, :erlang.binary_to_term(value)} - end - end - - def cast(value), do: {:ok, value} - - def load(value) when is_binary(value) do - {:ok, :erlang.binary_to_term(value)} - end - - def dump(value) do - {:ok, :erlang.term_to_binary(value)} - end -end diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex new file mode 100644 index 000000000..d852c0abd --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/date_time.ex @@ -0,0 +1,38 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.DateTime do + @moduledoc """ + The AP standard defines the date fields in AP as xsd:DateTime. Elixir's + DateTime can't parse this, but it can parse the related iso8601. This + module punches the date until it looks like iso8601 and normalizes to + it. + + DateTimes without a timezone offset are treated as UTC. + + Reference: https://www.w3.org/TR/activitystreams-vocabulary/#dfn-published + """ + use Ecto.Type + + def type, do: :string + + def cast(datetime) when is_binary(datetime) do + with {:ok, datetime, _} <- DateTime.from_iso8601(datetime) do + {:ok, DateTime.to_iso8601(datetime)} + else + {:error, :missing_offset} -> cast("#{datetime}Z") + _e -> :error + end + end + + def cast(_), do: :error + + def dump(data) do + {:ok, data} + end + + def load(data) do + {:ok, data} + end +end diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex new file mode 100644 index 000000000..8034235b0 --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/object_id.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID do + use Ecto.Type + + def type, do: :string + + def cast(object) when is_binary(object) do + # Host has to be present and scheme has to be an http scheme (for now) + case URI.parse(object) do + %URI{host: nil} -> :error + %URI{host: ""} -> :error + %URI{scheme: scheme} when scheme in ["https", "http"] -> {:ok, object} + _ -> :error + end + end + + def cast(%{"id" => object}), do: cast(object) + + def cast(_), do: :error + + def dump(data), do: {:ok, data} + + def load(data), do: {:ok, data} +end diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex new file mode 100644 index 000000000..205527a96 --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/recipients.ex @@ -0,0 +1,40 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Recipients do + use Ecto.Type + + alias Pleroma.EctoType.ActivityPub.ObjectValidators.ObjectID + + def type, do: {:array, ObjectID} + + def cast(object) when is_binary(object) do + cast([object]) + end + + def cast(data) when is_list(data) do + data + |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> + case ObjectID.cast(element) do + {:ok, id} -> + {:cont, {:ok, [id | list]}} + + _ -> + {:halt, :error} + end + end) + end + + def cast(_) do + :error + end + + def dump(data) do + {:ok, data} + end + + def load(data) do + {:ok, data} + end +end diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex new file mode 100644 index 000000000..7f0405c7b --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/safe_text.ex @@ -0,0 +1,25 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.SafeText do + use Ecto.Type + + alias Pleroma.HTML + + def type, do: :string + + def cast(str) when is_binary(str) do + {:ok, HTML.filter_tags(str)} + end + + def cast(_), do: :error + + def dump(data) do + {:ok, data} + end + + def load(data) do + {:ok, data} + end +end diff --git a/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex b/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex new file mode 100644 index 000000000..2054c26be --- /dev/null +++ b/lib/pleroma/ecto_type/activity_pub/object_validators/uri.ex @@ -0,0 +1,24 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.ActivityPub.ObjectValidators.Uri do + use Ecto.Type + + def type, do: :string + + def cast(uri) when is_binary(uri) do + case URI.parse(uri) do + %URI{host: nil} -> :error + %URI{host: ""} -> :error + %URI{scheme: scheme} when scheme in ["https", "http"] -> {:ok, uri} + _ -> :error + end + end + + def cast(_), do: :error + + def dump(data), do: {:ok, data} + + def load(data), do: {:ok, data} +end diff --git a/lib/pleroma/ecto_type/config/atom.ex b/lib/pleroma/ecto_type/config/atom.ex new file mode 100644 index 000000000..df565d432 --- /dev/null +++ b/lib/pleroma/ecto_type/config/atom.ex @@ -0,0 +1,26 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.Config.Atom do + use Ecto.Type + + def type, do: :atom + + def cast(key) when is_atom(key) do + {:ok, key} + end + + def cast(key) when is_binary(key) do + {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} + end + + def cast(_), do: :error + + def load(key) do + {:ok, Pleroma.ConfigDB.string_to_elixir_types(key)} + end + + def dump(key) when is_atom(key), do: {:ok, inspect(key)} + def dump(_), do: :error +end diff --git a/lib/pleroma/ecto_type/config/binary_value.ex b/lib/pleroma/ecto_type/config/binary_value.ex new file mode 100644 index 000000000..bbd2608c5 --- /dev/null +++ b/lib/pleroma/ecto_type/config/binary_value.ex @@ -0,0 +1,27 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.EctoType.Config.BinaryValue do + use Ecto.Type + + def type, do: :term + + def cast(value) when is_binary(value) do + if String.valid?(value) do + {:ok, value} + else + {:ok, :erlang.binary_to_term(value)} + end + end + + def cast(value), do: {:ok, value} + + def load(value) when is_binary(value) do + {:ok, :erlang.binary_to_term(value)} + end + + def dump(value) do + {:ok, :erlang.term_to_binary(value)} + end +end diff --git a/lib/pleroma/signature.ex b/lib/pleroma/signature.ex index d01728361..3aa6909d2 100644 --- a/lib/pleroma/signature.ex +++ b/lib/pleroma/signature.ex @@ -5,10 +5,10 @@ defmodule Pleroma.Signature do @behaviour HTTPSignatures.Adapter + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Keys alias Pleroma.User alias Pleroma.Web.ActivityPub.ActivityPub - alias Pleroma.Web.ActivityPub.ObjectValidators.Types def key_id_to_actor_id(key_id) do uri = @@ -24,7 +24,7 @@ defmodule Pleroma.Signature do maybe_ap_id = URI.to_string(uri) - case Types.ObjectID.cast(maybe_ap_id) do + case ObjectValidators.ObjectID.cast(maybe_ap_id) do {:ok, ap_id} -> {:ok, ap_id} diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 52ac9052b..686ab0123 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -14,6 +14,7 @@ defmodule Pleroma.User do alias Pleroma.Config alias Pleroma.Conversation.Participation alias Pleroma.Delivery + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Emoji alias Pleroma.FollowingRelationship alias Pleroma.Formatter @@ -30,7 +31,6 @@ defmodule Pleroma.User do alias Pleroma.Web alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Builder - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.CommonAPI @@ -115,7 +115,7 @@ defmodule Pleroma.User do field(:is_admin, :boolean, default: false) field(:show_role, :boolean, default: true) field(:settings, :map, default: nil) - field(:uri, Types.Uri, default: nil) + field(:uri, ObjectValidators.Uri, default: nil) field(:hide_followers_count, :boolean, default: false) field(:hide_follows_count, :boolean, default: false) field(:hide_followers, :boolean, default: false) diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index c01c5f780..3d699e8a5 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do """ alias Pleroma.Object + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator @@ -17,7 +18,6 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do alias Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator alias Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator alias Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator @spec validate(map(), keyword()) :: {:ok, map(), keyword()} | {:error, any()} @@ -120,7 +120,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do def stringify_keys(object), do: object def fetch_actor(object) do - with {:ok, actor} <- Types.ObjectID.cast(object["actor"]) do + with {:ok, actor} <- ObjectValidators.ObjectID.cast(object["actor"]) do User.get_or_fetch_by_ap_id(actor) end end diff --git a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex index 40f861f47..6f757f49c 100644 --- a/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/announce_validator.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object alias Pleroma.User - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -19,14 +19,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:object, Types.ObjectID) - field(:actor, Types.ObjectID) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) field(:context, :string, autogenerate: {Utils, :generate_context_id, []}) - field(:to, Types.Recipients, default: []) - field(:cc, Types.Recipients, default: []) - field(:published, Types.DateTime) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:published, ObjectValidators.DateTime) end def cast_and_validate(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex index 138736f23..c481d79e0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/chat_message_validator.ex @@ -5,9 +5,9 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AttachmentValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset import Pleroma.Web.ActivityPub.Transmogrifier, only: [fix_emoji: 1] @@ -16,12 +16,12 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator do @derive Jason.Encoder embedded_schema do - field(:id, Types.ObjectID, primary_key: true) - field(:to, Types.Recipients, default: []) + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:to, ObjectValidators.Recipients, default: []) field(:type, :string) - field(:content, Types.SafeText) - field(:actor, Types.ObjectID) - field(:published, Types.DateTime) + field(:content, ObjectValidators.SafeText) + field(:actor, ObjectValidators.ObjectID) + field(:published, ObjectValidators.DateTime) field(:emoji, :map, default: %{}) embeds_one(:attachment, AttachmentValidator) diff --git a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex index fc582400b..7269f9ff0 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_chat_message_validator.ex @@ -7,9 +7,9 @@ # - doesn't embed, will only get the object id defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -17,11 +17,11 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateChatMessageValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) - field(:actor, Types.ObjectID) + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:actor, ObjectValidators.ObjectID) field(:type, :string) - field(:to, Types.Recipients, default: []) - field(:object, Types.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:object, ObjectValidators.ObjectID) end def cast_and_apply(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex index 926804ce7..316bd0c07 100644 --- a/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/create_note_validator.ex @@ -5,16 +5,16 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.CreateNoteValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) - field(:actor, Types.ObjectID) + field(:id, ObjectValidators.ObjectID, primary_key: true) + field(:actor, ObjectValidators.ObjectID) field(:type, :string) field(:to, {:array, :string}) field(:cc, {:array, :string}) diff --git a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex index e5d08eb5c..93a7b0e0b 100644 --- a/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/delete_validator.ex @@ -6,8 +6,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do use Ecto.Schema alias Pleroma.Activity + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.User - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -15,13 +15,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.DeleteValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:actor, Types.ObjectID) - field(:to, Types.Recipients, default: []) - field(:cc, Types.Recipients, default: []) - field(:deleted_activity_id, Types.ObjectID) - field(:object, Types.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) + field(:deleted_activity_id, ObjectValidators.ObjectID) + field(:object, ObjectValidators.ObjectID) end def cast_data(data) do diff --git a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex index e87519c59..a543af1f8 100644 --- a/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/emoji_react_validator.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ObjectValidators.Types import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -14,10 +14,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.EmojiReactValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:object, Types.ObjectID) - field(:actor, Types.ObjectID) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) field(:context, :string) field(:content, :string) field(:to, {:array, :string}, default: []) diff --git a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex index 034f25492..493e4c247 100644 --- a/lib/pleroma/web/activity_pub/object_validators/like_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/like_validator.ex @@ -5,8 +5,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do use Ecto.Schema + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.Object - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Utils import Ecto.Changeset @@ -15,13 +15,13 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:object, Types.ObjectID) - field(:actor, Types.ObjectID) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) field(:context, :string) - field(:to, Types.Recipients, default: []) - field(:cc, Types.Recipients, default: []) + field(:to, ObjectValidators.Recipients, default: []) + field(:cc, ObjectValidators.Recipients, default: []) end def cast_and_validate(data) do @@ -67,7 +67,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.LikeValidator do with {[], []} <- {to, cc}, %Object{data: %{"actor" => actor}} <- Object.get_cached_by_ap_id(object), - {:ok, actor} <- Types.ObjectID.cast(actor) do + {:ok, actor} <- ObjectValidators.ObjectID.cast(actor) do cng |> put_change(:to, [actor]) else diff --git a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex index 462a5620a..a10728ac6 100644 --- a/lib/pleroma/web/activity_pub/object_validators/note_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/note_validator.ex @@ -5,14 +5,14 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do use Ecto.Schema - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.EctoType.ActivityPub.ObjectValidators import Ecto.Changeset @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:to, {:array, :string}, default: []) field(:cc, {:array, :string}, default: []) field(:bto, {:array, :string}, default: []) @@ -22,10 +22,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:type, :string) field(:content, :string) field(:context, :string) - field(:actor, Types.ObjectID) - field(:attributedTo, Types.ObjectID) + field(:actor, ObjectValidators.ObjectID) + field(:attributedTo, ObjectValidators.ObjectID) field(:summary, :string) - field(:published, Types.DateTime) + field(:published, ObjectValidators.DateTime) # TODO: Write type field(:emoji, :map, default: %{}) field(:sensitive, :boolean, default: false) @@ -35,7 +35,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.NoteValidator do field(:like_count, :integer, default: 0) field(:announcement_count, :integer, default: 0) field(:inRepyTo, :string) - field(:uri, Types.Uri) + field(:uri, ObjectValidators.Uri) field(:likes, {:array, :string}, default: []) field(:announcements, {:array, :string}, default: []) diff --git a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex b/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex deleted file mode 100644 index 4f412fcde..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/types/date_time.ex +++ /dev/null @@ -1,34 +0,0 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.DateTime do - @moduledoc """ - The AP standard defines the date fields in AP as xsd:DateTime. Elixir's - DateTime can't parse this, but it can parse the related iso8601. This - module punches the date until it looks like iso8601 and normalizes to - it. - - DateTimes without a timezone offset are treated as UTC. - - Reference: https://www.w3.org/TR/activitystreams-vocabulary/#dfn-published - """ - use Ecto.Type - - def type, do: :string - - def cast(datetime) when is_binary(datetime) do - with {:ok, datetime, _} <- DateTime.from_iso8601(datetime) do - {:ok, DateTime.to_iso8601(datetime)} - else - {:error, :missing_offset} -> cast("#{datetime}Z") - _e -> :error - end - end - - def cast(_), do: :error - - def dump(data) do - {:ok, data} - end - - def load(data) do - {:ok, data} - end -end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex b/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex deleted file mode 100644 index f71f76370..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/types/object_id.ex +++ /dev/null @@ -1,23 +0,0 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID do - use Ecto.Type - - def type, do: :string - - def cast(object) when is_binary(object) do - # Host has to be present and scheme has to be an http scheme (for now) - case URI.parse(object) do - %URI{host: nil} -> :error - %URI{host: ""} -> :error - %URI{scheme: scheme} when scheme in ["https", "http"] -> {:ok, object} - _ -> :error - end - end - - def cast(%{"id" => object}), do: cast(object) - - def cast(_), do: :error - - def dump(data), do: {:ok, data} - - def load(data), do: {:ok, data} -end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex b/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex deleted file mode 100644 index 408e0f6ee..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/types/recipients.ex +++ /dev/null @@ -1,36 +0,0 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Recipients do - use Ecto.Type - - alias Pleroma.Web.ActivityPub.ObjectValidators.Types.ObjectID - - def type, do: {:array, ObjectID} - - def cast(object) when is_binary(object) do - cast([object]) - end - - def cast(data) when is_list(data) do - data - |> Enum.reduce_while({:ok, []}, fn element, {:ok, list} -> - case ObjectID.cast(element) do - {:ok, id} -> - {:cont, {:ok, [id | list]}} - - _ -> - {:halt, :error} - end - end) - end - - def cast(_) do - :error - end - - def dump(data) do - {:ok, data} - end - - def load(data) do - {:ok, data} - end -end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex b/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex deleted file mode 100644 index 95c948123..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/types/safe_text.ex +++ /dev/null @@ -1,25 +0,0 @@ -# Pleroma: A lightweight social networking server -# Copyright © 2017-2020 Pleroma Authors -# SPDX-License-Identifier: AGPL-3.0-only - -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.SafeText do - use Ecto.Type - - alias Pleroma.HTML - - def type, do: :string - - def cast(str) when is_binary(str) do - {:ok, HTML.filter_tags(str)} - end - - def cast(_), do: :error - - def dump(data) do - {:ok, data} - end - - def load(data) do - {:ok, data} - end -end diff --git a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex b/lib/pleroma/web/activity_pub/object_validators/types/uri.ex deleted file mode 100644 index 24845bcc0..000000000 --- a/lib/pleroma/web/activity_pub/object_validators/types/uri.ex +++ /dev/null @@ -1,20 +0,0 @@ -defmodule Pleroma.Web.ActivityPub.ObjectValidators.Types.Uri do - use Ecto.Type - - def type, do: :string - - def cast(uri) when is_binary(uri) do - case URI.parse(uri) do - %URI{host: nil} -> :error - %URI{host: ""} -> :error - %URI{scheme: scheme} when scheme in ["https", "http"] -> {:ok, uri} - _ -> :error - end - end - - def cast(_), do: :error - - def dump(data), do: {:ok, data} - - def load(data), do: {:ok, data} -end diff --git a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex index d0ba418e8..e8d2d39c1 100644 --- a/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/undo_validator.ex @@ -6,7 +6,7 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do use Ecto.Schema alias Pleroma.Activity - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.EctoType.ActivityPub.ObjectValidators import Ecto.Changeset import Pleroma.Web.ActivityPub.ObjectValidators.CommonValidations @@ -14,10 +14,10 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidators.UndoValidator do @primary_key false embedded_schema do - field(:id, Types.ObjectID, primary_key: true) + field(:id, ObjectValidators.ObjectID, primary_key: true) field(:type, :string) - field(:object, Types.ObjectID) - field(:actor, Types.ObjectID) + field(:object, ObjectValidators.ObjectID) + field(:actor, ObjectValidators.ObjectID) field(:to, {:array, :string}, default: []) field(:cc, {:array, :string}, default: []) end diff --git a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex index 47e231150..f64fac46d 100644 --- a/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validators/url_object_validator.ex @@ -1,14 +1,18 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2020 Pleroma Authors +# SPDX-License-Identifier: AGPL-3.0-only + defmodule Pleroma.Web.ActivityPub.ObjectValidators.UrlObjectValidator do use Ecto.Schema - alias Pleroma.Web.ActivityPub.ObjectValidators.Types + alias Pleroma.EctoType.ActivityPub.ObjectValidators import Ecto.Changeset @primary_key false embedded_schema do field(:type, :string) - field(:href, Types.Uri) + field(:href, ObjectValidators.Uri) field(:mediaType, :string) end diff --git a/lib/pleroma/web/activity_pub/transmogrifier.ex b/lib/pleroma/web/activity_pub/transmogrifier.ex index 985921aa0..851f474b8 100644 --- a/lib/pleroma/web/activity_pub/transmogrifier.ex +++ b/lib/pleroma/web/activity_pub/transmogrifier.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do """ alias Pleroma.Activity alias Pleroma.EarmarkRenderer + alias Pleroma.EctoType.ActivityPub.ObjectValidators alias Pleroma.FollowingRelationship alias Pleroma.Maps alias Pleroma.Notification @@ -18,7 +19,6 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do alias Pleroma.Web.ActivityPub.ActivityPub alias Pleroma.Web.ActivityPub.Builder alias Pleroma.Web.ActivityPub.ObjectValidator - alias Pleroma.Web.ActivityPub.ObjectValidators.Types alias Pleroma.Web.ActivityPub.Pipeline alias Pleroma.Web.ActivityPub.Utils alias Pleroma.Web.ActivityPub.Visibility @@ -725,7 +725,7 @@ defmodule Pleroma.Web.ActivityPub.Transmogrifier do else {:error, {:validate_object, _}} = e -> # Check if we have a create activity for this - with {:ok, object_id} <- Types.ObjectID.cast(data["object"]), + with {:ok, object_id} <- ObjectValidators.ObjectID.cast(data["object"]), %Activity{data: %{"actor" => actor}} <- Activity.create_by_object_ap_id(object_id) |> Repo.one(), # We have one, insert a tombstone and retry -- cgit v1.2.3 From 5c0e1039ce41a2717598992a590658d4d079451c Mon Sep 17 00:00:00 2001 From: rinpatch Date: Tue, 16 Jun 2020 23:45:59 +0300 Subject: Chunk the notification type backfill migration Long-term we want that migration to be done entirely in SQL, but for now this is a hotfix to not cause OOMs on large databases. This is using a homegrown version of `Repo.stream`, it's worse in terms of performance than the upstream since it doesn't use the same prepared query for chunk queries, but unlike the upstream it supports preloads. --- .../migration_helper/notification_backfill.ex | 2 +- lib/pleroma/repo.ex | 28 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex index 09647d12a..b3770307a 100644 --- a/lib/pleroma/migration_helper/notification_backfill.ex +++ b/lib/pleroma/migration_helper/notification_backfill.ex @@ -18,7 +18,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do ) query - |> Repo.all() + |> Repo.chunk_stream(100) |> Enum.each(fn notification -> type = notification.activity diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index f62138466..6d85d70bc 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + import Ecto.Query require Logger defmodule Instrumenter do @@ -78,6 +79,33 @@ defmodule Pleroma.Repo do :ok end end + + def chunk_stream(query, chunk_size) do + # We don't actually need start and end funcitons of resource streaming, + # but it seems to be the only way to not fetch records one-by-one and + # have individual records be the elements of the stream, instead of + # lists of records + Stream.resource( + fn -> 0 end, + fn + last_id -> + query + |> order_by(asc: :id) + |> where([r], r.id > ^last_id) + |> limit(^chunk_size) + |> all() + |> case do + [] -> + {:halt, last_id} + + records -> + last_id = List.last(records).id + {records, last_id} + end + end, + fn _ -> :ok end + ) + end end defmodule Pleroma.Repo.UnappliedMigrationsError do -- cgit v1.2.3 From a77b0388f48084bd3a420855a232bf2e504f0bce Mon Sep 17 00:00:00 2001 From: Alexander Strizhakov Date: Wed, 17 Jun 2020 10:31:06 +0300 Subject: credo fix --- lib/pleroma/web/activity_pub/object_validator.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/activity_pub/object_validator.ex b/lib/pleroma/web/activity_pub/object_validator.ex index 3d699e8a5..6a83a2c33 100644 --- a/lib/pleroma/web/activity_pub/object_validator.ex +++ b/lib/pleroma/web/activity_pub/object_validator.ex @@ -9,8 +9,8 @@ defmodule Pleroma.Web.ActivityPub.ObjectValidator do the system. """ - alias Pleroma.Object alias Pleroma.EctoType.ActivityPub.ObjectValidators + alias Pleroma.Object alias Pleroma.User alias Pleroma.Web.ActivityPub.ObjectValidators.AnnounceValidator alias Pleroma.Web.ActivityPub.ObjectValidators.ChatMessageValidator -- cgit v1.2.3 From 74fd761637f737822d01aed945b6e0c75ced7008 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 12:56:30 +0000 Subject: Apply suggestion to lib/pleroma/web/media_proxy/invalidation.ex --- lib/pleroma/web/media_proxy/invalidation.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 83ff8589c..6da7eb720 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -32,6 +32,6 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do def prepare_urls(urls) do urls |> List.wrap() - |> Enum.map(&MediaProxy.url(&1)) + |> Enum.map(&MediaProxy.url/1) end end -- cgit v1.2.3 From 44ce97a9c9e90d5906386d1b51dea144cd258c32 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 13:12:32 +0000 Subject: Apply suggestion to lib/pleroma/web/media_proxy/invalidations/script.ex --- lib/pleroma/web/media_proxy/invalidations/script.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index 0217b119d..b0f44e8e2 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -10,7 +10,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts \\ %{}) do + def purge(urls, opts \\ []) do args = urls |> List.wrap() -- cgit v1.2.3 From 9a371bf5f6245b0f372bec7af15e2f4fc41d4ab7 Mon Sep 17 00:00:00 2001 From: Maksim Date: Wed, 17 Jun 2020 13:12:38 +0000 Subject: Apply suggestion to lib/pleroma/web/media_proxy/invalidations/script.ex --- lib/pleroma/web/media_proxy/invalidations/script.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/invalidations/script.ex b/lib/pleroma/web/media_proxy/invalidations/script.ex index b0f44e8e2..d32ffc50b 100644 --- a/lib/pleroma/web/media_proxy/invalidations/script.ex +++ b/lib/pleroma/web/media_proxy/invalidations/script.ex @@ -18,7 +18,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Script do |> Enum.join(" ") opts - |> Keyword.get(:script_path, nil) + |> Keyword.get(:script_path) |> do_purge([args]) |> handle_result(urls) end -- cgit v1.2.3 From d4b5a9730e8fe7adf5a2eca15bf40fafff85f30d Mon Sep 17 00:00:00 2001 From: Egor Kislitsyn Date: Wed, 17 Jun 2020 18:47:59 +0400 Subject: Remove `poll` from `notification_type` OpenAPI spec --- lib/pleroma/web/api_spec/operations/notification_operation.ex | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index c966b553a..41328b5f2 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -183,7 +183,6 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "favourite", "reblog", "mention", - "poll", "pleroma:emoji_reaction", "pleroma:chat_mention", "move", -- cgit v1.2.3 From 71a5d9bffb33d9424ea28900ea006678617e0096 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 12:54:02 -0500 Subject: Empty list as default --- lib/pleroma/web/media_proxy/invalidations/http.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/invalidations/http.ex b/lib/pleroma/web/media_proxy/invalidations/http.ex index 3694b56e8..bb81d8888 100644 --- a/lib/pleroma/web/media_proxy/invalidations/http.ex +++ b/lib/pleroma/web/media_proxy/invalidations/http.ex @@ -9,7 +9,7 @@ defmodule Pleroma.Web.MediaProxy.Invalidation.Http do require Logger @impl Pleroma.Web.MediaProxy.Invalidation - def purge(urls, opts) do + def purge(urls, opts \\ []) do method = Keyword.get(opts, :method, :purge) headers = Keyword.get(opts, :headers, []) options = Keyword.get(opts, :options, []) -- cgit v1.2.3 From c08c9db0c137d36896910194a6dc50a391a8fee2 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 13:02:01 -0500 Subject: Remove misleading is_ prefix from boolean function --- lib/pleroma/web/media_proxy/media_proxy.ex | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 59ca217ab..3dccd6b7f 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -37,15 +37,15 @@ defmodule Pleroma.Web.MediaProxy do def url("/" <> _ = url), do: url def url(url) do - if disabled?() or not is_url_proxiable?(url) do + if disabled?() or not url_proxiable?(url) do url else encode_url(url) end end - @spec is_url_proxiable?(String.t()) :: boolean() - def is_url_proxiable?(url) do + @spec url_proxiable?(String.t()) :: boolean() + def url_proxiable?(url) do if local?(url) or whitelisted?(url) do false else -- cgit v1.2.3 From 2731ea1334c2c91315465659a0874829cb9e1e11 Mon Sep 17 00:00:00 2001 From: Mark Felder Date: Wed, 17 Jun 2020 13:13:55 -0500 Subject: Change references from "deleted_urls" to "banned_urls" as nothing is handled via media deletions anymore; all actions are manual operations by an admin to ban the url --- lib/pleroma/application.ex | 2 +- lib/pleroma/plugs/uploaded_media.ex | 10 +++++----- .../controllers/media_proxy_cache_controller.ex | 6 +++--- lib/pleroma/web/media_proxy/media_proxy.ex | 20 ++++++++++---------- .../web/media_proxy/media_proxy_controller.ex | 4 ++-- 5 files changed, 21 insertions(+), 21 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index adebebc7a..4a21bf138 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -149,7 +149,7 @@ defmodule Pleroma.Application do build_cachex("web_resp", limit: 2500), build_cachex("emoji_packs", expiration: emoji_packs_expiration(), limit: 10), build_cachex("failed_proxy_url", limit: 2500), - build_cachex("deleted_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) + build_cachex("banned_urls", default_ttl: :timer.hours(24 * 30), limit: 5_000) ] end diff --git a/lib/pleroma/plugs/uploaded_media.ex b/lib/pleroma/plugs/uploaded_media.ex index 2f3fde002..40984cfc0 100644 --- a/lib/pleroma/plugs/uploaded_media.ex +++ b/lib/pleroma/plugs/uploaded_media.ex @@ -49,7 +49,7 @@ defmodule Pleroma.Plugs.UploadedMedia do with uploader <- Keyword.fetch!(config, :uploader), proxy_remote = Keyword.get(config, :proxy_remote, false), {:ok, get_method} <- uploader.get_file(file), - false <- media_is_deleted(conn, get_method) do + false <- media_is_banned(conn, get_method) do get_media(conn, get_method, proxy_remote, opts) else _ -> @@ -61,13 +61,13 @@ defmodule Pleroma.Plugs.UploadedMedia do def call(conn, _opts), do: conn - defp media_is_deleted(%{request_path: path} = _conn, {:static_dir, _}) do - MediaProxy.in_deleted_urls(Pleroma.Web.base_url() <> path) + defp media_is_banned(%{request_path: path} = _conn, {:static_dir, _}) do + MediaProxy.in_banned_urls(Pleroma.Web.base_url() <> path) end - defp media_is_deleted(_, {:url, url}), do: MediaProxy.in_deleted_urls(url) + defp media_is_banned(_, {:url, url}), do: MediaProxy.in_banned_urls(url) - defp media_is_deleted(_, _), do: false + defp media_is_banned(_, _), do: false defp get_media(conn, {:static_dir, directory}, _, opts) do static_opts = diff --git a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex index e3fa0ac28..e2759d59f 100644 --- a/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex +++ b/lib/pleroma/web/admin_api/controllers/media_proxy_cache_controller.ex @@ -27,7 +27,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do def index(%{assigns: %{user: _}} = conn, params) do cursor = - :deleted_urls_cache + :banned_urls_cache |> :ets.table([{:traverse, {:select, Cachex.Query.create(true, :key)}}]) |> :qlc.cursor() @@ -47,7 +47,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do end def delete(%{assigns: %{user: _}, body_params: %{urls: urls}} = conn, _) do - MediaProxy.remove_from_deleted_urls(urls) + MediaProxy.remove_from_banned_urls(urls) render(conn, "index.json", urls: urls) end @@ -55,7 +55,7 @@ defmodule Pleroma.Web.AdminAPI.MediaProxyCacheController do MediaProxy.Invalidation.purge(urls) if ban do - MediaProxy.put_in_deleted_urls(urls) + MediaProxy.put_in_banned_urls(urls) end render(conn, "index.json", urls: urls) diff --git a/lib/pleroma/web/media_proxy/media_proxy.ex b/lib/pleroma/web/media_proxy/media_proxy.ex index 3dccd6b7f..077fabe47 100644 --- a/lib/pleroma/web/media_proxy/media_proxy.ex +++ b/lib/pleroma/web/media_proxy/media_proxy.ex @@ -10,27 +10,27 @@ defmodule Pleroma.Web.MediaProxy do @base64_opts [padding: false] - @spec in_deleted_urls(String.t()) :: boolean() - def in_deleted_urls(url), do: elem(Cachex.exists?(:deleted_urls_cache, url(url)), 1) + @spec in_banned_urls(String.t()) :: boolean() + def in_banned_urls(url), do: elem(Cachex.exists?(:banned_urls_cache, url(url)), 1) - def remove_from_deleted_urls(urls) when is_list(urls) do - Cachex.execute!(:deleted_urls_cache, fn cache -> + def remove_from_banned_urls(urls) when is_list(urls) do + Cachex.execute!(:banned_urls_cache, fn cache -> Enum.each(Invalidation.prepare_urls(urls), &Cachex.del(cache, &1)) end) end - def remove_from_deleted_urls(url) when is_binary(url) do - Cachex.del(:deleted_urls_cache, url(url)) + def remove_from_banned_urls(url) when is_binary(url) do + Cachex.del(:banned_urls_cache, url(url)) end - def put_in_deleted_urls(urls) when is_list(urls) do - Cachex.execute!(:deleted_urls_cache, fn cache -> + def put_in_banned_urls(urls) when is_list(urls) do + Cachex.execute!(:banned_urls_cache, fn cache -> Enum.each(Invalidation.prepare_urls(urls), &Cachex.put(cache, &1, true)) end) end - def put_in_deleted_urls(url) when is_binary(url) do - Cachex.put(:deleted_urls_cache, url(url), true) + def put_in_banned_urls(url) when is_binary(url) do + Cachex.put(:banned_urls_cache, url(url), true) end def url(url) when is_nil(url) or url == "", do: nil diff --git a/lib/pleroma/web/media_proxy/media_proxy_controller.ex b/lib/pleroma/web/media_proxy/media_proxy_controller.ex index ff0158d83..9a64b0ef3 100644 --- a/lib/pleroma/web/media_proxy/media_proxy_controller.ex +++ b/lib/pleroma/web/media_proxy/media_proxy_controller.ex @@ -14,11 +14,11 @@ defmodule Pleroma.Web.MediaProxy.MediaProxyController do with config <- Pleroma.Config.get([:media_proxy], []), true <- Keyword.get(config, :enabled, false), {:ok, url} <- MediaProxy.decode_url(sig64, url64), - {_, false} <- {:in_deleted_urls, MediaProxy.in_deleted_urls(url)}, + {_, false} <- {:in_banned_urls, MediaProxy.in_banned_urls(url)}, :ok <- filename_matches(params, conn.request_path, url) do ReverseProxy.call(conn, url, Keyword.get(config, :proxy_opts, @default_proxy_opts)) else - error when error in [false, {:in_deleted_urls, true}] -> + error when error in [false, {:in_banned_urls, true}] -> send_resp(conn, 404, Plug.Conn.Status.reason_phrase(404)) {:error, :invalid_signature} -> -- cgit v1.2.3 From 4044f24e2e4935757e038e7f06373ed1c9172560 Mon Sep 17 00:00:00 2001 From: Maksim Pechnikov Date: Thu, 18 Jun 2020 05:02:33 +0300 Subject: fix test --- lib/pleroma/web/media_proxy/invalidation.ex | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/pleroma/web/media_proxy/invalidation.ex b/lib/pleroma/web/media_proxy/invalidation.ex index 6da7eb720..5808861e6 100644 --- a/lib/pleroma/web/media_proxy/invalidation.ex +++ b/lib/pleroma/web/media_proxy/invalidation.ex @@ -26,7 +26,8 @@ defmodule Pleroma.Web.MediaProxy.Invalidation do defp do_purge(urls) do provider = Config.get([:media_proxy, :invalidation, :provider]) - provider.purge(urls, Config.get(provider)) + options = Config.get(provider) + provider.purge(urls, options) end def prepare_urls(urls) do -- cgit v1.2.3 From 5237a2df9f123f661de30a53193b7d9fec69ecae Mon Sep 17 00:00:00 2001 From: Ivan Tashkinov Date: Fri, 19 Jun 2020 16:14:06 +0300 Subject: [#1873] Fixes missing :offset pagination param support. Added pagination support for hashtags search. --- lib/pleroma/pagination.ex | 6 +++++ lib/pleroma/web/api_spec/helpers.ex | 6 +++++ .../mastodon_api/controllers/search_controller.ex | 31 +++++++++++++--------- 3 files changed, 31 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/lib/pleroma/pagination.ex b/lib/pleroma/pagination.ex index 1b99e44f9..9a3795769 100644 --- a/lib/pleroma/pagination.ex +++ b/lib/pleroma/pagination.ex @@ -64,6 +64,12 @@ defmodule Pleroma.Pagination do @spec paginate(Ecto.Query.t(), map(), type(), atom() | nil) :: [Ecto.Schema.t()] def paginate(query, options, method \\ :keyset, table_binding \\ nil) + def paginate(list, options, _method, _table_binding) when is_list(list) do + offset = options[:offset] || 0 + limit = options[:limit] || 0 + Enum.slice(list, offset, limit) + end + def paginate(query, options, :keyset, table_binding) do query |> restrict(:min_id, options, table_binding) diff --git a/lib/pleroma/web/api_spec/helpers.ex b/lib/pleroma/web/api_spec/helpers.ex index a9cfe0fed..a258e8421 100644 --- a/lib/pleroma/web/api_spec/helpers.ex +++ b/lib/pleroma/web/api_spec/helpers.ex @@ -39,6 +39,12 @@ defmodule Pleroma.Web.ApiSpec.Helpers do :string, "Return the newest items newer than this ID" ), + Operation.parameter( + :offset, + :query, + %Schema{type: :integer, default: 0}, + "Return items past this number of items" + ), Operation.parameter( :limit, :query, diff --git a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex index 3be0ca095..e50980122 100644 --- a/lib/pleroma/web/mastodon_api/controllers/search_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/search_controller.ex @@ -107,21 +107,21 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do ) end - defp resource_search(:v2, "hashtags", query, _options) do + defp resource_search(:v2, "hashtags", query, options) do tags_path = Web.base_url() <> "/tag/" query - |> prepare_tags() + |> prepare_tags(options) |> Enum.map(fn tag -> %{name: tag, url: tags_path <> tag} end) end - defp resource_search(:v1, "hashtags", query, _options) do - prepare_tags(query) + defp resource_search(:v1, "hashtags", query, options) do + prepare_tags(query, options) end - defp prepare_tags(query, add_joined_tag \\ true) do + defp prepare_tags(query, options) do tags = query |> preprocess_uri_query() @@ -139,13 +139,20 @@ defmodule Pleroma.Web.MastodonAPI.SearchController do tags = Enum.map(tags, fn tag -> String.trim_leading(tag, "#") end) - if Enum.empty?(explicit_tags) && add_joined_tag do - tags - |> Kernel.++([joined_tag(tags)]) - |> Enum.uniq_by(&String.downcase/1) - else - tags - end + tags = + if Enum.empty?(explicit_tags) && !options[:skip_joined_tag] do + add_joined_tag(tags) + else + tags + end + + Pleroma.Pagination.paginate(tags, options) + end + + defp add_joined_tag(tags) do + tags + |> Kernel.++([joined_tag(tags)]) + |> Enum.uniq_by(&String.downcase/1) end # If `query` is a URI, returns last component of its path, otherwise returns `query` -- cgit v1.2.3