diff options
33 files changed, 558 insertions, 39 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f9745122a..8b5131dc3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -45,7 +45,8 @@ docs-build: unit-testing: stage: test services: - - name: postgres:9.6.2 + - name: lainsoykaf/postgres-with-rum + alias: postgres command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] script: - mix deps.get @@ -54,6 +55,21 @@ unit-testing: - mix test --trace --preload-modules - mix coveralls +unit-testing-rum: + stage: test + services: + - name: lainsoykaf/postgres-with-rum + alias: postgres + command: ["postgres", "-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] + variables: + RUM_ENABLED: "true" + script: + - mix deps.get + - mix ecto.create + - mix ecto.migrate + - "mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/" + - mix test --trace --preload-modules + lint: stage: test script: @@ -65,7 +81,6 @@ analysis: - mix deps.get - mix credo --strict --only=warnings,todo,fixme,consistency,readability - docs-deploy: stage: deploy image: alpine:3.9 diff --git a/CHANGELOG.md b/CHANGELOG.md index b50ca895c..3ff70e6e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Configuration: `report_uri` option - Pleroma API: User subscriptions - Pleroma API: Healthcheck endpoint +- Pleroma API: `/api/v1/pleroma/mascot` per-user frontend mascot configuration endpoints - Admin API: Endpoints for listing/revoking invite tokens - Admin API: Endpoints for making users follow/unfollow each other - Admin API: added filters (role, tags, email, name) for users endpoint @@ -72,6 +73,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Deps: Updated Ecto to 3.0.7 - Don't ship finmoji by default, they can be installed as an emoji pack - Hide deactivated users and their statuses +- Posts which are marked sensitive or tagged nsfw no longer have link previews. +- HTTP connection timeout is now set to 10 seconds. +- Respond with a 404 Not implemented JSON error message when requested API is not implemented ### Fixed - Added an FTS index on objects. Running `vacuum analyze` and setting a larger `work_mem` is recommended. @@ -103,6 +107,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - Mastodon API: Correct `reblogged`, `favourited`, and `bookmarked` values in the reblog status JSON - Mastodon API: Exposing default scope of the user to anyone - Mastodon API: Make `irreversible` field default to `false` [`POST /api/v1/filters`] +- User-Agent is now sent correctly for all HTTP requests. ## Removed - Configuration: `config :pleroma, :fe` in favor of the more flexible `config :pleroma, :frontend_configurations` diff --git a/config/config.exs b/config/config.exs index 387c1a5a7..c3301b2ed 100644 --- a/config/config.exs +++ b/config/config.exs @@ -192,6 +192,7 @@ config :tesla, adapter: Tesla.Adapter.Hackney # Configures http settings, upstream proxy etc. config :pleroma, :http, proxy_url: nil, + send_user_agent: true, adapter: [ ssl_options: [ # We don't support TLS v1.3 yet @@ -275,6 +276,19 @@ config :pleroma, :frontend_configurations, showInstanceSpecificPanel: true } +config :pleroma, :assets, + mascots: [ + pleroma_fox_tan: %{ + url: "/images/pleroma-fox-tan-smol.png", + mime_type: "image/png" + }, + pleroma_fox_tan_shy: %{ + url: "/images/pleroma-fox-tan-shy.png", + mime_type: "image/png" + } + ], + default_mascot: :pleroma_fox_tan + config :pleroma, :activitypub, accept_blocks: true, unfollow_blocked: true, @@ -297,6 +311,7 @@ config :pleroma, :mrf_simple, media_removal: [], media_nsfw: [], federated_timeline_removal: [], + report_removal: [], reject: [], accept: [] @@ -467,6 +482,8 @@ config :pleroma, :oauth2, token_expires_in: 600, issue_new_refresh_token: true +config :pleroma, :database, rum_enabled: false + config :http_signatures, adapter: Pleroma.Signature diff --git a/config/test.exs b/config/test.exs index 40db66170..6100989c4 100644 --- a/config/test.exs +++ b/config/test.exs @@ -63,6 +63,12 @@ config :pleroma, :app_account_creation, max_requests: 5 config :pleroma, :http_security, report_uri: "https://endpoint.com" +config :pleroma, :http, send_user_agent: false + +rum_enabled = System.get_env("RUM_ENABLED") == "true" +config :pleroma, :database, rum_enabled: rum_enabled +IO.puts("RUM enabled: #{rum_enabled}") + try do import_config "test.secret.exs" rescue diff --git a/docs/api/pleroma_api.md b/docs/api/pleroma_api.md index dd0b6ca73..4d99a2d2b 100644 --- a/docs/api/pleroma_api.md +++ b/docs/api/pleroma_api.md @@ -252,6 +252,45 @@ See [Admin-API](Admin-API.md) ] ``` +## `/api/v1/pleroma/mascot` +### Gets user mascot image +* Method `GET` +* Authentication: required + +* Response: JSON. Returns a mastodon media attachment entity. +* Example response: +```json +{ + "id": "abcdefg", + "url": "https://pleroma.example.org/media/abcdefg.png", + "type": "image", + "pleroma": { + "mime_type": "image/png" + } +} +``` + +### Updates user mascot image +* Method `PUT` +* Authentication: required +* Params: + * `image`: Multipart image +* Response: JSON. Returns a mastodon media attachment entity + when successful, otherwise returns HTTP 415 `{"error": "error_msg"}` +* Example response: +```json +{ + "id": "abcdefg", + "url": "https://pleroma.example.org/media/abcdefg.png", + "type": "image", + "pleroma": { + "mime_type": "image/png" + } +} +``` +* Note: Behaves exactly the same as `POST /api/v1/upload`. + Can only accept images - any attempt to upload non-image files will be met with `HTTP 415 Unsupported Media Type`. + ## `/api/pleroma/notification_settings` ### Updates user notification settings * Method `PUT` diff --git a/docs/config.md b/docs/config.md index c2af5c012..197326bbd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -203,6 +203,16 @@ This section is used to configure Pleroma-FE, unless ``:managed_config`` in ``:i * `hide_post_stats`: Hide notices statistics(repeats, favorites, …) * `hide_user_stats`: Hide profile statistics(posts, posts per day, followers, followings, …) +## :assets + +This section configures assets to be used with various frontends. Currently the only option +relates to mascots on the mastodon frontend + +* `mascots`: KeywordList of mascots, each element __MUST__ contain both a `url` and a + `mime_type` key. +* `default_mascot`: An element from `mascots` - This will be used as the default mascot + on MastoFE (default: `:pleroma_fox_tan`) + ## :mrf_simple * `media_removal`: List of instances to remove medias from * `media_nsfw`: List of instances to put medias as NSFW(sensitive) from @@ -544,3 +554,18 @@ Configure OAuth 2 provider capabilities: * `shortcode_globs`: Location of custom emoji files. `*` can be used as a wildcard. Example `["/emoji/custom/**/*.png"]` * `groups`: Emojis are ordered in groups (tags). This is an array of key-value pairs where the key is the groupname and the value the location or array of locations. `*` can be used as a wildcard. Example `[Custom: ["/emoji/*.png", "/emoji/custom/*.png"]]` * `default_manifest`: Location of the JSON-manifest. This manifest contains information about the emoji-packs you can download. Currently only one manifest can be added (no arrays). + +## Database options + +### RUM indexing for full text search +* `rum_enabled`: If RUM indexes should be used. Defaults to `false`. + +RUM indexes are an alternative indexing scheme that is not included in PostgreSQL by default. While they may eventually be mainlined, for now they have to be installed as a PostgreSQL extension from https://github.com/postgrespro/rum. + +Their advantage over the standard GIN indexes is that they allow efficient ordering of search results by timestamp, which makes search queries a lot faster on larger servers, by one or two orders of magnitude. They take up around 3 times as much space as GIN indexes. + +To enable them, both the `rum_enabled` flag has to be set and the following special migration has to be run: + +`mix ecto.migrate --migrations-path priv/repo/optional_migrations/rum_indexing/` + +This will probably take a long time. diff --git a/docs/config/mrf.md b/docs/config/mrf.md index 2cc16cef0..45be18fc5 100644 --- a/docs/config/mrf.md +++ b/docs/config/mrf.md @@ -5,11 +5,12 @@ Possible uses include: * marking incoming messages with media from a given account or instance as sensitive * rejecting messages from a specific instance +* rejecting reports (flags) from a specific instance * removing/unlisting messages from the public timelines * removing media from messages * sending only public messages to a specific instance -The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module. +The MRF provides user-configurable policies. The default policy is `NoOpPolicy`, which disables the MRF functionality. Pleroma also includes an easy to use policy called `SimplePolicy` which maps messages matching certain pre-defined criterion to actions built into the policy module. It is possible to use multiple, active MRF policies at the same time. ## Quarantine Instances @@ -41,12 +42,13 @@ Once `SimplePolicy` is enabled, you can configure various groups in the `:mrf_si * `media_nsfw`: Servers in this group will have the #nsfw tag and sensitive setting injected into incoming messages which contain media. * `reject`: Servers in this group will have their messages rejected. * `federated_timeline_removal`: Servers in this group will have their messages unlisted from the public timelines by flipping the `to` and `cc` fields. +* `report_removal`: Servers in this group will have their reports (flags) rejected. Servers should be configured as lists. ### Example -This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com` and remove messages from `spam.university` from the federated timeline: +This example will enable `SimplePolicy`, block media from `illegalporn.biz`, mark media as NSFW from `porn.biz` and `porn.business`, reject messages from `spam.com`, remove messages from `spam.university` from the federated timeline and block reports (flags) from `whiny.whiner`: ``` config :pleroma, :instance, @@ -56,7 +58,8 @@ config :pleroma, :mrf_simple, media_removal: ["illegalporn.biz"], media_nsfw: ["porn.biz", "porn.business"], reject: ["spam.com"], - federated_timeline_removal: ["spam.university"] + federated_timeline_removal: ["spam.university"], + report_removal: ["whiny.whiner"] ``` diff --git a/lib/pleroma/activity.ex b/lib/pleroma/activity.ex index 4e54b15ba..99589590c 100644 --- a/lib/pleroma/activity.ex +++ b/lib/pleroma/activity.ex @@ -10,6 +10,7 @@ defmodule Pleroma.Activity do alias Pleroma.Notification alias Pleroma.Object alias Pleroma.Repo + alias Pleroma.ThreadMute alias Pleroma.User import Ecto.Changeset @@ -37,6 +38,7 @@ defmodule Pleroma.Activity do field(:local, :boolean, default: true) field(:actor, :string) field(:recipients, {:array, :string}, default: []) + field(:thread_muted?, :boolean, virtual: true) # This is a fake relation, do not use outside of with_preloaded_bookmark/get_bookmark has_one(:bookmark, Bookmark) has_many(:notifications, Notification, on_delete: :delete_all) @@ -90,6 +92,16 @@ defmodule Pleroma.Activity do def with_preloaded_bookmark(query, _), do: query + def with_set_thread_muted_field(query, %User{} = user) do + from([a] in query, + left_join: tm in ThreadMute, + on: tm.user_id == ^user.id and tm.context == fragment("?->>'context'", a.data), + select: %Activity{a | thread_muted?: not is_nil(tm.id)} + ) + end + + def with_set_thread_muted_field(query, _), do: query + def get_by_ap_id(ap_id) do Repo.one( from( diff --git a/lib/pleroma/application.ex b/lib/pleroma/application.ex index eeb415084..dab45a0b2 100644 --- a/lib/pleroma/application.ex +++ b/lib/pleroma/application.ex @@ -131,19 +131,22 @@ defmodule Pleroma.Application do defp setup_instrumenters do require Prometheus.Registry - :ok = - :telemetry.attach( - "prometheus-ecto", - [:pleroma, :repo, :query], - &Pleroma.Repo.Instrumenter.handle_event/4, - %{} - ) + if Application.get_env(:prometheus, Pleroma.Repo.Instrumenter) do + :ok = + :telemetry.attach( + "prometheus-ecto", + [:pleroma, :repo, :query], + &Pleroma.Repo.Instrumenter.handle_event/4, + %{} + ) + + Pleroma.Repo.Instrumenter.setup() + end Prometheus.Registry.register_collector(:prometheus_process_collector) Pleroma.Web.Endpoint.MetricsExporter.setup() Pleroma.Web.Endpoint.PipelineInstrumenter.setup() Pleroma.Web.Endpoint.Instrumenter.setup() - Pleroma.Repo.Instrumenter.setup() end def enabled_hackney_pools do diff --git a/lib/pleroma/formatter.ex b/lib/pleroma/formatter.ex index 3d7c36d21..3e3b9fe97 100644 --- a/lib/pleroma/formatter.ex +++ b/lib/pleroma/formatter.ex @@ -8,7 +8,7 @@ defmodule Pleroma.Formatter do alias Pleroma.User alias Pleroma.Web.MediaProxy - @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/ + @safe_mention_regex ~r/^(\s*(?<mentions>@.+?\s+)+)(?<rest>.*)/s @link_regex ~r"((?:http(s)?:\/\/)?[\w.-]+(?:\.[\w\.-]+)+[\w\-\._~%:/?#[\]@!\$&'\(\)\*\+,;=.]+)|[0-9a-z+\-\.]+:[0-9a-z$-_.+!*'(),]+"ui @markdown_characters_regex ~r/(`|\*|_|{|}|[|]|\(|\)|#|\+|-|\.|!)/ diff --git a/lib/pleroma/http/connection.ex b/lib/pleroma/http/connection.ex index c0173465a..558005c19 100644 --- a/lib/pleroma/http/connection.ex +++ b/lib/pleroma/http/connection.ex @@ -8,7 +8,7 @@ defmodule Pleroma.HTTP.Connection do """ @hackney_options [ - connect_timeout: 2_000, + connect_timeout: 10_000, recv_timeout: 20_000, follow_redirect: true, pool: :federation diff --git a/lib/pleroma/http/request_builder.ex b/lib/pleroma/http/request_builder.ex index 5f2cff2c0..e23457999 100644 --- a/lib/pleroma/http/request_builder.ex +++ b/lib/pleroma/http/request_builder.ex @@ -45,8 +45,15 @@ defmodule Pleroma.HTTP.RequestBuilder do Add headers to the request """ @spec headers(map(), list(tuple)) :: map() - def headers(request, h) do - Map.put_new(request, :headers, h) + def headers(request, header_list) do + header_list = + if Pleroma.Config.get([:http, :send_user_agent]) do + header_list ++ [{"User-Agent", Pleroma.Application.user_agent()}] + else + header_list + end + + Map.put_new(request, :headers, header_list) end @doc """ diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 28da310ee..05fe58f7c 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -1402,4 +1402,24 @@ defmodule Pleroma.User do |> put_embed(:info, info_changeset) |> update_and_set_cache() end + + def get_mascot(%{info: %{mascot: %{} = mascot}}) when not is_nil(mascot) do + mascot + end + + def get_mascot(%{info: %{mascot: mascot}}) when is_nil(mascot) do + # use instance-default + config = Pleroma.Config.get([:assets, :mascots]) + default_mascot = Pleroma.Config.get([:assets, :default_mascot]) + mascot = Keyword.get(config, default_mascot) + + %{ + "id" => "default-mascot", + "url" => mascot[:url], + "preview_url" => mascot[:url], + "pleroma" => %{ + "mime_type" => mascot[:mime_type] + } + } + end end diff --git a/lib/pleroma/user/info.ex b/lib/pleroma/user/info.ex index 5f0cefc00..6397e2737 100644 --- a/lib/pleroma/user/info.ex +++ b/lib/pleroma/user/info.ex @@ -43,6 +43,7 @@ defmodule Pleroma.User.Info do field(:hide_favorites, :boolean, default: true) field(:pinned_activities, {:array, :string}, default: []) field(:flavour, :string, default: nil) + field(:mascot, :map, default: nil) field(:emoji, {:array, :map}, default: []) field(:notification_settings, :map, @@ -248,6 +249,14 @@ defmodule Pleroma.User.Info do |> validate_required([:flavour]) end + def mascot_update(info, url) do + params = %{mascot: url} + + info + |> cast(params, [:mascot]) + |> validate_required([:mascot]) + end + def set_source_data(info, source_data) do params = %{source_data: source_data} diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index 5c3156978..3d9679ec0 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -834,6 +834,13 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Activity.with_preloaded_bookmark(opts["user"]) end + defp maybe_set_thread_muted_field(query, %{"skip_preload" => true}), do: query + + defp maybe_set_thread_muted_field(query, opts) do + query + |> Activity.with_set_thread_muted_field(opts["user"]) + end + defp maybe_order(query, %{order: :desc}) do query |> order_by(desc: :id) @@ -852,6 +859,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do base_query |> maybe_preload_objects(opts) |> maybe_preload_bookmarks(opts) + |> maybe_set_thread_muted_field(opts) |> maybe_order(opts) |> restrict_recipients(recipients, opts["user"]) |> restrict_tag(opts) diff --git a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex index 50426e920..7190652d2 100644 --- a/lib/pleroma/web/activity_pub/mrf/simple_policy.ex +++ b/lib/pleroma/web/activity_pub/mrf/simple_policy.ex @@ -48,10 +48,9 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do %{host: actor_host} = _actor_info, %{ "type" => "Create", - "object" => %{"attachment" => child_attachment} = child_object + "object" => child_object } = object - ) - when length(child_attachment) > 0 do + ) do object = if Enum.member?(Pleroma.Config.get([:mrf_simple, :media_nsfw]), actor_host) do tags = (child_object["tag"] || []) ++ ["nsfw"] @@ -95,6 +94,16 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} end + defp check_report_removal(%{host: actor_host} = _actor_info, %{"type" => "Flag"} = object) do + if actor_host in Pleroma.Config.get([:mrf_simple, :report_removal]) do + {:reject, nil} + else + {:ok, object} + end + end + + defp check_report_removal(_actor_info, object), do: {:ok, object} + @impl true def filter(object) do actor_info = URI.parse(object["actor"]) @@ -103,7 +112,8 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicy do {:ok, object} <- check_reject(actor_info, object), {:ok, object} <- check_media_removal(actor_info, object), {:ok, object} <- check_media_nsfw(actor_info, object), - {:ok, object} <- check_ftl_removal(actor_info, object) do + {:ok, object} <- check_ftl_removal(actor_info, object), + {:ok, object} <- check_report_removal(actor_info, object) do {:ok, object} else _e -> {:reject, nil} diff --git a/lib/pleroma/web/common_api/common_api.ex b/lib/pleroma/web/common_api/common_api.ex index a599ffee5..5a312d673 100644 --- a/lib/pleroma/web/common_api/common_api.ex +++ b/lib/pleroma/web/common_api/common_api.ex @@ -157,6 +157,7 @@ defmodule Pleroma.Web.CommonAPI do {to, cc} <- to_for_user_and_mentions(user, mentions, in_reply_to, visibility), context <- make_context(in_reply_to), cw <- data["spoiler_text"] || "", + sensitive <- data["sensitive"] || Enum.member?(tags, {"#nsfw", "nsfw"}), full_payload <- String.trim(status <> cw), length when length in 1..limit <- String.length(full_payload), object <- @@ -169,7 +170,8 @@ defmodule Pleroma.Web.CommonAPI do in_reply_to, tags, cw, - cc + cc, + sensitive ), object <- Map.put( diff --git a/lib/pleroma/web/common_api/utils.ex b/lib/pleroma/web/common_api/utils.ex index bee2fd159..d93c0d46e 100644 --- a/lib/pleroma/web/common_api/utils.ex +++ b/lib/pleroma/web/common_api/utils.ex @@ -223,7 +223,8 @@ defmodule Pleroma.Web.CommonAPI.Utils do in_reply_to, tags, cw \\ nil, - cc \\ [] + cc \\ [], + sensitive \\ false ) do object = %{ "type" => "Note", @@ -231,6 +232,7 @@ defmodule Pleroma.Web.CommonAPI.Utils do "cc" => cc, "content" => content_html, "summary" => cw, + "sensitive" => !Enum.member?(["false", "False", "0", false], sensitive), "context" => context, "attachment" => attachments, "actor" => actor, diff --git a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex index 1b776fbca..1ec0f30a1 100644 --- a/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex +++ b/lib/pleroma/web/mastodon_api/mastodon_api_controller.ex @@ -707,6 +707,41 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + def set_mascot(%{assigns: %{user: user}} = conn, %{"file" => file}) do + with {:ok, object} <- ActivityPub.upload(file, actor: User.ap_id(user)), + %{} = attachment_data <- Map.put(object.data, "id", object.id), + %{type: type} = rendered <- + StatusView.render("attachment.json", %{attachment: attachment_data}) do + # Reject if not an image + if type == "image" do + # Sure! + # Save to the user's info + info_changeset = User.Info.mascot_update(user.info, rendered) + + user_changeset = + user + |> Ecto.Changeset.change() + |> Ecto.Changeset.put_embed(:info, info_changeset) + + {:ok, _user} = User.update_and_set_cache(user_changeset) + + conn + |> json(rendered) + else + conn + |> put_resp_content_type("application/json") + |> send_resp(415, Jason.encode!(%{"error" => "mascots can only be images"})) + end + end + end + + def get_mascot(%{assigns: %{user: user}} = conn, _params) do + mascot = User.get_mascot(user) + + conn + |> json(mascot) + end + def favourited_by(%{assigns: %{user: user}} = conn, %{"id" => id}) do with %Activity{data: %{"object" => object}} <- Repo.get(Activity, id), %Object{data: %{"likes" => likes}} <- Object.normalize(object) do @@ -1009,6 +1044,30 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end end + def status_search_query_with_gin(q, query) do + from([a, o] in q, + where: + fragment( + "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", + o.data, + ^query + ), + order_by: [desc: :id] + ) + end + + def status_search_query_with_rum(q, query) do + from([a, o] in q, + where: + fragment( + "? @@ plainto_tsquery('english', ?)", + o.fts_content, + ^query + ), + order_by: [fragment("? <=> now()::date", o.inserted_at)] + ) + end + def status_search(user, query) do fetched = if Regex.match?(~r/https?:/, query) do @@ -1022,20 +1081,19 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do end || [] q = - from( - [a, o] in Activity.with_preloaded_object(Activity), + from([a, o] in Activity.with_preloaded_object(Activity), where: fragment("?->>'type' = 'Create'", a.data), where: "https://www.w3.org/ns/activitystreams#Public" in a.recipients, - where: - fragment( - "to_tsvector('english', ?->>'content') @@ plainto_tsquery('english', ?)", - o.data, - ^query - ), - limit: 20, - order_by: [desc: :id] + limit: 20 ) + q = + if Pleroma.Config.get([:database, :rum_enabled]) do + status_search_query_with_rum(q, query) + else + status_search_query_with_gin(q, query) + end + Repo.all(q) ++ fetched end @@ -1306,7 +1364,7 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController do display_sensitive_media: false, reduce_motion: false, max_toot_chars: limit, - mascot: "/images/pleroma-fox-tan-smol.png" + mascot: User.get_mascot(user)["url"] }, rights: %{ delete_others_notice: present?(user.info.is_moderator), diff --git a/lib/pleroma/web/mastodon_api/views/status_view.ex b/lib/pleroma/web/mastodon_api/views/status_view.ex index c93d915e5..e55f9b96e 100644 --- a/lib/pleroma/web/mastodon_api/views/status_view.ex +++ b/lib/pleroma/web/mastodon_api/views/status_view.ex @@ -157,6 +157,12 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do bookmarked = Activity.get_bookmark(activity, opts[:for]) != nil + thread_muted? = + case activity.thread_muted? do + thread_muted? when is_boolean(thread_muted?) -> thread_muted? + nil -> CommonAPI.thread_muted?(user, activity) + end + attachment_data = object.data["attachment"] || [] attachments = render_many(attachment_data, StatusView, "attachment.json", as: :attachment) @@ -228,7 +234,7 @@ defmodule Pleroma.Web.MastodonAPI.StatusView do reblogged: reblogged?(activity, opts[:for]), favourited: present?(favorited), bookmarked: present?(bookmarked), - muted: CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user), + muted: thread_muted? || User.mutes?(opts[:for], user), pinned: pinned?(activity, user), sensitive: sensitive, spoiler_text: summary_html, diff --git a/lib/pleroma/web/rich_media/helpers.ex b/lib/pleroma/web/rich_media/helpers.ex index 0162a5be9..9bc8f2559 100644 --- a/lib/pleroma/web/rich_media/helpers.ex +++ b/lib/pleroma/web/rich_media/helpers.ex @@ -24,6 +24,7 @@ defmodule Pleroma.Web.RichMedia.Helpers do def fetch_data_for_activity(%Activity{data: %{"type" => "Create"}} = activity) do with true <- Pleroma.Config.get([:rich_media, :enabled]), %Object{} = object <- Object.normalize(activity), + false <- object.data["sensitive"] || false, {:ok, page_url} <- HTML.extract_first_external_url(object, object.data["content"]), :ok <- validate_page_url(page_url), {:ok, rich_media} <- Parser.parse(page_url) do diff --git a/lib/pleroma/web/router.ex b/lib/pleroma/web/router.ex index 552778c92..352268b96 100644 --- a/lib/pleroma/web/router.ex +++ b/lib/pleroma/web/router.ex @@ -352,6 +352,9 @@ defmodule Pleroma.Web.Router do post("/pleroma/flavour/:flavour", MastodonAPIController, :set_flavour) + get("/pleroma/mascot", MastodonAPIController, :get_mascot) + put("/pleroma/mascot", MastodonAPIController, :set_mascot) + post("/reports", MastodonAPIController, :reports) end @@ -712,6 +715,7 @@ defmodule Pleroma.Web.Router do scope "/", Fallback do get("/registration/:token", RedirectController, :registration_page) get("/:maybe_nickname_or_id", RedirectController, :redirector_with_meta) + get("/api*path", RedirectController, :api_not_implemented) get("/*path", RedirectController, :redirector) options("/*path", RedirectController, :empty) @@ -723,6 +727,12 @@ defmodule Fallback.RedirectController do alias Pleroma.User alias Pleroma.Web.Metadata + def api_not_implemented(conn, _params) do + conn + |> put_status(404) + |> json(%{error: "Not implemented"}) + end + def redirector(conn, _params, code \\ 200) do conn |> put_resp_content_type("text/html") diff --git a/lib/pleroma/web/twitter_api/views/activity_view.ex b/lib/pleroma/web/twitter_api/views/activity_view.ex index 44bcafe0e..e84af84dc 100644 --- a/lib/pleroma/web/twitter_api/views/activity_view.ex +++ b/lib/pleroma/web/twitter_api/views/activity_view.ex @@ -284,6 +284,12 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) ) + thread_muted? = + case activity.thread_muted? do + thread_muted? when is_boolean(thread_muted?) -> thread_muted? + nil -> CommonAPI.thread_muted?(user, activity) + end + %{ "id" => activity.id, "uri" => object.data["id"], @@ -314,7 +320,7 @@ defmodule Pleroma.Web.TwitterAPI.ActivityView do "summary" => summary, "summary_html" => summary |> Formatter.emojify(object.data["emoji"]), "card" => card, - "muted" => CommonAPI.thread_muted?(user, activity) || User.mutes?(opts[:for], user) + "muted" => thread_muted? || User.mutes?(opts[:for], user) } end diff --git a/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs new file mode 100644 index 000000000..b6a24441a --- /dev/null +++ b/priv/repo/optional_migrations/rum_indexing/20190510135645_add_fts_index_to_objects_two.exs @@ -0,0 +1,34 @@ +defmodule Pleroma.Repo.Migrations.AddFtsIndexToObjectsTwo do + use Ecto.Migration + + def up do + execute("create extension if not exists rum") + drop_if_exists index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + alter table(:objects) do + add(:fts_content, :tsvector) + end + + execute("CREATE FUNCTION objects_fts_update() RETURNS trigger AS $$ + begin + new.fts_content := to_tsvector('english', new.data->>'content'); + return new; + end + $$ LANGUAGE plpgsql") + execute("create index objects_fts on objects using RUM (fts_content rum_tsvector_addon_ops, inserted_at) with (attach = 'inserted_at', to = 'fts_content');") + + execute("CREATE TRIGGER tsvectorupdate BEFORE INSERT OR UPDATE ON objects + FOR EACH ROW EXECUTE PROCEDURE objects_fts_update()") + + execute("UPDATE objects SET updated_at = NOW()") + end + + def down do + execute "drop index objects_fts" + execute "drop trigger tsvectorupdate on objects" + execute "drop function objects_fts_update()" + alter table(:objects) do + remove(:fts_content, :tsvector) + end + create index(:objects, ["(to_tsvector('english', data->>'content'))"], using: :gin, name: :objects_fts) + end +end diff --git a/test/activity_test.exs b/test/activity_test.exs index 7e91d534b..15c95502a 100644 --- a/test/activity_test.exs +++ b/test/activity_test.exs @@ -6,6 +6,7 @@ defmodule Pleroma.ActivityTest do use Pleroma.DataCase alias Pleroma.Activity alias Pleroma.Bookmark + alias Pleroma.ThreadMute import Pleroma.Factory test "returns an activity by it's AP id" do @@ -47,6 +48,31 @@ defmodule Pleroma.ActivityTest do assert queried_activity.bookmark == bookmark3 end + test "setting thread_muted?" do + activity = insert(:note_activity) + user = insert(:user) + annoyed_user = insert(:user) + {:ok, _} = ThreadMute.add_mute(annoyed_user.id, activity.data["context"]) + + activity_with_unset_thread_muted_field = + Ecto.Query.from(Activity) + |> Repo.one() + + activity_for_user = + Ecto.Query.from(Activity) + |> Activity.with_set_thread_muted_field(user) + |> Repo.one() + + activity_for_annoyed_user = + Ecto.Query.from(Activity) + |> Activity.with_set_thread_muted_field(annoyed_user) + |> Repo.one() + + assert activity_with_unset_thread_muted_field.thread_muted? == nil + assert activity_for_user.thread_muted? == false + assert activity_for_annoyed_user.thread_muted? == true + end + describe "getting a bookmark" do test "when association is loaded" do user = insert(:user) diff --git a/test/fixtures/sound.mp3 b/test/fixtures/sound.mp3 Binary files differnew file mode 100644 index 000000000..9f0f661a3 --- /dev/null +++ b/test/fixtures/sound.mp3 diff --git a/test/formatter_test.exs b/test/formatter_test.exs index 5e7011160..47b91b121 100644 --- a/test/formatter_test.exs +++ b/test/formatter_test.exs @@ -206,6 +206,15 @@ defmodule Pleroma.FormatterTest do assert mentions == [] assert expected_text == text end + + test "given the 'safe_mention' option, it will keep text after newlines" do + user = insert(:user) + text = " @#{user.nickname}\n hey dude\n\nhow are you doing?" + + {expected_text, _, _} = Formatter.linkify(text, safe_mention: true) + + assert expected_text =~ "how are you doing?" + end end describe ".parse_tags" do diff --git a/test/user_test.exs b/test/user_test.exs index 10e463ff8..cb6afbe07 100644 --- a/test/user_test.exs +++ b/test/user_test.exs @@ -902,7 +902,7 @@ defmodule Pleroma.UserTest do assert [activity] == ActivityPub.fetch_public_activities(%{}) |> Repo.preload(:bookmark) - assert [activity] == + assert [%{activity | thread_muted?: CommonAPI.thread_muted?(user2, activity)}] == ActivityPub.fetch_activities([user2.ap_id | user2.following], %{"user" => user2}) {:ok, _user} = User.deactivate(user) diff --git a/test/web/activity_pub/mrf/simple_policy_test.exs b/test/web/activity_pub/mrf/simple_policy_test.exs index 1e0511975..74af7dcde 100644 --- a/test/web/activity_pub/mrf/simple_policy_test.exs +++ b/test/web/activity_pub/mrf/simple_policy_test.exs @@ -15,6 +15,7 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do media_removal: [], media_nsfw: [], federated_timeline_removal: [], + report_removal: [], reject: [], accept: [] ) @@ -85,6 +86,33 @@ defmodule Pleroma.Web.ActivityPub.MRF.SimplePolicyTest do } end + describe "when :report_removal" do + test "is empty" do + Config.put([:mrf_simple, :report_removal], []) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:ok, report_message} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + + test "has a matching host" do + Config.put([:mrf_simple, :report_removal], ["remote.instance"]) + report_message = build_report_message() + local_message = build_local_message() + + assert SimplePolicy.filter(report_message) == {:reject, nil} + assert SimplePolicy.filter(local_message) == {:ok, local_message} + end + end + + defp build_report_message do + %{ + "actor" => "https://remote.instance/users/bob", + "type" => "Flag" + } + end + describe "when :federated_timeline_removal" do test "is empty" do Config.put([:mrf_simple, :federated_timeline_removal], []) diff --git a/test/web/admin_api/admin_api_controller_test.exs b/test/web/admin_api/admin_api_controller_test.exs index ca12c7215..c15c67e31 100644 --- a/test/web/admin_api/admin_api_controller_test.exs +++ b/test/web/admin_api/admin_api_controller_test.exs @@ -397,14 +397,14 @@ defmodule Pleroma.Web.AdminAPI.AdminAPIControllerTest do end end - test "/api/pleroma/admin/invite_token" do + test "/api/pleroma/admin/users/invite_token" do admin = insert(:user, info: %{is_admin: true}) conn = build_conn() |> assign(:user, admin) |> put_req_header("accept", "application/json") - |> get("/api/pleroma/admin/invite_token") + |> get("/api/pleroma/admin/users/invite_token") assert conn.status == 200 end diff --git a/test/web/fallback_test.exs b/test/web/fallback_test.exs new file mode 100644 index 000000000..cc78b3ae1 --- /dev/null +++ b/test/web/fallback_test.exs @@ -0,0 +1,52 @@ +# Pleroma: A lightweight social networking server +# Copyright © 2017-2019 Pleroma Authors <https://pleroma.social/> +# SPDX-License-Identifier: AGPL-3.0-only + +defmodule Pleroma.Web.FallbackTest do + use Pleroma.Web.ConnCase + import Pleroma.Factory + + test "GET /registration/:token", %{conn: conn} do + assert conn + |> get("/registration/foo") + |> html_response(200) =~ "<!--server-generated-meta-->" + end + + test "GET /:maybe_nickname_or_id", %{conn: conn} do + user = insert(:user) + + assert conn + |> get("/foo") + |> html_response(200) =~ "<!--server-generated-meta-->" + + refute conn + |> get("/" <> user.nickname) + |> html_response(200) =~ "<!--server-generated-meta-->" + end + + test "GET /api*path", %{conn: conn} do + assert conn + |> get("/api/foo") + |> json_response(404) == %{"error" => "Not implemented"} + end + + test "GET /*path", %{conn: conn} do + assert conn + |> get("/foo") + |> html_response(200) =~ "<!--server-generated-meta-->" + + assert conn + |> get("/foo/bar") + |> html_response(200) =~ "<!--server-generated-meta-->" + end + + test "OPTIONS /*path", %{conn: conn} do + assert conn + |> options("/foo") + |> response(204) == "" + + assert conn + |> options("/foo/bar") + |> response(204) == "" + end +end diff --git a/test/web/mastodon_api/mastodon_api_controller_test.exs b/test/web/mastodon_api/mastodon_api_controller_test.exs index cbff141c8..1d9f5a816 100644 --- a/test/web/mastodon_api/mastodon_api_controller_test.exs +++ b/test/web/mastodon_api/mastodon_api_controller_test.exs @@ -1455,6 +1455,72 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIControllerTest do assert object.data["actor"] == User.ap_id(user) end + test "mascot upload", %{conn: conn} do + user = insert(:user) + + non_image_file = %Plug.Upload{ + content_type: "audio/mpeg", + path: Path.absname("test/fixtures/sound.mp3"), + filename: "sound.mp3" + } + + conn = + conn + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => non_image_file}) + + assert json_response(conn, 415) + + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + conn = + build_conn() + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => file}) + + assert %{"id" => _, "type" => image} = json_response(conn, 200) + end + + test "mascot retrieving", %{conn: conn} do + user = insert(:user) + # When user hasn't set a mascot, we should just get pleroma tan back + conn = + conn + |> assign(:user, user) + |> get("/api/v1/pleroma/mascot") + + assert %{"url" => url} = json_response(conn, 200) + assert url =~ "pleroma-fox-tan-smol" + + # When a user sets their mascot, we should get that back + file = %Plug.Upload{ + content_type: "image/jpg", + path: Path.absname("test/fixtures/image.jpg"), + filename: "an_image.jpg" + } + + conn = + build_conn() + |> assign(:user, user) + |> put("/api/v1/pleroma/mascot", %{"file" => file}) + + assert json_response(conn, 200) + + user = User.get_cached_by_id(user.id) + + conn = + build_conn() + |> assign(:user, user) + |> get("/api/v1/pleroma/mascot") + + assert %{"url" => url, "type" => "image"} = json_response(conn, 200) + assert url =~ "an_image" + end + test "hashtag timeline", %{conn: conn} do following = insert(:user) diff --git a/test/web/rich_media/helpers_test.exs b/test/web/rich_media/helpers_test.exs index 60d93768f..53b0596f5 100644 --- a/test/web/rich_media/helpers_test.exs +++ b/test/web/rich_media/helpers_test.exs @@ -1,6 +1,7 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do use Pleroma.DataCase + alias Pleroma.Object alias Pleroma.Web.CommonAPI import Pleroma.Factory @@ -59,4 +60,43 @@ defmodule Pleroma.Web.RichMedia.HelpersTest do Pleroma.Config.put([:rich_media, :enabled], false) end + + test "refuses to crawl URLs from posts marked sensitive" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "http://example.com/ogp", + "sensitive" => true + }) + + %Object{} = object = Object.normalize(activity) + + assert object.data["sensitive"] + + Pleroma.Config.put([:rich_media, :enabled], true) + + assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + + Pleroma.Config.put([:rich_media, :enabled], false) + end + + test "refuses to crawl URLs from posts tagged NSFW" do + user = insert(:user) + + {:ok, activity} = + CommonAPI.post(user, %{ + "status" => "http://example.com/ogp #nsfw" + }) + + %Object{} = object = Object.normalize(activity) + + assert object.data["sensitive"] + + Pleroma.Config.put([:rich_media, :enabled], true) + + assert %{} = Pleroma.Web.RichMedia.Helpers.fetch_data_for_activity(activity) + + Pleroma.Config.put([:rich_media, :enabled], false) + end end |