diff options
19 files changed, 130 insertions, 48 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aad28a2d8..b4bd59b43 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: elixir:1.8.1 +image: elixir:1.9.4 variables: &global_variables POSTGRES_DB: pleroma_test @@ -170,8 +170,7 @@ stop_review_app: amd64: stage: release - # TODO: Replace with upstream image when 1.9.0 comes out - image: rinpatch/elixir:1.9.0-rc.0 + image: elixir:1.10.3 only: &release-only - stable@pleroma/pleroma - develop@pleroma/pleroma @@ -208,8 +207,7 @@ amd64-musl: stage: release artifacts: *release-artifacts only: *release-only - # TODO: Replace with upstream image when 1.9.0 comes out - image: rinpatch/elixir:1.9.0-rc.0-alpine + image: elixir:1.10.3-alpine cache: *release-cache variables: *release-variables before_script: &before-release-musl @@ -225,8 +223,7 @@ arm: only: *release-only tags: - arm32 - # TODO: Replace with upstream image when 1.9.0 comes out - image: rinpatch/elixir:1.9.0-rc.0-arm + image: elixir:1.10.3 cache: *release-cache variables: *release-variables before_script: *before-release @@ -238,8 +235,7 @@ arm-musl: only: *release-only tags: - arm32 - # TODO: Replace with upstream image when 1.9.0 comes out - image: rinpatch/elixir:1.9.0-rc.0-arm-alpine + image: elixir:1.10.3-alpine cache: *release-cache variables: *release-variables before_script: *before-release-musl @@ -251,8 +247,7 @@ arm64: only: *release-only tags: - arm - # TODO: Replace with upstream image when 1.9.0 comes out - image: rinpatch/elixir:1.9.0-rc.0-arm64 + image: elixir:1.10.3 cache: *release-cache variables: *release-variables before_script: *before-release @@ -265,7 +260,7 @@ arm64-musl: tags: - arm # TODO: Replace with upstream image when 1.9.0 comes out - image: rinpatch/elixir:1.9.0-rc.0-arm64-alpine + image: elixir:1.10.3-alpine cache: *release-cache variables: *release-variables before_script: *before-release-musl diff --git a/CHANGELOG.md b/CHANGELOG.md index d2629bf84..3ee13904f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## [unreleased] ### Changed +- **Breaking:** Elixir >=1.9 is now required (was >= 1.8) +- In Conversations, return only direct messages as `last_status` - MFR policy to set global expiration for all local Create activities - OGP rich media parser merged with TwitterCard <details> diff --git a/elixir_buildpack.config b/elixir_buildpack.config index c23b08fb8..946408c12 100644 --- a/elixir_buildpack.config +++ b/elixir_buildpack.config @@ -1,2 +1,2 @@ -elixir_version=1.8.2 -erlang_version=21.3.7 +elixir_version=1.9.4 +erlang_version=22.3.4.1 diff --git a/lib/pleroma/conversation/participation.ex b/lib/pleroma/conversation/participation.ex index ce7bd2396..8bc3e85d6 100644 --- a/lib/pleroma/conversation/participation.ex +++ b/lib/pleroma/conversation/participation.ex @@ -162,10 +162,13 @@ defmodule Pleroma.Conversation.Participation do for_user(user, params) |> Enum.map(fn participation -> activity_id = - ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ - user: user, - blocking_user: user - }) + ActivityPub.fetch_latest_direct_activity_id_for_context( + participation.conversation.ap_id, + %{ + user: user, + blocking_user: user + } + ) %{ participation diff --git a/lib/pleroma/migration_helper/notification_backfill.ex b/lib/pleroma/migration_helper/notification_backfill.ex index 09647d12a..b3770307a 100644 --- a/lib/pleroma/migration_helper/notification_backfill.ex +++ b/lib/pleroma/migration_helper/notification_backfill.ex @@ -18,7 +18,7 @@ defmodule Pleroma.MigrationHelper.NotificationBackfill do ) query - |> Repo.all() + |> Repo.chunk_stream(100) |> Enum.each(fn notification -> type = notification.activity diff --git a/lib/pleroma/repo.ex b/lib/pleroma/repo.ex index f62138466..6d85d70bc 100644 --- a/lib/pleroma/repo.ex +++ b/lib/pleroma/repo.ex @@ -8,6 +8,7 @@ defmodule Pleroma.Repo do adapter: Ecto.Adapters.Postgres, migration_timestamps: [type: :naive_datetime_usec] + import Ecto.Query require Logger defmodule Instrumenter do @@ -78,6 +79,33 @@ defmodule Pleroma.Repo do :ok end end + + def chunk_stream(query, chunk_size) do + # We don't actually need start and end funcitons of resource streaming, + # but it seems to be the only way to not fetch records one-by-one and + # have individual records be the elements of the stream, instead of + # lists of records + Stream.resource( + fn -> 0 end, + fn + last_id -> + query + |> order_by(asc: :id) + |> where([r], r.id > ^last_id) + |> limit(^chunk_size) + |> all() + |> case do + [] -> + {:halt, last_id} + + records -> + last_id = List.last(records).id + {records, last_id} + end + end, + fn _ -> :ok end + ) + end end defmodule Pleroma.Repo.UnappliedMigrationsError do diff --git a/lib/pleroma/user.ex b/lib/pleroma/user.ex index 686ab0123..19ce9fb56 100644 --- a/lib/pleroma/user.ex +++ b/lib/pleroma/user.ex @@ -79,6 +79,7 @@ defmodule Pleroma.User do schema "users" do field(:bio, :string) + field(:raw_bio, :string) field(:email, :string) field(:name, :string) field(:nickname, :string) @@ -432,6 +433,7 @@ defmodule Pleroma.User do params, [ :bio, + :raw_bio, :name, :emoji, :avatar, @@ -607,7 +609,16 @@ defmodule Pleroma.User do struct |> confirmation_changeset(need_confirmation: need_confirmation?) - |> cast(params, [:bio, :email, :name, :nickname, :password, :password_confirmation, :emoji]) + |> cast(params, [ + :bio, + :raw_bio, + :email, + :name, + :nickname, + :password, + :password_confirmation, + :emoji + ]) |> validate_required([:name, :nickname, :password, :password_confirmation]) |> validate_confirmation(:password) |> unique_constraint(:email) diff --git a/lib/pleroma/web/activity_pub/activity_pub.ex b/lib/pleroma/web/activity_pub/activity_pub.ex index c9dc6135c..3e4f3ad30 100644 --- a/lib/pleroma/web/activity_pub/activity_pub.ex +++ b/lib/pleroma/web/activity_pub/activity_pub.ex @@ -210,7 +210,7 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do conversation = Repo.preload(conversation, :participations) last_activity_id = - fetch_latest_activity_id_for_context(conversation.ap_id, %{ + fetch_latest_direct_activity_id_for_context(conversation.ap_id, %{ user: user, blocking_user: user }) @@ -517,11 +517,12 @@ defmodule Pleroma.Web.ActivityPub.ActivityPub do |> Repo.all() end - @spec fetch_latest_activity_id_for_context(String.t(), keyword() | map()) :: + @spec fetch_latest_direct_activity_id_for_context(String.t(), keyword() | map()) :: FlakeId.Ecto.CompatType.t() | nil - def fetch_latest_activity_id_for_context(context, opts \\ %{}) do + def fetch_latest_direct_activity_id_for_context(context, opts \\ %{}) do context |> fetch_activities_for_context_query(Map.merge(%{skip_preload: true}, opts)) + |> restrict_visibility(%{visibility: "direct"}) |> limit(1) |> select([a], a.id) |> Repo.one() diff --git a/lib/pleroma/web/api_spec/operations/notification_operation.ex b/lib/pleroma/web/api_spec/operations/notification_operation.ex index c966b553a..41328b5f2 100644 --- a/lib/pleroma/web/api_spec/operations/notification_operation.ex +++ b/lib/pleroma/web/api_spec/operations/notification_operation.ex @@ -183,7 +183,6 @@ defmodule Pleroma.Web.ApiSpec.NotificationOperation do "favourite", "reblog", "mention", - "poll", "pleroma:emoji_reaction", "pleroma:chat_mention", "move", diff --git a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex index 7cdd8f458..c38c2b895 100644 --- a/lib/pleroma/web/mastodon_api/controllers/account_controller.ex +++ b/lib/pleroma/web/mastodon_api/controllers/account_controller.ex @@ -165,6 +165,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountController do end) |> Maps.put_if_present(:name, params[:display_name]) |> Maps.put_if_present(:bio, params[:note]) + |> Maps.put_if_present(:raw_bio, params[:note]) |> Maps.put_if_present(:avatar, params[:avatar]) |> Maps.put_if_present(:banner, params[:header]) |> Maps.put_if_present(:background, params[:pleroma_background_image]) diff --git a/lib/pleroma/web/mastodon_api/views/account_view.ex b/lib/pleroma/web/mastodon_api/views/account_view.ex index 9fc06bf9d..68beb69b8 100644 --- a/lib/pleroma/web/mastodon_api/views/account_view.ex +++ b/lib/pleroma/web/mastodon_api/views/account_view.ex @@ -224,7 +224,7 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do fields: user.fields, bot: bot, source: %{ - note: prepare_user_bio(user), + note: user.raw_bio || "", sensitive: false, fields: user.raw_fields, pleroma: %{ @@ -260,17 +260,6 @@ defmodule Pleroma.Web.MastodonAPI.AccountView do |> maybe_put_unread_notification_count(user, opts[:for]) end - defp prepare_user_bio(%User{bio: ""}), do: "" - - defp prepare_user_bio(%User{bio: bio}) when is_binary(bio) do - bio - |> String.replace(~r(<br */?>), "\n") - |> Pleroma.HTML.strip_tags() - |> HtmlEntities.decode() - end - - defp prepare_user_bio(_), do: "" - defp username_from_nickname(string) when is_binary(string) do hd(String.split(string, "@")) end diff --git a/lib/pleroma/web/mastodon_api/views/conversation_view.ex b/lib/pleroma/web/mastodon_api/views/conversation_view.ex index fbe618377..06f0c1728 100644 --- a/lib/pleroma/web/mastodon_api/views/conversation_view.ex +++ b/lib/pleroma/web/mastodon_api/views/conversation_view.ex @@ -23,10 +23,13 @@ defmodule Pleroma.Web.MastodonAPI.ConversationView do last_activity_id = with nil <- participation.last_activity_id do - ActivityPub.fetch_latest_activity_id_for_context(participation.conversation.ap_id, %{ - user: user, - blocking_user: user - }) + ActivityPub.fetch_latest_direct_activity_id_for_context( + participation.conversation.ap_id, + %{ + user: user, + blocking_user: user + } + ) end activity = Activity.get_by_id_with_object(last_activity_id) @@ -5,7 +5,7 @@ defmodule Pleroma.Mixfile do [ app: :pleroma, version: version("2.0.50"), - elixir: "~> 1.8", + elixir: "~> 1.9", elixirc_paths: elixirc_paths(Mix.env()), compilers: [:phoenix, :gettext] ++ Mix.compilers(), elixirc_options: [warnings_as_errors: warnings_as_errors(Mix.env())], diff --git a/priv/repo/migrations/20200322174133_user_raw_bio.exs b/priv/repo/migrations/20200322174133_user_raw_bio.exs new file mode 100644 index 000000000..ddf9be4f5 --- /dev/null +++ b/priv/repo/migrations/20200322174133_user_raw_bio.exs @@ -0,0 +1,9 @@ +defmodule Pleroma.Repo.Migrations.UserRawBio do + use Ecto.Migration + + def change do + alter table(:users) do + add_if_not_exists(:raw_bio, :text) + end + end +end diff --git a/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs b/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs new file mode 100644 index 000000000..cb35db3f5 --- /dev/null +++ b/priv/repo/migrations/20200328193433_populate_user_raw_bio.exs @@ -0,0 +1,25 @@ +defmodule Pleroma.Repo.Migrations.PopulateUserRawBio do + use Ecto.Migration + import Ecto.Query + alias Pleroma.User + alias Pleroma.Repo + + def change do + {:ok, _} = Application.ensure_all_started(:fast_sanitize) + + User.Query.build(%{local: true}) + |> select([u], struct(u, [:id, :ap_id, :bio])) + |> Repo.stream() + |> Enum.each(fn %{bio: bio} = user -> + if bio do + raw_bio = + bio + |> String.replace(~r(<br */?>), "\n") + |> Pleroma.HTML.strip_tags() + + Ecto.Changeset.cast(user, %{raw_bio: raw_bio}, [:raw_bio]) + |> Repo.update() + end + end) + end +end diff --git a/test/support/factory.ex b/test/support/factory.ex index e517d5bc6..6e22b66a4 100644 --- a/test/support/factory.ex +++ b/test/support/factory.ex @@ -42,7 +42,8 @@ defmodule Pleroma.Factory do user | ap_id: User.ap_id(user), follower_address: User.ap_followers(user), - following_address: User.ap_following(user) + following_address: User.ap_following(user), + raw_bio: user.bio } end diff --git a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs index 7c420985d..76e6d603a 100644 --- a/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs +++ b/test/web/mastodon_api/controllers/account_controller/update_credentials_test.exs @@ -83,10 +83,9 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do test "updates the user's bio", %{conn: conn} do user2 = insert(:user) - conn = - patch(conn, "/api/v1/accounts/update_credentials", %{ - "note" => "I drink #cofe with @#{user2.nickname}\n\nsuya.." - }) + raw_bio = "I drink #cofe with @#{user2.nickname}\n\nsuya.." + + conn = patch(conn, "/api/v1/accounts/update_credentials", %{"note" => raw_bio}) assert user_data = json_response_and_validate_schema(conn, 200) @@ -94,6 +93,12 @@ defmodule Pleroma.Web.MastodonAPI.MastodonAPIController.UpdateCredentialsTest do ~s(I drink <a class="hashtag" data-tag="cofe" href="http://localhost:4001/tag/cofe">#cofe</a> with <span class="h-card"><a class="u-url mention" data-user="#{ user2.id }" href="#{user2.ap_id}" rel="ugc">@<span>#{user2.nickname}</span></a></span><br/><br/>suya..) + + assert user_data["source"]["note"] == raw_bio + + user = Repo.get(User, user_data["id"]) + + assert user.raw_bio == raw_bio end test "updates the user's locking status", %{conn: conn} do diff --git a/test/web/mastodon_api/views/account_view_test.exs b/test/web/mastodon_api/views/account_view_test.exs index 044f088a4..80b1f734c 100644 --- a/test/web/mastodon_api/views/account_view_test.exs +++ b/test/web/mastodon_api/views/account_view_test.exs @@ -33,7 +33,8 @@ defmodule Pleroma.Web.MastodonAPI.AccountViewTest do bio: "<script src=\"invalid-html\"></script><span>valid html</span>. a<br>b<br/>c<br >d<br />f '&<>\"", inserted_at: ~N[2017-08-15 15:47:06.597036], - emoji: %{"karjalanpiirakka" => "/file.png"} + emoji: %{"karjalanpiirakka" => "/file.png"}, + raw_bio: "valid html. a\nb\nc\nd\nf '&<>\"" }) expected = %{ diff --git a/test/web/mastodon_api/views/conversation_view_test.exs b/test/web/mastodon_api/views/conversation_view_test.exs index 6f84366f8..2e8203c9b 100644 --- a/test/web/mastodon_api/views/conversation_view_test.exs +++ b/test/web/mastodon_api/views/conversation_view_test.exs @@ -15,8 +15,17 @@ defmodule Pleroma.Web.MastodonAPI.ConversationViewTest do user = insert(:user) other_user = insert(:user) + {:ok, parent} = CommonAPI.post(user, %{status: "parent"}) + {:ok, activity} = - CommonAPI.post(user, %{status: "hey @#{other_user.nickname}", visibility: "direct"}) + CommonAPI.post(user, %{ + status: "hey @#{other_user.nickname}", + visibility: "direct", + in_reply_to_id: parent.id + }) + + {:ok, _reply_activity} = + CommonAPI.post(user, %{status: "hu", visibility: "public", in_reply_to_id: parent.id}) [participation] = Participation.for_user_with_last_activity_id(user) |